├── .coveragerc ├── .editorconfig ├── .git-blame-ignore-revs ├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── AUTHORS.txt ├── CHANGELOG.rst ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── README.rst ├── cogapp ├── __init__.py ├── __main__.py ├── cogapp.py ├── hashhandler.py ├── makefiles.py ├── test_cogapp.py ├── test_makefiles.py ├── test_whiteutils.py ├── utils.py └── whiteutils.py ├── docs ├── Makefile ├── changes.rst ├── conf.py ├── design.rst ├── index.rst ├── module.rst ├── running.rst └── source.rst ├── pyproject.toml ├── requirements.pip ├── success ├── README.txt ├── cog-success.rst └── cog.png └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | # coverage configuration for Cog. 2 | [run] 3 | branch = True 4 | parallel = True 5 | source = cogapp 6 | 7 | [report] 8 | exclude_lines = 9 | pragma: no cover 10 | raise CogInternalError\( 11 | precision = 2 12 | 13 | [html] 14 | title = Cog coverage 15 | 16 | [paths] 17 | source = 18 | cogapp 19 | # GitHub Actions uses a few different home dir styles 20 | */cog/cogapp 21 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs. 2 | # More information at http://EditorConfig.org 3 | 4 | root = true 5 | 6 | [*] 7 | charset = utf-8 8 | end_of_line = lf 9 | indent_size = 4 10 | indent_style = space 11 | insert_final_newline = true 12 | max_line_length = 80 13 | trim_trailing_whitespace = true 14 | 15 | [*.py] 16 | max_line_length = 100 17 | 18 | [*.yml] 19 | indent_size = 2 20 | 21 | [*.rst] 22 | max_line_length = 79 23 | 24 | [Makefile] 25 | indent_style = tab 26 | indent_size = 8 27 | 28 | [*,cover] 29 | trim_trailing_whitespace = false 30 | 31 | [*.diff] 32 | trim_trailing_whitespace = false 33 | 34 | [.git/*] 35 | trim_trailing_whitespace = false 36 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Commits to ignore when doing git-blame. 2 | 3 | # 2024-04-24 refactor: reformat Python code via ruff 4 | e591c4d3557974f1aa41070c60f1c95e2f949b75 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # From: 2 | # https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot 3 | # Set update schedule for GitHub Actions 4 | 5 | version: 2 6 | updates: 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | # Check for updates to GitHub Actions once a week 11 | interval: "weekly" 12 | groups: 13 | action-dependencies: 14 | patterns: 15 | - "*" 16 | commit-message: 17 | prefix: "chore" 18 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: "CI" 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | defaults: 8 | run: 9 | shell: bash 10 | 11 | permissions: 12 | contents: read 13 | 14 | concurrency: 15 | group: "${{ github.workflow }}-${{ github.ref }}" 16 | cancel-in-progress: true 17 | 18 | jobs: 19 | 20 | format: 21 | runs-on: ubuntu-latest 22 | steps: 23 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 24 | with: 25 | persist-credentials: false 26 | 27 | - uses: astral-sh/ruff-action@84f83ecf9e1e15d26b7984c7ec9cf73d39ffc946 # v3.3.1 28 | with: 29 | args: 'format --check' 30 | 31 | lint: 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 35 | with: 36 | persist-credentials: false 37 | 38 | - uses: astral-sh/ruff-action@84f83ecf9e1e15d26b7984c7ec9cf73d39ffc946 # v3.3.1 39 | 40 | - name: "Set up Python" 41 | uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 42 | with: 43 | python-version: "3.13" 44 | 45 | - name: "Install dependencies" 46 | run: | 47 | python -m pip install -r requirements.pip 48 | 49 | - name: "Check the docs are up-to-date" 50 | run: | 51 | make lintdoc 52 | 53 | tests: 54 | name: "Python ${{ matrix.python }} on ${{ matrix.os }}" 55 | runs-on: "${{ matrix.os }}-${{ matrix.os-version || 'latest' }}" 56 | env: 57 | MATRIX_ID: "${{ matrix.python }}.${{ matrix.os }}" 58 | 59 | strategy: 60 | fail-fast: false 61 | matrix: 62 | os: 63 | - ubuntu 64 | - macos 65 | - windows 66 | python: 67 | # When changing this list, be sure to check the [gh] list in 68 | # tox.ini so that tox will run properly. 69 | - "3.9" 70 | - "3.10" 71 | - "3.11" 72 | - "3.12" 73 | - "3.13" 74 | include: 75 | - python: "3.9" 76 | os: "macos" 77 | os-version: "13" 78 | 79 | steps: 80 | - name: "Check out the repo" 81 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 82 | with: 83 | persist-credentials: false 84 | 85 | - name: "Set up Python" 86 | uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 87 | with: 88 | python-version: "${{ matrix.python }}" 89 | 90 | - name: "Install dependencies" 91 | run: | 92 | python -m pip install -r requirements.pip 93 | 94 | - name: "Run tox for ${{ matrix.python }}" 95 | run: | 96 | python -m tox 97 | python -m coverage debug data 98 | 99 | - name: "Upload coverage data" 100 | uses: actions/upload-artifact@v4 101 | with: 102 | name: covdata-${{ env.MATRIX_ID }} 103 | path: .coverage.* 104 | include-hidden-files: true 105 | 106 | combine: 107 | name: "Combine and report coverage" 108 | needs: tests 109 | runs-on: ubuntu-latest 110 | 111 | steps: 112 | - name: "Check out the repo" 113 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 114 | with: 115 | fetch-depth: "0" 116 | persist-credentials: false 117 | 118 | - name: "Set up Python" 119 | uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 120 | with: 121 | python-version: "3.9" 122 | 123 | - name: "Install dependencies" 124 | run: | 125 | python -m pip install -r requirements.pip 126 | 127 | - name: "Download coverage data" 128 | uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 129 | with: 130 | pattern: covdata-* 131 | merge-multiple: true 132 | 133 | - name: "Combine and report" 134 | run: | 135 | python -m coverage combine 136 | python -m coverage report -m 137 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Files that can appear anywhere in the tree. 2 | *.pyc 3 | *.pyo 4 | *.pyd 5 | *$py.class 6 | *.bak 7 | 8 | # Stuff in the root. 9 | build 10 | dist 11 | .coverage 12 | .coverage.* 13 | coverage.xml 14 | htmlcov 15 | MANIFEST 16 | setuptools-*.egg 17 | cogapp.egg-info 18 | .tox 19 | .*cache 20 | 21 | docs/_build 22 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | 3 | - repo: https://github.com/astral-sh/ruff-pre-commit 4 | rev: v0.4.1 5 | hooks: 6 | - id: ruff-format 7 | - id: ruff 8 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # ReadTheDocs configuration. 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html 3 | 4 | version: 2 5 | 6 | build: 7 | os: ubuntu-22.04 8 | tools: 9 | python: "3.11" 10 | 11 | sphinx: 12 | builder: html 13 | configuration: docs/conf.py 14 | 15 | # Build all the formats 16 | formats: all 17 | 18 | python: 19 | install: 20 | - requirements: requirements.pip 21 | - method: pip 22 | path: . 23 | -------------------------------------------------------------------------------- /AUTHORS.txt: -------------------------------------------------------------------------------- 1 | Cog was written by Ned Batchelder (ned@nedbatchelder.com). 2 | 3 | Contributions have been made by: 4 | 5 | Alexander Belchenko 6 | Anders Hovmöller 7 | Blake Winton 8 | Daniel Murdin 9 | Doug Hellmann 10 | Hugh Perkins 11 | Jean-François Giraud 12 | Panayiotis Gavriil 13 | Petr Gladkiy 14 | Phil Kirkpatrick 15 | Ryan Santos 16 | Tim Vergenz 17 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | .. 5 | 6 | split out from the main page. 7 | 2.1: -u flag 8 | more 2.1 stuff 9 | add a pointer to the russian. 10 | started the 2.2 list. 11 | 2.2 12 | 2.3 13 | 2.4 14 | 2.5.1 15 | 3.0.0 16 | 3.1.0 17 | 18 | 19 | These are changes to Cog over time. 20 | 21 | Unreleased 22 | ---------- 23 | 24 | - Added a ``--diff`` option to show the diff of what changed to fail a 25 | ``--check`` run. 26 | 27 | - Embedded code can change the current directory, cog will change back to the 28 | original directory when the code is done. 29 | 30 | - Changed the checksum format to use shorter base64 encoding instead of hex, 31 | making checksums less visually distracting. The old hex format will still be 32 | accepted, but will be updated to the new format automatically when writing. 33 | 34 | - Added ``--help`` option as an alias for ``-h``. 35 | 36 | - Dropped support for Python 3.7 and 3.8, and added 3.13. 37 | 38 | 39 | 3.4.1 – March 7 2024 40 | -------------------- 41 | 42 | - Dropped support for Python 2.7, 3.5, and 3.6, and added 3.11 and 3.12. 43 | 44 | - Removed the ``cog.py`` installed file. Use the ``cog`` command, or ``python 45 | -m cogapp`` to run cog. 46 | 47 | - Processing long files has been made much faster. Thanks, Panayiotis Gavriil. 48 | 49 | - Files listing other files to process can now be specified as 50 | ``&files_to_cog.txt`` to interpret the file names relative to the location of 51 | the list file. The existing ``@files_to_cog.txt`` syntax interprets file 52 | names relative to the current working directory. Thanks, Phil Kirkpatrick. 53 | 54 | - Support FIPS mode computers by marking our MD5 use as not related to 55 | security. Thanks, Ryan Santos. 56 | 57 | - Docs have moved to https://cog.readthedocs.io 58 | 59 | 60 | 3.3.0 – November 19 2021 61 | ------------------------ 62 | 63 | - Added the ``--check`` option to check whether files would change if run 64 | again, for use in continuous integration scenarios. 65 | 66 | 67 | 3.2.0 – November 7 2021 68 | ----------------------- 69 | 70 | - Added the ``-P`` option to use `print()` instead of `cog.outl()` for code 71 | output. 72 | 73 | 74 | 3.1.0 – August 31 2021 75 | ---------------------- 76 | 77 | - Fix a problem with Python 3.8.10 and 3.9.5 that require absolute paths in 78 | sys.path. `issue 16`_. 79 | 80 | - Python 3.9 and 3.10 are supported. 81 | 82 | .. _issue 16: https://github.com/nedbat/cog/issues/16 83 | 84 | 85 | 3.0.0 – April 2 2019 86 | -------------------- 87 | 88 | - Dropped support for Pythons 2.6, 3.3, and 3.4. 89 | 90 | - Errors occurring during content generation now print accurate tracebacks, 91 | showing the correct filename, line number, and source line. 92 | 93 | - Cog can now (again?) be run as just "cog" on the command line. 94 | 95 | - The ``-p=PROLOGUE`` option was added to specify Python text to prepend to 96 | embedded code. Thanks, Anders Hovmöller. 97 | 98 | - Wildcards in command line arguments will be expanded by cog to help on 99 | Windows. Thanks, Hugh Perkins. 100 | 101 | - When using implicitly imported "cog", a new module is made for each run. 102 | This is important when using the cog API multi-threaded. Thanks, Daniel 103 | Murdin. 104 | 105 | - Moved development to GitHub. 106 | 107 | 108 | 2.5.1 – October 19 2016 109 | ----------------------- 110 | 111 | - Corrected a long-standing oversight: added a LICENSE.txt file. 112 | 113 | 2.5 – February 13 2016 114 | ---------------------- 115 | 116 | - When specifying an output file with ``-o``, directories will be created as 117 | needed to write the file. Thanks, Jean-François Giraud. 118 | 119 | 2.4 – January 11 2015 120 | --------------------- 121 | 122 | - A ``--markers`` option lets you control the three markers that separate the 123 | cog code and result from the rest of the file. Thanks, Doug Hellmann. 124 | 125 | - A ``-n=ENCODING`` option that lets you specify the encoding for the input and 126 | output files. Thanks, Petr Gladkiy. 127 | 128 | - A ``--verbose`` option that lets you control how much chatter is in the 129 | output while cogging. 130 | 131 | 2.3 – February 27 2012 132 | ---------------------- 133 | 134 | - Python 3 is now supported. Older Pythons (2.5 and below) no longer are. 135 | 136 | - Added the `cog.previous` attribute to get the output from the last time cog was 137 | run. 138 | 139 | - An input file name of "-" will read input from standard in. 140 | 141 | - Cog can now be run with "python3 -m cogapp [args]". 142 | 143 | - All files are assumed to be encoded with UTF-8. 144 | 145 | 146 | 2.2 – June 25 2009 147 | ------------------ 148 | 149 | - Jython 2.5 is now supported. 150 | 151 | - Removed a warning about using the no-longer-recommended md5 module. 152 | 153 | - Removed handyxml: most Cog users don't need it. 154 | 155 | 156 | 2.1 – May 22 2008 157 | ----------------- 158 | 159 | - Added the ``-U`` switch to create Unix newlines on Windows. 160 | 161 | - Improved argument validation: ``-d`` can be used with stdout-destined output, 162 | and switches are validated for every line of an @file, to prevent bad 163 | interactions. 164 | 165 | 166 | 2.0 – October 6 2005 167 | -------------------- 168 | 169 | Incompatible changes: 170 | 171 | - Python 2.2 is no longer supported. 172 | 173 | - In 1.4, you could put some generator code on the ``[[[cog`` line and some on 174 | the ``]]]`` line, to make the generators more compact. Unfortunately, this 175 | also made it more difficult to seamlessly embed those markers in source files 176 | of all kinds. Now code is only allowed on marker lines when the entire 177 | generator is single-line. 178 | 179 | - In 1.x, you could leave out the ``[[[end]]]`` marker, and it would be assumed 180 | at the end of the file. Now that behavior must be enabled with a ``-z`` 181 | switch. Without the switch, omitting the end marker is an error. 182 | 183 | Beneficial changes: 184 | 185 | - The new ``-d`` switch removes all the generator code from the output file 186 | while running it to generate output (thanks, Blake). 187 | 188 | - The new ``-D`` switch lets you define global string values for the 189 | generators. 190 | 191 | - The new ``-s`` switch lets you mark generated output lines with a suffix. 192 | 193 | - @-files now can have command line switches in addition to file names. 194 | 195 | - Cog error messages now print without a traceback, and use a format similar to 196 | compiler error messages, so that clicking the message will likely bring you 197 | to the spot in your code (thanks, Mike). 198 | 199 | - New cog method #1: `cog.error(msg)` will raise an error and end processing 200 | without creating a scary Python traceback (thanks, Alexander). 201 | 202 | - New cog method #2: `cog.msg(msg)` will print the msg to stdout. This is 203 | better than print because it allows for all cog output to be managed through 204 | Cog. 205 | 206 | - The sequence of Cog marker lines is much stricter. This helps to ensure that 207 | Cog isn't eating up your precious source code (thanks, Kevin). 208 | 209 | 210 | 211 | 1.4 – February 25 2005 212 | ---------------------- 213 | 214 | - Added the ``-x`` switch to excise generated output. 215 | 216 | - Added the ``-c`` switch to checksum the generated output. 217 | 218 | 219 | 220 | 1.3 – December 30 2004 221 | ---------------------- 222 | 223 | - All of the generators in a single file are now run with a common globals 224 | dictionary, so that state may be carried from one to the next. 225 | 226 | 227 | 228 | 1.2 – December 29 2004 229 | ---------------------- 230 | 231 | - Added module attributes `cog.inFile`, `cog.outFile`, and `cog.firstLineNum`. 232 | 233 | - Made the `sOut` argument optional in `cog.out` and `cog.outl`. 234 | 235 | - Added the compact one-line form of cog markers. 236 | 237 | - Some warning messages weren't properly printing the file name. 238 | 239 | 240 | 241 | 1.12 – June 21 2004 242 | ------------------- 243 | 244 | - Changed all the line endings in the source to the more-portable LF from the 245 | Windows-only CRLF. 246 | 247 | 248 | 249 | 1.11 – June 5 2004 250 | ------------------ 251 | 252 | Just bug fixes: 253 | 254 | - Cog's whitespace handling deals correctly with a completely blank line (no 255 | whitespace at all) in a chunk of Cog code. 256 | 257 | - Elements returned by handyxml can now have attributes assigned to them after 258 | parsing. 259 | 260 | 261 | 262 | 1.1 – March 21 2004 263 | ------------------- 264 | 265 | - Now if the cog marker lines and all the lines they contain have the same 266 | prefix characters, then the prefix is removed from each line. This allows 267 | cog to be used with languages that don't support multi-line comments. 268 | 269 | - Ensure the last line of the output ends with a newline, or it will merge with 270 | the end marker, ruining cog's idempotency. 271 | 272 | - Add the ``-v`` command line option, to print the version. 273 | 274 | - Running cog with no options prints the usage help. 275 | 276 | 277 | 278 | 1.0 – February 10 2004 279 | ---------------------- 280 | 281 | First version. 282 | 283 | .. 284 | # History moved from cogapp.py: 285 | # 20040210: First public version. 286 | # 20040220: Text preceding the start and end marker are removed from Python lines. 287 | # -v option on the command line shows the version. 288 | # 20040311: Make sure the last line of output is properly ended with a newline. 289 | # 20040605: Fixed some blank line handling in cog. 290 | # Fixed problems with assigning to xml elements in handyxml. 291 | # 20040621: Changed all line-ends to LF from CRLF. 292 | # 20041002: Refactor some option handling to simplify unittesting the options. 293 | # 20041118: cog.out and cog.outl have optional string arguments. 294 | # 20041119: File names weren't being properly passed around for warnings, etc. 295 | # 20041122: Added cog.firstLineNum: a property with the line number of the [[[cog line. 296 | # Added cog.inFile and cog.outFile: the names of the input and output file. 297 | # 20041218: Single-line cog generators, with start marker and end marker on 298 | # the same line. 299 | # 20041230: Keep a single globals dict for all the code fragments in a single 300 | # file so they can share state. 301 | # 20050206: Added the -x switch to remove all generated output. 302 | # 20050218: Now code can be on the marker lines as well. 303 | # 20050219: Added -c switch to checksum the output so that edits can be 304 | # detected before they are obliterated. 305 | # 20050521: Added cog.error, contributed by Alexander Belchenko. 306 | # 20050720: Added code deletion and settable globals contributed by Blake Winton. 307 | # 20050724: Many tweaks to improve code coverage. 308 | # 20050726: Error messages are now printed with no traceback. 309 | # Code can no longer appear on the marker lines, 310 | # except for single-line style. 311 | # -z allows omission of the [[[end]]] marker, and it will be assumed 312 | # at the end of the file. 313 | # 20050729: Refactor option parsing into a separate class, in preparation for 314 | # future features. 315 | # 20050805: The cogmodule.path wasn't being properly maintained. 316 | # 20050808: Added the -D option to define a global value. 317 | # 20050810: The %s in the -w command is dealt with more robustly. 318 | # Added the -s option to suffix output lines with a marker. 319 | # 20050817: Now @files can have arguments on each line to change the cog's 320 | # behavior for that line. 321 | # 20051006: Version 2.0 322 | # 20080521: -U options lets you create Unix newlines on Windows. Thanks, 323 | # Alexander Belchenko. 324 | # 20080522: It's now ok to have -d with output to stdout, and now we validate 325 | # the args after each line of an @file. 326 | # 20090520: Use hashlib where it's available, to avoid a warning. 327 | # Use the builtin compile() instead of compiler, for Jython. 328 | # Explicitly close files we opened, Jython likes this. 329 | # 20120205: Port to Python 3. Lowest supported version is 2.6. 330 | # 20150104: -markers option added by Doug Hellmann. 331 | # 20150104: -n ENCODING option added by Petr Gladkiy. 332 | # 20150107: Added -verbose to control what files get listed. 333 | # 20150111: Version 2.4 334 | # 20160213: v2.5: -o makes needed directories, thanks Jean-François Giraud. 335 | # 20161019: Added a LICENSE.txt file. 336 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2004-2024 Ned Batchelder 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include .coveragerc 2 | include .editorconfig 3 | include .git-blame-ignore-revs 4 | include .pre-commit-config.yaml 5 | include .readthedocs.yaml 6 | include CHANGELOG.rst 7 | include LICENSE.txt 8 | include Makefile 9 | include README.rst 10 | include requirements.pip 11 | include tox.ini 12 | 13 | recursive-include cogapp *.py 14 | recursive-include docs Makefile *.py *.rst 15 | recursive-include docs/_static * 16 | recursive-include success * 17 | 18 | prune doc/_build 19 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for cog work. 2 | 3 | # A command to get the current version from cogapp.py 4 | VERSION := $$(python -c "import cogapp.cogapp; print(cogapp.cogapp.__version__)") 5 | 6 | .PHONY: help clean sterile test 7 | 8 | help: ## Show this help. 9 | @echo "Available targets:" 10 | @grep '^[a-zA-Z]' $(MAKEFILE_LIST) | sort | awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}' 11 | 12 | clean: ## Remove artifacts of test execution, installation, etc. 13 | -rm -rf build 14 | -rm -rf dist 15 | -rm -f MANIFEST 16 | -rm -f *.pyc */*.pyc */*/*.pyc */*/*/*.pyc 17 | -rm -f *.pyo */*.pyo */*/*.pyo */*/*/*.pyo 18 | -rm -f *$$py.class */*$$py.class */*/*$$py.class */*/*/*$$py.class 19 | -rm -rf __pycache__ */__pycache__ */*/__pycache__ 20 | -rm -f *.bak */*.bak */*/*.bak */*/*/*.bak 21 | -rm -f .coverage .coverage.* coverage.xml 22 | -rm -rf cogapp.egg-info htmlcov 23 | -rm -rf docs/_build 24 | 25 | sterile: clean ## Remove all non-controlled content. 26 | -rm -rf .tox* 27 | -rm -rf .*_cache 28 | 29 | test: ## Run the test suite. 30 | tox -q 31 | 32 | # Docs 33 | 34 | .PHONY: cogdoc lintdoc dochtml 35 | 36 | # Normally I'd put this in a comment in index.px, but the 37 | # quoting/escaping would be impossible. 38 | COGARGS = -cP --markers='{{{cog }}} {{{end}}}' docs/running.rst 39 | 40 | cogdoc: ## Run cog to keep the docs correct. 41 | python -m cogapp -r $(COGARGS) 42 | 43 | lintdoc: ## Check that the docs are up-to-date. 44 | @python -m cogapp --check --diff $(COGARGS); \ 45 | if [ $$? -ne 0 ]; then \ 46 | echo 'Docs need to be updated: `make cogdoc`'; \ 47 | exit 1; \ 48 | fi 49 | 50 | dochtml: ## Build local docs. 51 | $(MAKE) -C docs html 52 | 53 | # Release 54 | 55 | .PHONY: dist pypi testpypi tag release check_release _check_credentials _check_manifest _check_tree _check_version 56 | 57 | dist: ## Build distribution artifacts. 58 | python -m build 59 | twine check dist/* 60 | 61 | pypi: ## Upload distributions to PyPI. 62 | twine upload --verbose dist/* 63 | 64 | testpypi: ## Upload distributions to test PyPI 65 | twine upload --verbose --repository testpypi --password $$TWINE_TEST_PASSWORD dist/* 66 | 67 | tag: ## Make a git tag with the version number 68 | git tag -s -m "Version $(VERSION)" v$(VERSION) 69 | git push --all 70 | 71 | release: _check_credentials clean check_release dist pypi tag ## Do all the steps for a release 72 | @echo "Release $(VERSION) complete!" 73 | 74 | check_release: _check_manifest _check_tree _check_version ## Check that we are ready for a release 75 | @echo "Release checks passed" 76 | 77 | _check_credentials: 78 | @if [[ -z "$$TWINE_PASSWORD" ]]; then \ 79 | echo 'Missing TWINE_PASSWORD'; \ 80 | exit 1; \ 81 | fi 82 | 83 | _check_manifest: 84 | python -m check_manifest 85 | 86 | _check_tree: 87 | @if [[ -n $$(git status --porcelain) ]]; then \ 88 | echo 'There are modified files! Did you forget to check them in?'; \ 89 | exit 1; \ 90 | fi 91 | 92 | _check_version: 93 | @if git tag | grep -q -w v$(VERSION); then \ 94 | echo 'A git tag for this version exists! Did you forget to bump the version?'; \ 95 | exit 1; \ 96 | fi 97 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | | |license| |versions| |status| 2 | | |ci-status| |kit| |format| 3 | | |sponsor| |bluesky-nedbat| |mastodon-nedbat| 4 | 5 | === 6 | Cog 7 | === 8 | 9 | Cog content generation tool. Small bits of computation for static files. 10 | 11 | See the `cog docs`_ for details. 12 | 13 | .. _cog docs: https://cog.readthedocs.io/en/latest/ 14 | 15 | Code repository and issue tracker are at 16 | `GitHub `_. 17 | 18 | To run the tests:: 19 | 20 | $ pip install -r requirements.pip 21 | $ tox 22 | 23 | 24 | .. |ci-status| image:: https://github.com/nedbat/cog/actions/workflows/ci.yml/badge.svg?branch=master&event=push 25 | :target: https://github.com/nedbat/cog/actions/workflows/ci.yml 26 | :alt: CI status 27 | .. |kit| image:: https://img.shields.io/pypi/v/cogapp.svg 28 | :target: https://pypi.org/project/cogapp/ 29 | :alt: PyPI status 30 | .. |format| image:: https://img.shields.io/pypi/format/cogapp.svg 31 | :target: https://pypi.org/project/cogapp/ 32 | :alt: Kit format 33 | .. |license| image:: https://img.shields.io/pypi/l/cogapp.svg 34 | :target: https://pypi.org/project/cogapp/ 35 | :alt: License 36 | .. |versions| image:: https://img.shields.io/pypi/pyversions/cogapp.svg 37 | :target: https://pypi.org/project/cogapp/ 38 | :alt: Python versions supported 39 | .. |status| image:: https://img.shields.io/pypi/status/cogapp.svg 40 | :target: https://pypi.org/project/cogapp/ 41 | :alt: Package stability 42 | .. |mastodon-nedbat| image:: https://img.shields.io/badge/dynamic/json?style=flat&labelColor=450657&logo=mastodon&logoColor=ffffff&label=@nedbat&query=followers_count&url=https%3A%2F%2Fhachyderm.io%2Fapi%2Fv1%2Faccounts%2Flookup%3Facct=nedbat 43 | :target: https://hachyderm.io/@nedbat 44 | :alt: nedbat on Mastodon 45 | .. |bluesky-nedbat| image:: https://img.shields.io/badge/dynamic/json?style=flat&color=96a3b0&labelColor=3686f7&logo=icloud&logoColor=white&label=@nedbat&url=https%3A%2F%2Fpublic.api.bsky.app%2Fxrpc%2Fapp.bsky.actor.getProfile%3Factor=nedbat.com&query=followersCount 46 | :target: https://bsky.app/profile/nedbat.com 47 | :alt: nedbat on Bluesky 48 | .. |sponsor| image:: https://img.shields.io/badge/%E2%9D%A4-Sponsor%20me-brightgreen?style=flat&logo=GitHub 49 | :target: https://github.com/sponsors/nedbat 50 | :alt: Sponsor me on GitHub 51 | -------------------------------------------------------------------------------- /cogapp/__init__.py: -------------------------------------------------------------------------------- 1 | """Cog content generation tool. 2 | http://nedbatchelder.com/code/cog 3 | 4 | Copyright 2004-2024, Ned Batchelder. 5 | """ 6 | 7 | from .cogapp import Cog as Cog, CogUsageError as CogUsageError, main as main 8 | -------------------------------------------------------------------------------- /cogapp/__main__.py: -------------------------------------------------------------------------------- 1 | """Make Cog runnable directly from the module.""" 2 | 3 | import sys 4 | 5 | from cogapp import Cog 6 | 7 | sys.exit(Cog().main(sys.argv)) 8 | -------------------------------------------------------------------------------- /cogapp/cogapp.py: -------------------------------------------------------------------------------- 1 | """Cog content generation tool.""" 2 | 3 | import copy 4 | import difflib 5 | import getopt 6 | import glob 7 | import io 8 | import linecache 9 | import os 10 | import re 11 | import shlex 12 | import sys 13 | import traceback 14 | import types 15 | 16 | from .whiteutils import common_prefix, reindent_block, white_prefix 17 | from .utils import NumberedFileReader, Redirectable, change_dir, md5 18 | from .hashhandler import HashHandler 19 | 20 | __version__ = "3.5.0" 21 | 22 | usage = """\ 23 | cog - generate content with inlined Python code. 24 | 25 | cog [OPTIONS] [INFILE | @FILELIST | &FILELIST] ... 26 | 27 | INFILE is the name of an input file, '-' will read from stdin. 28 | FILELIST is the name of a text file containing file names or 29 | other @FILELISTs. 30 | 31 | For @FILELIST, paths in the file list are relative to the working 32 | directory where cog was called. For &FILELIST, paths in the file 33 | list are relative to the file list location. 34 | 35 | OPTIONS: 36 | -c Checksum the output to protect it against accidental change. 37 | -d Delete the generator code from the output file. 38 | -D name=val Define a global string available to your generator code. 39 | -e Warn if a file has no cog code in it. 40 | -I PATH Add PATH to the list of directories for data files and modules. 41 | -n ENCODING Use ENCODING when reading and writing files. 42 | -o OUTNAME Write the output to OUTNAME. 43 | -p PROLOGUE Prepend the generator source with PROLOGUE. Useful to insert an 44 | import line. Example: -p "import math" 45 | -P Use print() instead of cog.outl() for code output. 46 | -r Replace the input file with the output. 47 | -s STRING Suffix all generated output lines with STRING. 48 | -U Write the output with Unix newlines (only LF line-endings). 49 | -w CMD Use CMD if the output file needs to be made writable. 50 | A %s in the CMD will be filled with the filename. 51 | -x Excise all the generated output without running the generators. 52 | -z The end-output marker can be omitted, and is assumed at eof. 53 | -v Print the version of cog and exit. 54 | --check Check that the files would not change if run again. 55 | --diff With --check, show a diff of what failed the check. 56 | --markers='START END END-OUTPUT' 57 | The patterns surrounding cog inline instructions. Should 58 | include three values separated by spaces, the start, end, 59 | and end-output markers. Defaults to '[[[cog ]]] [[[end]]]'. 60 | --verbosity=VERBOSITY 61 | Control the amount of output. 2 (the default) lists all files, 62 | 1 lists only changed files, 0 lists no files. 63 | -h, --help Print this help. 64 | """ 65 | 66 | 67 | class CogError(Exception): 68 | """Any exception raised by Cog.""" 69 | 70 | def __init__(self, msg, file="", line=0): 71 | if file: 72 | super().__init__(f"{file}({line}): {msg}") 73 | else: 74 | super().__init__(msg) 75 | 76 | 77 | class CogUsageError(CogError): 78 | """An error in usage of command-line arguments in cog.""" 79 | 80 | pass 81 | 82 | 83 | class CogInternalError(CogError): 84 | """An error in the coding of Cog. Should never happen.""" 85 | 86 | pass 87 | 88 | 89 | class CogGeneratedError(CogError): 90 | """An error raised by a user's cog generator.""" 91 | 92 | pass 93 | 94 | 95 | class CogUserException(CogError): 96 | """An exception caught when running a user's cog generator. 97 | 98 | The argument is the traceback message to print. 99 | 100 | """ 101 | 102 | pass 103 | 104 | 105 | class CogCheckFailed(CogError): 106 | """A --check failed.""" 107 | 108 | pass 109 | 110 | 111 | class CogGenerator(Redirectable): 112 | """A generator pulled from a source file.""" 113 | 114 | def __init__(self, options=None): 115 | super().__init__() 116 | self.markers = [] 117 | self.lines = [] 118 | self.options = options or CogOptions() 119 | 120 | def parse_marker(self, line): 121 | self.markers.append(line) 122 | 123 | def parse_line(self, line): 124 | self.lines.append(line.strip("\n")) 125 | 126 | def get_code(self): 127 | """Extract the executable Python code from the generator.""" 128 | # If the markers and lines all have the same prefix 129 | # (end-of-line comment chars, for example), 130 | # then remove it from all the lines. 131 | pref_in = common_prefix(self.markers + self.lines) 132 | if pref_in: 133 | self.markers = [line.replace(pref_in, "", 1) for line in self.markers] 134 | self.lines = [line.replace(pref_in, "", 1) for line in self.lines] 135 | 136 | return reindent_block(self.lines, "") 137 | 138 | def evaluate(self, cog, globals, fname): 139 | # figure out the right whitespace prefix for the output 140 | pref_out = white_prefix(self.markers) 141 | 142 | intext = self.get_code() 143 | if not intext: 144 | return "" 145 | 146 | prologue = "import " + cog.cogmodulename + " as cog\n" 147 | if self.options.prologue: 148 | prologue += self.options.prologue + "\n" 149 | code = compile(prologue + intext, str(fname), "exec") 150 | 151 | # Make sure the "cog" module has our state. 152 | cog.cogmodule.msg = self.msg 153 | cog.cogmodule.out = self.out 154 | cog.cogmodule.outl = self.outl 155 | cog.cogmodule.error = self.error 156 | 157 | real_stdout = sys.stdout 158 | if self.options.print_output: 159 | sys.stdout = captured_stdout = io.StringIO() 160 | 161 | self.outstring = "" 162 | try: 163 | eval(code, globals) 164 | except CogError: 165 | raise 166 | except: # noqa: E722 (we're just wrapping in CogUserException and rethrowing) 167 | typ, err, tb = sys.exc_info() 168 | frames = (tuple(fr) for fr in traceback.extract_tb(tb.tb_next)) 169 | frames = find_cog_source(frames, prologue) 170 | msg = "".join(traceback.format_list(frames)) 171 | msg += f"{typ.__name__}: {err}" 172 | raise CogUserException(msg) 173 | finally: 174 | sys.stdout = real_stdout 175 | 176 | if self.options.print_output: 177 | self.outstring = captured_stdout.getvalue() 178 | 179 | # We need to make sure that the last line in the output 180 | # ends with a newline, or it will be joined to the 181 | # end-output line, ruining cog's idempotency. 182 | if self.outstring and self.outstring[-1] != "\n": 183 | self.outstring += "\n" 184 | 185 | return reindent_block(self.outstring, pref_out) 186 | 187 | def msg(self, s): 188 | self.prout("Message: " + s) 189 | 190 | def out(self, sOut="", dedent=False, trimblanklines=False): 191 | """The cog.out function.""" 192 | if trimblanklines and ("\n" in sOut): 193 | lines = sOut.split("\n") 194 | if lines[0].strip() == "": 195 | del lines[0] 196 | if lines and lines[-1].strip() == "": 197 | del lines[-1] 198 | sOut = "\n".join(lines) + "\n" 199 | if dedent: 200 | sOut = reindent_block(sOut) 201 | self.outstring += sOut 202 | 203 | def outl(self, sOut="", **kw): 204 | """The cog.outl function.""" 205 | self.out(sOut, **kw) 206 | self.out("\n") 207 | 208 | def error(self, msg="Error raised by cog generator."): 209 | """The cog.error function. 210 | 211 | Instead of raising standard python errors, cog generators can use 212 | this function. It will display the error without a scary Python 213 | traceback. 214 | 215 | """ 216 | raise CogGeneratedError(msg) 217 | 218 | 219 | class CogOptions: 220 | """Options for a run of cog.""" 221 | 222 | def __init__(self): 223 | # Defaults for argument values. 224 | self.args = [] 225 | self.include_path = [] 226 | self.defines = {} 227 | self.show_version = False 228 | self.make_writable_cmd = None 229 | self.replace = False 230 | self.no_generate = False 231 | self.output_name = None 232 | self.warn_empty = False 233 | self.hash_output = False 234 | self.delete_code = False 235 | self.eof_can_be_end = False 236 | self.suffix = None 237 | self.newlines = False 238 | self.begin_spec = "[[[cog" 239 | self.end_spec = "]]]" 240 | self.end_output = "[[[end]]]" 241 | self.encoding = "utf-8" 242 | self.verbosity = 2 243 | self.prologue = "" 244 | self.print_output = False 245 | self.check = False 246 | self.diff = False 247 | 248 | def __eq__(self, other): 249 | """Comparison operator for tests to use.""" 250 | return self.__dict__ == other.__dict__ 251 | 252 | def clone(self): 253 | """Make a clone of these options, for further refinement.""" 254 | return copy.deepcopy(self) 255 | 256 | def add_to_include_path(self, dirs): 257 | """Add directories to the include path.""" 258 | dirs = dirs.split(os.pathsep) 259 | self.include_path.extend(dirs) 260 | 261 | def parse_args(self, argv): 262 | # Parse the command line arguments. 263 | try: 264 | opts, self.args = getopt.getopt( 265 | argv, 266 | "cdD:eI:n:o:rs:p:PUvw:xz", 267 | [ 268 | "check", 269 | "diff", 270 | "markers=", 271 | "verbosity=", 272 | ], 273 | ) 274 | except getopt.error as msg: 275 | raise CogUsageError(msg) 276 | 277 | # Handle the command line arguments. 278 | for o, a in opts: 279 | if o == "-c": 280 | self.hash_output = True 281 | elif o == "-d": 282 | self.delete_code = True 283 | elif o == "-D": 284 | if a.count("=") < 1: 285 | raise CogUsageError("-D takes a name=value argument") 286 | name, value = a.split("=", 1) 287 | self.defines[name] = value 288 | elif o == "-e": 289 | self.warn_empty = True 290 | elif o == "-I": 291 | self.add_to_include_path(os.path.abspath(a)) 292 | elif o == "-n": 293 | self.encoding = a 294 | elif o == "-o": 295 | self.output_name = a 296 | elif o == "-r": 297 | self.replace = True 298 | elif o == "-s": 299 | self.suffix = a 300 | elif o == "-p": 301 | self.prologue = a 302 | elif o == "-P": 303 | self.print_output = True 304 | elif o == "-U": 305 | self.newlines = True 306 | elif o == "-v": 307 | self.show_version = True 308 | elif o == "-w": 309 | self.make_writable_cmd = a 310 | elif o == "-x": 311 | self.no_generate = True 312 | elif o == "-z": 313 | self.eof_can_be_end = True 314 | elif o == "--check": 315 | self.check = True 316 | elif o == "--diff": 317 | self.diff = True 318 | elif o == "--markers": 319 | self._parse_markers(a) 320 | elif o == "--verbosity": 321 | self.verbosity = int(a) 322 | else: 323 | # Since getopt.getopt is given a list of possible flags, 324 | # this is an internal error. 325 | raise CogInternalError(f"Don't understand argument {o}") 326 | 327 | def _parse_markers(self, val): 328 | try: 329 | self.begin_spec, self.end_spec, self.end_output = val.split(" ") 330 | except ValueError: 331 | raise CogUsageError( 332 | f"--markers requires 3 values separated by spaces, could not parse {val!r}" 333 | ) 334 | 335 | def validate(self): 336 | """Does nothing if everything is OK, raises CogError's if it's not.""" 337 | if self.replace and self.delete_code: 338 | raise CogUsageError( 339 | "Can't use -d with -r (or you would delete all your source!)" 340 | ) 341 | 342 | if self.replace and self.output_name: 343 | raise CogUsageError("Can't use -o with -r (they are opposites)") 344 | 345 | if self.diff and not self.check: 346 | raise CogUsageError("Can't use --diff without --check") 347 | 348 | 349 | class Cog(Redirectable): 350 | """The Cog engine.""" 351 | 352 | def __init__(self): 353 | super().__init__() 354 | self.options = CogOptions() 355 | self.cogmodulename = "cog" 356 | self.create_cog_module() 357 | self.check_failed = False 358 | self.hash_handler = None 359 | self._fix_end_output_patterns() 360 | 361 | def _fix_end_output_patterns(self): 362 | self.hash_handler = HashHandler(self.options.end_output) 363 | 364 | def show_warning(self, msg): 365 | self.prout(f"Warning: {msg}") 366 | 367 | def is_begin_spec_line(self, s): 368 | return self.options.begin_spec in s 369 | 370 | def is_end_spec_line(self, s): 371 | return self.options.end_spec in s and not self.is_end_output_line(s) 372 | 373 | def is_end_output_line(self, s): 374 | return self.options.end_output in s 375 | 376 | def create_cog_module(self): 377 | """Make a cog "module" object. 378 | 379 | Imported Python modules can use "import cog" to get our state. 380 | 381 | """ 382 | self.cogmodule = types.SimpleNamespace() 383 | self.cogmodule.path = [] 384 | 385 | def open_output_file(self, fname): 386 | """Open an output file, taking all the details into account.""" 387 | opts = {} 388 | mode = "w" 389 | opts["encoding"] = self.options.encoding 390 | if self.options.newlines: 391 | opts["newline"] = "\n" 392 | fdir = os.path.dirname(fname) 393 | if os.path.dirname(fdir) and not os.path.exists(fdir): 394 | os.makedirs(fdir) 395 | return open(fname, mode, **opts) 396 | 397 | def open_input_file(self, fname): 398 | """Open an input file.""" 399 | if fname == "-": 400 | return sys.stdin 401 | else: 402 | return open(fname, encoding=self.options.encoding) 403 | 404 | def process_file(self, file_in, file_out, fname=None, globals=None): 405 | """Process an input file object to an output file object. 406 | 407 | `fileIn` and `fileOut` can be file objects, or file names. 408 | 409 | """ 410 | file_name_in = fname or "" 411 | file_name_out = fname or "" 412 | file_in_to_close = file_out_to_close = None 413 | # Convert filenames to files. 414 | if isinstance(file_in, (bytes, str)): 415 | # Open the input file. 416 | file_name_in = file_in 417 | file_in = file_in_to_close = self.open_input_file(file_in) 418 | if isinstance(file_out, (bytes, str)): 419 | # Open the output file. 420 | file_name_out = file_out 421 | file_out = file_out_to_close = self.open_output_file(file_out) 422 | 423 | start_dir = os.getcwd() 424 | 425 | try: 426 | file_in = NumberedFileReader(file_in) 427 | 428 | saw_cog = False 429 | 430 | self.cogmodule.inFile = file_name_in 431 | self.cogmodule.outFile = file_name_out 432 | self.cogmodulename = "cog_" + md5(file_name_out.encode()).hexdigest() 433 | sys.modules[self.cogmodulename] = self.cogmodule 434 | # if "import cog" explicitly done in code by user, note threading will cause clashes. 435 | sys.modules["cog"] = self.cogmodule 436 | 437 | # The globals dict we'll use for this file. 438 | if globals is None: 439 | globals = {} 440 | 441 | # If there are any global defines, put them in the globals. 442 | globals.update(self.options.defines) 443 | 444 | # loop over generator chunks 445 | line = file_in.readline() 446 | while line: 447 | # Find the next spec begin 448 | while line and not self.is_begin_spec_line(line): 449 | if self.is_end_spec_line(line): 450 | raise CogError( 451 | f"Unexpected {self.options.end_spec!r}", 452 | file=file_name_in, 453 | line=file_in.linenumber(), 454 | ) 455 | if self.is_end_output_line(line): 456 | raise CogError( 457 | f"Unexpected {self.options.end_output!r}", 458 | file=file_name_in, 459 | line=file_in.linenumber(), 460 | ) 461 | file_out.write(line) 462 | line = file_in.readline() 463 | if not line: 464 | break 465 | if not self.options.delete_code: 466 | file_out.write(line) 467 | 468 | # `line` is the begin spec 469 | gen = CogGenerator(options=self.options) 470 | gen.set_output(stdout=self.stdout) 471 | gen.parse_marker(line) 472 | first_line_num = file_in.linenumber() 473 | self.cogmodule.firstLineNum = first_line_num 474 | 475 | # If the spec begin is also a spec end, then process the single 476 | # line of code inside. 477 | if self.is_end_spec_line(line): 478 | beg = line.find(self.options.begin_spec) 479 | end = line.find(self.options.end_spec) 480 | if beg > end: 481 | raise CogError( 482 | "Cog code markers inverted", 483 | file=file_name_in, 484 | line=first_line_num, 485 | ) 486 | else: 487 | code = line[beg + len(self.options.begin_spec) : end].strip() 488 | gen.parse_line(code) 489 | else: 490 | # Deal with an ordinary code block. 491 | line = file_in.readline() 492 | 493 | # Get all the lines in the spec 494 | while line and not self.is_end_spec_line(line): 495 | if self.is_begin_spec_line(line): 496 | raise CogError( 497 | f"Unexpected {self.options.begin_spec!r}", 498 | file=file_name_in, 499 | line=file_in.linenumber(), 500 | ) 501 | if self.is_end_output_line(line): 502 | raise CogError( 503 | f"Unexpected {self.options.end_output!r}", 504 | file=file_name_in, 505 | line=file_in.linenumber(), 506 | ) 507 | if not self.options.delete_code: 508 | file_out.write(line) 509 | gen.parse_line(line) 510 | line = file_in.readline() 511 | if not line: 512 | raise CogError( 513 | "Cog block begun but never ended.", 514 | file=file_name_in, 515 | line=first_line_num, 516 | ) 517 | 518 | if not self.options.delete_code: 519 | file_out.write(line) 520 | gen.parse_marker(line) 521 | 522 | line = file_in.readline() 523 | 524 | # Eat all the lines in the output section. While reading past 525 | # them, compute the md5 hash of the old output. 526 | previous = [] 527 | while line and not self.is_end_output_line(line): 528 | if self.is_begin_spec_line(line): 529 | raise CogError( 530 | f"Unexpected {self.options.begin_spec!r}", 531 | file=file_name_in, 532 | line=file_in.linenumber(), 533 | ) 534 | if self.is_end_spec_line(line): 535 | raise CogError( 536 | f"Unexpected {self.options.end_spec!r}", 537 | file=file_name_in, 538 | line=file_in.linenumber(), 539 | ) 540 | previous.append(line) 541 | line = file_in.readline() 542 | cur_hash = self.hash_handler.compute_lines_hash(previous) 543 | 544 | if not line and not self.options.eof_can_be_end: 545 | # We reached end of file before we found the end output line. 546 | raise CogError( 547 | f"Missing {self.options.end_output!r} before end of file.", 548 | file=file_name_in, 549 | line=file_in.linenumber(), 550 | ) 551 | 552 | # Make the previous output available to the current code 553 | self.cogmodule.previous = "".join(previous) 554 | 555 | # Write the output of the spec to be the new output if we're 556 | # supposed to generate code. 557 | if not self.options.no_generate: 558 | fname = f"" 559 | gen = gen.evaluate(cog=self, globals=globals, fname=fname) 560 | gen = self.suffix_lines(gen) 561 | new_hash = self.hash_handler.compute_hash(gen) 562 | file_out.write(gen) 563 | else: 564 | new_hash = "" 565 | 566 | saw_cog = True 567 | 568 | # Write the ending output line 569 | if self.options.hash_output: 570 | try: 571 | self.hash_handler.validate_hash(line, cur_hash) 572 | except ValueError as e: 573 | raise CogError( 574 | str(e), 575 | file=file_name_in, 576 | line=file_in.linenumber(), 577 | ) 578 | line = self.hash_handler.format_end_line_with_hash( 579 | line, 580 | new_hash, 581 | add_hash=True, 582 | preserve_format=self.options.check, 583 | ) 584 | else: 585 | line = self.hash_handler.format_end_line_with_hash( 586 | line, new_hash, add_hash=False 587 | ) 588 | 589 | if not self.options.delete_code: 590 | file_out.write(line) 591 | line = file_in.readline() 592 | 593 | if not saw_cog and self.options.warn_empty: 594 | self.show_warning(f"no cog code found in {file_name_in}") 595 | finally: 596 | if file_in_to_close: 597 | file_in_to_close.close() 598 | if file_out_to_close: 599 | file_out_to_close.close() 600 | os.chdir(start_dir) 601 | 602 | # A regex for non-empty lines, used by suffixLines. 603 | re_non_empty_lines = re.compile(r"^\s*\S+.*$", re.MULTILINE) 604 | 605 | def suffix_lines(self, text): 606 | """Add suffixes to the lines in text, if our options desire it. 607 | 608 | `text` is many lines, as a single string. 609 | 610 | """ 611 | if self.options.suffix: 612 | # Find all non-blank lines, and add the suffix to the end. 613 | repl = r"\g<0>" + self.options.suffix.replace("\\", "\\\\") 614 | text = self.re_non_empty_lines.sub(repl, text) 615 | return text 616 | 617 | def process_string(self, input, fname=None): 618 | """Process `input` as the text to cog. 619 | 620 | Return the cogged output as a string. 621 | 622 | """ 623 | file_old = io.StringIO(input) 624 | file_new = io.StringIO() 625 | self.process_file(file_old, file_new, fname=fname) 626 | return file_new.getvalue() 627 | 628 | def replace_file(self, old_path, new_text): 629 | """Replace file oldPath with the contents newText""" 630 | if not os.access(old_path, os.W_OK): 631 | # Need to ensure we can write. 632 | if self.options.make_writable_cmd: 633 | # Use an external command to make the file writable. 634 | cmd = self.options.make_writable_cmd.replace("%s", old_path) 635 | with os.popen(cmd) as cmdout: 636 | self.stdout.write(cmdout.read()) 637 | if not os.access(old_path, os.W_OK): 638 | raise CogError(f"Couldn't make {old_path} writable") 639 | else: 640 | # Can't write! 641 | raise CogError(f"Can't overwrite {old_path}") 642 | f = self.open_output_file(old_path) 643 | f.write(new_text) 644 | f.close() 645 | 646 | def save_include_path(self): 647 | self.saved_include = self.options.include_path[:] 648 | self.saved_sys_path = sys.path[:] 649 | 650 | def restore_include_path(self): 651 | self.options.include_path = self.saved_include 652 | self.cogmodule.path = self.options.include_path 653 | sys.path = self.saved_sys_path 654 | 655 | def add_to_include_path(self, include_path): 656 | self.cogmodule.path.extend(include_path) 657 | sys.path.extend(include_path) 658 | 659 | def process_one_file(self, fname): 660 | """Process one filename through cog.""" 661 | 662 | self.save_include_path() 663 | need_newline = False 664 | 665 | try: 666 | self.add_to_include_path(self.options.include_path) 667 | # Since we know where the input file came from, 668 | # push its directory onto the include path. 669 | self.add_to_include_path([os.path.dirname(fname)]) 670 | 671 | # How we process the file depends on where the output is going. 672 | if self.options.output_name: 673 | self.process_file(fname, self.options.output_name, fname) 674 | elif self.options.replace or self.options.check: 675 | # We want to replace the cog file with the output, 676 | # but only if they differ. 677 | verb = "Cogging" if self.options.replace else "Checking" 678 | if self.options.verbosity >= 2: 679 | self.prout(f"{verb} {fname}", end="") 680 | need_newline = True 681 | 682 | try: 683 | file_old_file = self.open_input_file(fname) 684 | old_text = file_old_file.read() 685 | file_old_file.close() 686 | new_text = self.process_string(old_text, fname=fname) 687 | if old_text != new_text: 688 | if self.options.verbosity >= 1: 689 | if self.options.verbosity < 2: 690 | self.prout(f"{verb} {fname}", end="") 691 | self.prout(" (changed)") 692 | need_newline = False 693 | if self.options.replace: 694 | self.replace_file(fname, new_text) 695 | else: 696 | assert self.options.check 697 | self.check_failed = True 698 | if self.options.diff: 699 | old_lines = old_text.splitlines() 700 | new_lines = new_text.splitlines() 701 | diff = difflib.unified_diff( 702 | old_lines, 703 | new_lines, 704 | fromfile=f"current {fname}", 705 | tofile=f"changed {fname}", 706 | lineterm="", 707 | ) 708 | for diff_line in diff: 709 | self.prout(diff_line) 710 | finally: 711 | # The try-finally block is so we can print a partial line 712 | # with the name of the file, and print (changed) on the 713 | # same line, but also make sure to break the line before 714 | # any traceback. 715 | if need_newline: 716 | self.prout("") 717 | else: 718 | self.process_file(fname, self.stdout, fname) 719 | finally: 720 | self.restore_include_path() 721 | 722 | def process_wildcards(self, fname): 723 | files = glob.glob(fname) 724 | if files: 725 | for matching_file in files: 726 | self.process_one_file(matching_file) 727 | else: 728 | self.process_one_file(fname) 729 | 730 | def process_file_list(self, file_name_list): 731 | """Process the files in a file list.""" 732 | flist = self.open_input_file(file_name_list) 733 | lines = flist.readlines() 734 | flist.close() 735 | for line in lines: 736 | # Use shlex to parse the line like a shell. 737 | lex = shlex.shlex(line, posix=True) 738 | lex.whitespace_split = True 739 | lex.commenters = "#" 740 | # No escapes, so that backslash can be part of the path 741 | lex.escape = "" 742 | args = list(lex) 743 | if args: 744 | self.process_arguments(args) 745 | 746 | def process_arguments(self, args): 747 | """Process one command-line.""" 748 | saved_options = self.options 749 | self.options = self.options.clone() 750 | 751 | self.options.parse_args(args[1:]) 752 | self.options.validate() 753 | 754 | if args[0][0] == "@": 755 | if self.options.output_name: 756 | raise CogUsageError("Can't use -o with @file") 757 | self.process_file_list(args[0][1:]) 758 | elif args[0][0] == "&": 759 | if self.options.output_name: 760 | raise CogUsageError("Can't use -o with &file") 761 | file_list = args[0][1:] 762 | with change_dir(os.path.dirname(file_list)): 763 | self.process_file_list(os.path.basename(file_list)) 764 | else: 765 | self.process_wildcards(args[0]) 766 | 767 | self.options = saved_options 768 | 769 | def callable_main(self, argv): 770 | """All of command-line cog, but in a callable form. 771 | 772 | This is used by main. `argv` is the equivalent of sys.argv. 773 | 774 | """ 775 | argv = argv[1:] 776 | 777 | # Provide help if asked for anywhere in the command line. 778 | if "-?" in argv or "-h" in argv or "--help" in argv: 779 | self.prerr(usage, end="") 780 | return 781 | 782 | self.options.parse_args(argv) 783 | self.options.validate() 784 | self._fix_end_output_patterns() 785 | 786 | if self.options.show_version: 787 | self.prout(f"Cog version {__version__}") 788 | return 789 | 790 | if self.options.args: 791 | for a in self.options.args: 792 | self.process_arguments([a]) 793 | else: 794 | raise CogUsageError("No files to process") 795 | 796 | if self.check_failed: 797 | raise CogCheckFailed("Check failed") 798 | 799 | def main(self, argv): 800 | """Handle the command-line execution for cog.""" 801 | 802 | try: 803 | self.callable_main(argv) 804 | return 0 805 | except CogUsageError as err: 806 | self.prerr(err) 807 | self.prerr("(for help use --help)") 808 | return 2 809 | except CogGeneratedError as err: 810 | self.prerr(f"Error: {err}") 811 | return 3 812 | except CogUserException as err: 813 | self.prerr("Traceback (most recent call last):") 814 | self.prerr(err.args[0]) 815 | return 4 816 | except CogCheckFailed as err: 817 | self.prerr(err) 818 | return 5 819 | except CogError as err: 820 | self.prerr(err) 821 | return 1 822 | 823 | 824 | def find_cog_source(frame_summary, prologue): 825 | """Find cog source lines in a frame summary list, for printing tracebacks. 826 | 827 | Arguments: 828 | frame_summary: a list of 4-item tuples, as returned by traceback.extract_tb. 829 | prologue: the text of the code prologue. 830 | 831 | Returns 832 | A list of 4-item tuples, updated to correct the cog entries. 833 | 834 | """ 835 | prolines = prologue.splitlines() 836 | for filename, lineno, funcname, source in frame_summary: 837 | if not source: 838 | m = re.search(r"^$", filename) 839 | if m: 840 | if lineno <= len(prolines): 841 | filename = "" 842 | source = prolines[lineno - 1] 843 | lineno -= ( 844 | 1 # Because "import cog" is the first line in the prologue 845 | ) 846 | else: 847 | filename, coglineno = m.groups() 848 | coglineno = int(coglineno) 849 | lineno += coglineno - len(prolines) 850 | source = linecache.getline(filename, lineno).strip() 851 | yield filename, lineno, funcname, source 852 | 853 | 854 | def main(): 855 | """Main function for entry_points to use.""" 856 | return Cog().main(sys.argv) 857 | -------------------------------------------------------------------------------- /cogapp/hashhandler.py: -------------------------------------------------------------------------------- 1 | """Hash handling for cog output verification.""" 2 | 3 | import base64 4 | import re 5 | from .utils import md5 6 | 7 | 8 | class HashHandler: 9 | """Handles checksum generation and verification for cog output.""" 10 | 11 | def __init__(self, end_output_marker): 12 | """Initialize the hash handler with the end output marker pattern. 13 | 14 | Args: 15 | end_output_marker: The end output marker string (e.g., "[[[end]]]") 16 | """ 17 | self.end_output_marker = end_output_marker 18 | self._setup_patterns() 19 | 20 | def _setup_patterns(self): 21 | """Set up regex patterns for hash detection and formatting.""" 22 | end_output = re.escape(self.end_output_marker) 23 | # Support both old format (checksum: 32-char hex) and new format (sum: 10-char base64) 24 | self.re_end_output_with_hash = re.compile( 25 | end_output 26 | + r"(?P *\((?:checksum: (?P[a-f0-9]{32})|sum: (?P[A-Za-z0-9+/]{10}))\))" 27 | ) 28 | self.end_format = self.end_output_marker + " (sum: %s)" 29 | 30 | def compute_hash(self, content): 31 | """Compute MD5 hash of the given content. 32 | 33 | Args: 34 | content: String content to hash 35 | 36 | Returns: 37 | str: Hexadecimal hash digest 38 | """ 39 | hasher = md5() 40 | hasher.update(content.encode("utf-8")) 41 | return hasher.hexdigest() 42 | 43 | def compute_lines_hash(self, lines): 44 | """Compute MD5 hash of a list of lines. 45 | 46 | Args: 47 | lines: List of line strings 48 | 49 | Returns: 50 | str: Hexadecimal hash digest 51 | """ 52 | hasher = md5() 53 | for line in lines: 54 | hasher.update(line.encode("utf-8")) 55 | return hasher.hexdigest() 56 | 57 | def hex_to_base64_hash(self, hex_hash): 58 | """Convert a 32-character hex hash to a 10-character base64 hash. 59 | 60 | Args: 61 | hex_hash: 32-character hexadecimal hash string 62 | 63 | Returns: 64 | str: 10-character base64 hash string 65 | """ 66 | # Convert hex to bytes 67 | hash_bytes = bytes.fromhex(hex_hash) 68 | # Encode to base64 and take first 10 characters 69 | b64_hash = base64.b64encode(hash_bytes).decode("ascii")[:10] 70 | return b64_hash 71 | 72 | def extract_hash_from_line(self, line): 73 | """Extract hash from an end output line if present. 74 | 75 | Args: 76 | line: The end output line to check 77 | 78 | Returns: 79 | tuple: (hash_type, hash_value) where hash_type is 'hex' or 'base64' 80 | and hash_value is the raw hash value, or (None, None) if not found 81 | """ 82 | hash_match = self.re_end_output_with_hash.search(line) 83 | if hash_match: 84 | # Check which format was matched 85 | if hash_match.group("hash"): 86 | # Old format: checksum with hex 87 | return ("hex", hash_match.group("hash")) 88 | else: 89 | # New format: sum with base64 90 | assert hash_match.group( 91 | "b64hash" 92 | ), "Regex matched but no hash group found" 93 | return ("base64", hash_match.group("b64hash")) 94 | return (None, None) 95 | 96 | def validate_hash(self, line, expected_hash): 97 | """Validate that the hash in the line matches the expected hash. 98 | 99 | Args: 100 | line: The end output line containing the hash 101 | expected_hash: The expected hash value (hex format) 102 | 103 | Returns: 104 | bool: True if hash matches or no hash present, False if mismatch 105 | 106 | Raises: 107 | ValueError: If hash is present but doesn't match expected 108 | """ 109 | hash_type, old_hash = self.extract_hash_from_line(line) 110 | if hash_type is not None: 111 | if hash_type == "hex": 112 | # Compare hex directly 113 | if old_hash != expected_hash: 114 | raise ValueError( 115 | "Output has been edited! Delete old checksum to unprotect." 116 | ) 117 | else: 118 | # Convert expected hex to base64 and compare 119 | assert hash_type == "base64", f"Unknown hash type: {hash_type}" 120 | expected_b64 = self.hex_to_base64_hash(expected_hash) 121 | if old_hash != expected_b64: 122 | raise ValueError( 123 | "Output has been edited! Delete old checksum to unprotect." 124 | ) 125 | return True 126 | 127 | def format_end_line_with_hash( 128 | self, line, new_hash, add_hash=True, preserve_format=False 129 | ): 130 | """Format the end output line with or without hash. 131 | 132 | Args: 133 | line: The original end output line 134 | new_hash: The hash to add if add_hash is True (hex format) 135 | add_hash: Whether to add hash to the output 136 | preserve_format: If True and an existing hash is found, preserve its format 137 | 138 | Returns: 139 | str: The formatted end output line 140 | """ 141 | hash_match = self.re_end_output_with_hash.search(line) 142 | 143 | if add_hash: 144 | if preserve_format and hash_match: 145 | # Preserve the original format 146 | hash_type, old_hash = self.extract_hash_from_line(line) 147 | if hash_type == "hex": 148 | # Keep hex format 149 | formatted_hash = f" (checksum: {new_hash})" 150 | else: 151 | # Keep base64 format 152 | assert hash_type == "base64", f"Unknown hash type: {hash_type}" 153 | b64_hash = self.hex_to_base64_hash(new_hash) 154 | formatted_hash = f" (sum: {b64_hash})" 155 | 156 | # Replace the hash section 157 | endpieces = line.split(hash_match.group(0), 1) 158 | line = (self.end_output_marker + formatted_hash).join(endpieces) 159 | else: 160 | # Use new format 161 | b64_hash = self.hex_to_base64_hash(new_hash) 162 | 163 | if hash_match: 164 | # Replace existing hash 165 | endpieces = line.split(hash_match.group(0), 1) 166 | else: 167 | # Add new hash 168 | endpieces = line.split(self.end_output_marker, 1) 169 | line = (self.end_format % b64_hash).join(endpieces) 170 | else: 171 | # Remove hash if present 172 | if hash_match: 173 | line = line.replace(hash_match["hashsect"], "", 1) 174 | 175 | return line 176 | -------------------------------------------------------------------------------- /cogapp/makefiles.py: -------------------------------------------------------------------------------- 1 | """Dictionary-to-filetree functions, to create test files for testing.""" 2 | 3 | import os.path 4 | 5 | from .whiteutils import reindent_block 6 | 7 | 8 | def make_files(d, basedir="."): 9 | """Create files from the dictionary `d` in the directory named by `basedir`.""" 10 | for name, contents in d.items(): 11 | child = os.path.join(basedir, name) 12 | if isinstance(contents, (bytes, str)): 13 | mode = "w" 14 | if isinstance(contents, bytes): 15 | mode += "b" 16 | with open(child, mode) as f: 17 | f.write(reindent_block(contents)) 18 | else: 19 | if not os.path.exists(child): 20 | os.mkdir(child) 21 | make_files(contents, child) 22 | 23 | 24 | def remove_files(d, basedir="."): 25 | """Remove the files created by `makeFiles`. 26 | 27 | Directories are removed if they are empty. 28 | 29 | """ 30 | for name, contents in d.items(): 31 | child = os.path.join(basedir, name) 32 | if isinstance(contents, (bytes, str)): 33 | os.remove(child) 34 | else: 35 | remove_files(contents, child) 36 | if not os.listdir(child): 37 | os.rmdir(child) 38 | -------------------------------------------------------------------------------- /cogapp/test_cogapp.py: -------------------------------------------------------------------------------- 1 | """Test cogapp.""" 2 | 3 | import io 4 | import os 5 | import os.path 6 | import random 7 | import re 8 | import shutil 9 | import stat 10 | import sys 11 | import tempfile 12 | import threading 13 | from unittest import TestCase 14 | 15 | from .cogapp import Cog, CogOptions, CogGenerator 16 | from .cogapp import CogError, CogUsageError, CogGeneratedError, CogUserException 17 | from .cogapp import usage, __version__, main 18 | from .hashhandler import HashHandler 19 | from .makefiles import make_files 20 | from .whiteutils import reindent_block 21 | 22 | 23 | class CogTestsInMemory(TestCase): 24 | """Test cases for cogapp.Cog()""" 25 | 26 | def test_no_cog(self): 27 | strings = [ 28 | "", 29 | " ", 30 | " \t \t \tx", 31 | "hello", 32 | "the cat\nin the\nhat.", 33 | "Horton\n\tHears A\n\t\tWho", 34 | ] 35 | for s in strings: 36 | self.assertEqual(Cog().process_string(s), s) 37 | 38 | def test_simple(self): 39 | infile = """\ 40 | Some text. 41 | //[[[cog 42 | import cog 43 | cog.outl("This is line one\\n") 44 | cog.outl("This is line two") 45 | //]]] 46 | gobbledegook. 47 | //[[[end]]] 48 | epilogue. 49 | """ 50 | 51 | outfile = """\ 52 | Some text. 53 | //[[[cog 54 | import cog 55 | cog.outl("This is line one\\n") 56 | cog.outl("This is line two") 57 | //]]] 58 | This is line one 59 | 60 | This is line two 61 | //[[[end]]] 62 | epilogue. 63 | """ 64 | 65 | self.assertEqual(Cog().process_string(infile), outfile) 66 | 67 | def test_empty_cog(self): 68 | # The cog clause can be totally empty. Not sure why you'd want it, 69 | # but it works. 70 | infile = """\ 71 | hello 72 | //[[[cog 73 | //]]] 74 | //[[[end]]] 75 | goodbye 76 | """ 77 | 78 | infile = reindent_block(infile) 79 | self.assertEqual(Cog().process_string(infile), infile) 80 | 81 | def test_multiple_cogs(self): 82 | # One file can have many cog chunks, even abutting each other. 83 | infile = """\ 84 | //[[[cog 85 | cog.out("chunk1") 86 | //]]] 87 | chunk1 88 | //[[[end]]] 89 | //[[[cog 90 | cog.out("chunk2") 91 | //]]] 92 | chunk2 93 | //[[[end]]] 94 | between chunks 95 | //[[[cog 96 | cog.out("chunk3") 97 | //]]] 98 | chunk3 99 | //[[[end]]] 100 | """ 101 | 102 | infile = reindent_block(infile) 103 | self.assertEqual(Cog().process_string(infile), infile) 104 | 105 | def test_trim_blank_lines(self): 106 | infile = """\ 107 | //[[[cog 108 | cog.out("This is line one\\n", trimblanklines=True) 109 | cog.out(''' 110 | This is line two 111 | ''', dedent=True, trimblanklines=True) 112 | cog.outl("This is line three", trimblanklines=True) 113 | //]]] 114 | This is line one 115 | This is line two 116 | This is line three 117 | //[[[end]]] 118 | """ 119 | 120 | infile = reindent_block(infile) 121 | self.assertEqual(Cog().process_string(infile), infile) 122 | 123 | def test_trim_empty_blank_lines(self): 124 | infile = """\ 125 | //[[[cog 126 | cog.out("This is line one\\n", trimblanklines=True) 127 | cog.out(''' 128 | This is line two 129 | ''', dedent=True, trimblanklines=True) 130 | cog.out('', dedent=True, trimblanklines=True) 131 | cog.outl("This is line three", trimblanklines=True) 132 | //]]] 133 | This is line one 134 | This is line two 135 | This is line three 136 | //[[[end]]] 137 | """ 138 | 139 | infile = reindent_block(infile) 140 | self.assertEqual(Cog().process_string(infile), infile) 141 | 142 | def test_trim_blank_lines_with_last_partial(self): 143 | infile = """\ 144 | //[[[cog 145 | cog.out("This is line one\\n", trimblanklines=True) 146 | cog.out("\\nLine two\\nLine three", trimblanklines=True) 147 | //]]] 148 | This is line one 149 | Line two 150 | Line three 151 | //[[[end]]] 152 | """ 153 | 154 | infile = reindent_block(infile) 155 | self.assertEqual(Cog().process_string(infile), infile) 156 | 157 | def test_cog_out_dedent(self): 158 | infile = """\ 159 | //[[[cog 160 | cog.out("This is the first line\\n") 161 | cog.out(''' 162 | This is dedent=True 1 163 | This is dedent=True 2 164 | ''', dedent=True, trimblanklines=True) 165 | cog.out(''' 166 | This is dedent=False 1 167 | This is dedent=False 2 168 | ''', dedent=False, trimblanklines=True) 169 | cog.out(''' 170 | This is dedent=default 1 171 | This is dedent=default 2 172 | ''', trimblanklines=True) 173 | cog.out("This is the last line\\n") 174 | //]]] 175 | This is the first line 176 | This is dedent=True 1 177 | This is dedent=True 2 178 | This is dedent=False 1 179 | This is dedent=False 2 180 | This is dedent=default 1 181 | This is dedent=default 2 182 | This is the last line 183 | //[[[end]]] 184 | """ 185 | 186 | infile = reindent_block(infile) 187 | self.assertEqual(Cog().process_string(infile), infile) 188 | 189 | def test22_end_of_line(self): 190 | # In Python 2.2, this cog file was not parsing because the 191 | # last line is indented but didn't end with a newline. 192 | infile = """\ 193 | //[[[cog 194 | import cog 195 | for i in range(3): 196 | cog.out("%d\\n" % i) 197 | //]]] 198 | 0 199 | 1 200 | 2 201 | //[[[end]]] 202 | """ 203 | 204 | infile = reindent_block(infile) 205 | self.assertEqual(Cog().process_string(infile), infile) 206 | 207 | def test_indented_code(self): 208 | infile = """\ 209 | first line 210 | [[[cog 211 | import cog 212 | for i in range(3): 213 | cog.out("xx%d\\n" % i) 214 | ]]] 215 | xx0 216 | xx1 217 | xx2 218 | [[[end]]] 219 | last line 220 | """ 221 | 222 | infile = reindent_block(infile) 223 | self.assertEqual(Cog().process_string(infile), infile) 224 | 225 | def test_prefixed_code(self): 226 | infile = """\ 227 | --[[[cog 228 | --import cog 229 | --for i in range(3): 230 | -- cog.out("xx%d\\n" % i) 231 | --]]] 232 | xx0 233 | xx1 234 | xx2 235 | --[[[end]]] 236 | """ 237 | 238 | infile = reindent_block(infile) 239 | self.assertEqual(Cog().process_string(infile), infile) 240 | 241 | def test_prefixed_indented_code(self): 242 | infile = """\ 243 | prologue 244 | --[[[cog 245 | -- import cog 246 | -- for i in range(3): 247 | -- cog.out("xy%d\\n" % i) 248 | --]]] 249 | xy0 250 | xy1 251 | xy2 252 | --[[[end]]] 253 | """ 254 | 255 | infile = reindent_block(infile) 256 | self.assertEqual(Cog().process_string(infile), infile) 257 | 258 | def test_bogus_prefix_match(self): 259 | infile = """\ 260 | prologue 261 | #[[[cog 262 | import cog 263 | # This comment should not be clobbered by removing the pound sign. 264 | for i in range(3): 265 | cog.out("xy%d\\n" % i) 266 | #]]] 267 | xy0 268 | xy1 269 | xy2 270 | #[[[end]]] 271 | """ 272 | 273 | infile = reindent_block(infile) 274 | self.assertEqual(Cog().process_string(infile), infile) 275 | 276 | def test_no_final_newline(self): 277 | # If the cog'ed output has no final newline, 278 | # it shouldn't eat up the cog terminator. 279 | infile = """\ 280 | prologue 281 | [[[cog 282 | import cog 283 | for i in range(3): 284 | cog.out("%d" % i) 285 | ]]] 286 | 012 287 | [[[end]]] 288 | epilogue 289 | """ 290 | 291 | infile = reindent_block(infile) 292 | self.assertEqual(Cog().process_string(infile), infile) 293 | 294 | def test_no_output_at_all(self): 295 | # If there is absolutely no cog output, that's ok. 296 | infile = """\ 297 | prologue 298 | [[[cog 299 | i = 1 300 | ]]] 301 | [[[end]]] 302 | epilogue 303 | """ 304 | 305 | infile = reindent_block(infile) 306 | self.assertEqual(Cog().process_string(infile), infile) 307 | 308 | def test_purely_blank_line(self): 309 | # If there is a blank line in the cog code with no whitespace 310 | # prefix, that should be OK. 311 | 312 | infile = """\ 313 | prologue 314 | [[[cog 315 | import sys 316 | cog.out("Hello") 317 | $ 318 | cog.out("There") 319 | ]]] 320 | HelloThere 321 | [[[end]]] 322 | epilogue 323 | """ 324 | 325 | infile = reindent_block(infile.replace("$", "")) 326 | self.assertEqual(Cog().process_string(infile), infile) 327 | 328 | def test_empty_outl(self): 329 | # Alexander Belchenko suggested the string argument to outl should 330 | # be optional. Does it work? 331 | 332 | infile = """\ 333 | prologue 334 | [[[cog 335 | cog.outl("x") 336 | cog.outl() 337 | cog.outl("y") 338 | cog.out() # Also optional, a complete no-op. 339 | cog.outl(trimblanklines=True) 340 | cog.outl("z") 341 | ]]] 342 | x 343 | 344 | y 345 | 346 | z 347 | [[[end]]] 348 | epilogue 349 | """ 350 | 351 | infile = reindent_block(infile) 352 | self.assertEqual(Cog().process_string(infile), infile) 353 | 354 | def test_first_line_num(self): 355 | infile = """\ 356 | fooey 357 | [[[cog 358 | cog.outl("started at line number %d" % cog.firstLineNum) 359 | ]]] 360 | started at line number 2 361 | [[[end]]] 362 | blah blah 363 | [[[cog 364 | cog.outl("and again at line %d" % cog.firstLineNum) 365 | ]]] 366 | and again at line 8 367 | [[[end]]] 368 | """ 369 | 370 | infile = reindent_block(infile) 371 | self.assertEqual(Cog().process_string(infile), infile) 372 | 373 | def test_compact_one_line_code(self): 374 | infile = """\ 375 | first line 376 | hey: [[[cog cog.outl("hello %d" % (3*3*3*3)) ]]] looky! 377 | get rid of this! 378 | [[[end]]] 379 | last line 380 | """ 381 | 382 | outfile = """\ 383 | first line 384 | hey: [[[cog cog.outl("hello %d" % (3*3*3*3)) ]]] looky! 385 | hello 81 386 | [[[end]]] 387 | last line 388 | """ 389 | 390 | infile = reindent_block(infile) 391 | self.assertEqual(Cog().process_string(infile), reindent_block(outfile)) 392 | 393 | def test_inside_out_compact(self): 394 | infile = """\ 395 | first line 396 | hey?: ]]] what is this? [[[cog strange! 397 | get rid of this! 398 | [[[end]]] 399 | last line 400 | """ 401 | with self.assertRaisesRegex( 402 | CogError, r"^infile.txt\(2\): Cog code markers inverted$" 403 | ): 404 | Cog().process_string(reindent_block(infile), "infile.txt") 405 | 406 | def test_sharing_globals(self): 407 | infile = """\ 408 | first line 409 | hey: [[[cog s="hey there" ]]] looky! 410 | [[[end]]] 411 | more literal junk. 412 | [[[cog cog.outl(s) ]]] 413 | [[[end]]] 414 | last line 415 | """ 416 | 417 | outfile = """\ 418 | first line 419 | hey: [[[cog s="hey there" ]]] looky! 420 | [[[end]]] 421 | more literal junk. 422 | [[[cog cog.outl(s) ]]] 423 | hey there 424 | [[[end]]] 425 | last line 426 | """ 427 | 428 | infile = reindent_block(infile) 429 | self.assertEqual(Cog().process_string(infile), reindent_block(outfile)) 430 | 431 | def test_assert_in_cog_code(self): 432 | # Check that we can test assertions in cog code in the test framework. 433 | infile = """\ 434 | [[[cog 435 | assert 1 == 2, "Oops" 436 | ]]] 437 | [[[end]]] 438 | """ 439 | infile = reindent_block(infile) 440 | with self.assertRaisesRegex(CogUserException, "AssertionError: Oops"): 441 | Cog().process_string(infile) 442 | 443 | def test_cog_previous(self): 444 | # Check that we can access the previous run's output. 445 | infile = """\ 446 | [[[cog 447 | assert cog.previous == "Hello there!\\n", "WTF??" 448 | cog.out(cog.previous) 449 | cog.outl("Ran again!") 450 | ]]] 451 | Hello there! 452 | [[[end]]] 453 | """ 454 | 455 | outfile = """\ 456 | [[[cog 457 | assert cog.previous == "Hello there!\\n", "WTF??" 458 | cog.out(cog.previous) 459 | cog.outl("Ran again!") 460 | ]]] 461 | Hello there! 462 | Ran again! 463 | [[[end]]] 464 | """ 465 | 466 | infile = reindent_block(infile) 467 | self.assertEqual(Cog().process_string(infile), reindent_block(outfile)) 468 | 469 | 470 | class CogOptionsTests(TestCase): 471 | """Test the CogOptions class.""" 472 | 473 | def test_equality(self): 474 | o = CogOptions() 475 | p = CogOptions() 476 | self.assertEqual(o, p) 477 | o.parse_args(["-r"]) 478 | self.assertNotEqual(o, p) 479 | p.parse_args(["-r"]) 480 | self.assertEqual(o, p) 481 | 482 | def test_cloning(self): 483 | o = CogOptions() 484 | o.parse_args(["-I", "fooey", "-I", "booey", "-s", " /*x*/"]) 485 | p = o.clone() 486 | self.assertEqual(o, p) 487 | p.parse_args(["-I", "huey", "-D", "foo=quux"]) 488 | self.assertNotEqual(o, p) 489 | q = CogOptions() 490 | q.parse_args( 491 | [ 492 | "-I", 493 | "fooey", 494 | "-I", 495 | "booey", 496 | "-s", 497 | " /*x*/", 498 | "-I", 499 | "huey", 500 | "-D", 501 | "foo=quux", 502 | ] 503 | ) 504 | self.assertEqual(p, q) 505 | 506 | def test_combining_flags(self): 507 | # Single-character flags can be combined. 508 | o = CogOptions() 509 | o.parse_args(["-e", "-r", "-z"]) 510 | p = CogOptions() 511 | p.parse_args(["-erz"]) 512 | self.assertEqual(o, p) 513 | 514 | def test_markers(self): 515 | o = CogOptions() 516 | o._parse_markers("a b c") 517 | self.assertEqual("a", o.begin_spec) 518 | self.assertEqual("b", o.end_spec) 519 | self.assertEqual("c", o.end_output) 520 | 521 | def test_markers_switch(self): 522 | o = CogOptions() 523 | o.parse_args(["--markers", "a b c"]) 524 | self.assertEqual("a", o.begin_spec) 525 | self.assertEqual("b", o.end_spec) 526 | self.assertEqual("c", o.end_output) 527 | 528 | 529 | class FileStructureTests(TestCase): 530 | """Test that we're properly strict about the structure of files.""" 531 | 532 | def is_bad(self, infile, msg=None): 533 | infile = reindent_block(infile) 534 | with self.assertRaisesRegex(CogError, "^" + re.escape(msg) + "$"): 535 | Cog().process_string(infile, "infile.txt") 536 | 537 | def test_begin_no_end(self): 538 | infile = """\ 539 | Fooey 540 | #[[[cog 541 | cog.outl('hello') 542 | """ 543 | self.is_bad(infile, "infile.txt(2): Cog block begun but never ended.") 544 | 545 | def test_no_eoo(self): 546 | infile = """\ 547 | Fooey 548 | #[[[cog 549 | cog.outl('hello') 550 | #]]] 551 | """ 552 | self.is_bad(infile, "infile.txt(4): Missing '[[[end]]]' before end of file.") 553 | 554 | infile2 = """\ 555 | Fooey 556 | #[[[cog 557 | cog.outl('hello') 558 | #]]] 559 | #[[[cog 560 | cog.outl('goodbye') 561 | #]]] 562 | """ 563 | self.is_bad(infile2, "infile.txt(5): Unexpected '[[[cog'") 564 | 565 | def test_start_with_end(self): 566 | infile = """\ 567 | #]]] 568 | """ 569 | self.is_bad(infile, "infile.txt(1): Unexpected ']]]'") 570 | 571 | infile2 = """\ 572 | #[[[cog 573 | cog.outl('hello') 574 | #]]] 575 | #[[[end]]] 576 | #]]] 577 | """ 578 | self.is_bad(infile2, "infile.txt(5): Unexpected ']]]'") 579 | 580 | def test_start_with_eoo(self): 581 | infile = """\ 582 | #[[[end]]] 583 | """ 584 | self.is_bad(infile, "infile.txt(1): Unexpected '[[[end]]]'") 585 | 586 | infile2 = """\ 587 | #[[[cog 588 | cog.outl('hello') 589 | #]]] 590 | #[[[end]]] 591 | #[[[end]]] 592 | """ 593 | self.is_bad(infile2, "infile.txt(5): Unexpected '[[[end]]]'") 594 | 595 | def test_no_end(self): 596 | infile = """\ 597 | #[[[cog 598 | cog.outl("hello") 599 | #[[[end]]] 600 | """ 601 | self.is_bad(infile, "infile.txt(3): Unexpected '[[[end]]]'") 602 | 603 | infile2 = """\ 604 | #[[[cog 605 | cog.outl('hello') 606 | #]]] 607 | #[[[end]]] 608 | #[[[cog 609 | cog.outl("hello") 610 | #[[[end]]] 611 | """ 612 | self.is_bad(infile2, "infile.txt(7): Unexpected '[[[end]]]'") 613 | 614 | def test_two_begins(self): 615 | infile = """\ 616 | #[[[cog 617 | #[[[cog 618 | cog.outl("hello") 619 | #]]] 620 | #[[[end]]] 621 | """ 622 | self.is_bad(infile, "infile.txt(2): Unexpected '[[[cog'") 623 | 624 | infile2 = """\ 625 | #[[[cog 626 | cog.outl("hello") 627 | #]]] 628 | #[[[end]]] 629 | #[[[cog 630 | #[[[cog 631 | cog.outl("hello") 632 | #]]] 633 | #[[[end]]] 634 | """ 635 | self.is_bad(infile2, "infile.txt(6): Unexpected '[[[cog'") 636 | 637 | def test_two_ends(self): 638 | infile = """\ 639 | #[[[cog 640 | cog.outl("hello") 641 | #]]] 642 | #]]] 643 | #[[[end]]] 644 | """ 645 | self.is_bad(infile, "infile.txt(4): Unexpected ']]]'") 646 | 647 | infile2 = """\ 648 | #[[[cog 649 | cog.outl("hello") 650 | #]]] 651 | #[[[end]]] 652 | #[[[cog 653 | cog.outl("hello") 654 | #]]] 655 | #]]] 656 | #[[[end]]] 657 | """ 658 | self.is_bad(infile2, "infile.txt(8): Unexpected ']]]'") 659 | 660 | 661 | class CogErrorTests(TestCase): 662 | """Test cases for cog.error().""" 663 | 664 | def test_error_msg(self): 665 | infile = """\ 666 | [[[cog cog.error("This ain't right!")]]] 667 | [[[end]]] 668 | """ 669 | 670 | infile = reindent_block(infile) 671 | with self.assertRaisesRegex(CogGeneratedError, "^This ain't right!$"): 672 | Cog().process_string(infile) 673 | 674 | def test_error_no_msg(self): 675 | infile = """\ 676 | [[[cog cog.error()]]] 677 | [[[end]]] 678 | """ 679 | 680 | infile = reindent_block(infile) 681 | with self.assertRaisesRegex( 682 | CogGeneratedError, "^Error raised by cog generator.$" 683 | ): 684 | Cog().process_string(infile) 685 | 686 | def test_no_error_if_error_not_called(self): 687 | infile = """\ 688 | --[[[cog 689 | --import cog 690 | --for i in range(3): 691 | -- if i > 10: 692 | -- cog.error("Something is amiss!") 693 | -- cog.out("xx%d\\n" % i) 694 | --]]] 695 | xx0 696 | xx1 697 | xx2 698 | --[[[end]]] 699 | """ 700 | 701 | infile = reindent_block(infile) 702 | self.assertEqual(Cog().process_string(infile), infile) 703 | 704 | 705 | class CogGeneratorGetCodeTests(TestCase): 706 | """Tests for CogGenerator.getCode().""" 707 | 708 | def setUp(self): 709 | # All tests get a generator to use, and short same-length names for 710 | # the functions we're going to use. 711 | self.gen = CogGenerator() 712 | self.m = self.gen.parse_marker 713 | self.parse_line = self.gen.parse_line 714 | 715 | def test_empty(self): 716 | self.m("// [[[cog") 717 | self.m("// ]]]") 718 | self.assertEqual(self.gen.get_code(), "") 719 | 720 | def test_simple(self): 721 | self.m("// [[[cog") 722 | self.parse_line(' print "hello"') 723 | self.parse_line(' print "bye"') 724 | self.m("// ]]]") 725 | self.assertEqual(self.gen.get_code(), 'print "hello"\nprint "bye"') 726 | 727 | def test_compressed1(self): 728 | # For a while, I supported compressed code blocks, but no longer. 729 | self.m('// [[[cog: print """') 730 | self.parse_line("// hello") 731 | self.parse_line("// bye") 732 | self.m('// """)]]]') 733 | self.assertEqual(self.gen.get_code(), "hello\nbye") 734 | 735 | def test_compressed2(self): 736 | # For a while, I supported compressed code blocks, but no longer. 737 | self.m('// [[[cog: print """') 738 | self.parse_line("hello") 739 | self.parse_line("bye") 740 | self.m('// """)]]]') 741 | self.assertEqual(self.gen.get_code(), "hello\nbye") 742 | 743 | def test_compressed3(self): 744 | # For a while, I supported compressed code blocks, but no longer. 745 | self.m("// [[[cog") 746 | self.parse_line('print """hello') 747 | self.parse_line("bye") 748 | self.m('// """)]]]') 749 | self.assertEqual(self.gen.get_code(), 'print """hello\nbye') 750 | 751 | def test_compressed4(self): 752 | # For a while, I supported compressed code blocks, but no longer. 753 | self.m('// [[[cog: print """') 754 | self.parse_line("hello") 755 | self.parse_line('bye""")') 756 | self.m("// ]]]") 757 | self.assertEqual(self.gen.get_code(), 'hello\nbye""")') 758 | 759 | def test_no_common_prefix_for_markers(self): 760 | # It's important to be able to use #if 0 to hide lines from a 761 | # C++ compiler. 762 | self.m("#if 0 //[[[cog") 763 | self.parse_line("\timport cog, sys") 764 | self.parse_line("") 765 | self.parse_line("\tprint sys.argv") 766 | self.m("#endif //]]]") 767 | self.assertEqual(self.gen.get_code(), "import cog, sys\n\nprint sys.argv") 768 | 769 | 770 | class TestCaseWithTempDir(TestCase): 771 | def new_cog(self): 772 | """Initialize the cog members for another run.""" 773 | # Create a cog engine, and catch its output. 774 | self.cog = Cog() 775 | self.output = io.StringIO() 776 | self.cog.set_output(stdout=self.output, stderr=self.output) 777 | 778 | def setUp(self): 779 | # Create a temporary directory. 780 | self.tempdir = os.path.join( 781 | tempfile.gettempdir(), "testcog_tempdir_" + str(random.random())[2:] 782 | ) 783 | os.mkdir(self.tempdir) 784 | self.olddir = os.getcwd() 785 | os.chdir(self.tempdir) 786 | self.new_cog() 787 | 788 | def tearDown(self): 789 | os.chdir(self.olddir) 790 | # Get rid of the temporary directory. 791 | shutil.rmtree(self.tempdir) 792 | 793 | def assertFilesSame(self, file_name1, file_name2): 794 | with open(os.path.join(self.tempdir, file_name1), "rb") as f1: 795 | text1 = f1.read() 796 | with open(os.path.join(self.tempdir, file_name2), "rb") as f2: 797 | text2 = f2.read() 798 | self.assertEqual(text1, text2) 799 | 800 | def assertFileContent(self, fname, content): 801 | absname = os.path.join(self.tempdir, fname) 802 | with open(absname, "rb") as f: 803 | file_content = f.read() 804 | self.assertEqual(file_content, content.encode("utf-8")) 805 | 806 | 807 | class ArgumentHandlingTests(TestCaseWithTempDir): 808 | def test_argument_failure(self): 809 | # Return value 2 means usage problem. 810 | self.assertEqual(self.cog.main(["argv0", "-j"]), 2) 811 | output = self.output.getvalue() 812 | self.assertIn("option -j not recognized", output) 813 | with self.assertRaisesRegex(CogUsageError, r"^No files to process$"): 814 | self.cog.callable_main(["argv0"]) 815 | with self.assertRaisesRegex(CogUsageError, r"^option -j not recognized$"): 816 | self.cog.callable_main(["argv0", "-j"]) 817 | 818 | def test_no_dash_o_and_at_file(self): 819 | make_files({"cogfiles.txt": "# Please run cog"}) 820 | with self.assertRaisesRegex(CogUsageError, r"^Can't use -o with @file$"): 821 | self.cog.callable_main(["argv0", "-o", "foo", "@cogfiles.txt"]) 822 | 823 | def test_no_dash_o_and_amp_file(self): 824 | make_files({"cogfiles.txt": "# Please run cog"}) 825 | with self.assertRaisesRegex(CogUsageError, r"^Can't use -o with &file$"): 826 | self.cog.callable_main(["argv0", "-o", "foo", "&cogfiles.txt"]) 827 | 828 | def test_no_diff_without_check(self): 829 | with self.assertRaisesRegex( 830 | CogUsageError, r"^Can't use --diff without --check$" 831 | ): 832 | self.cog.callable_main(["argv0", "--diff"]) 833 | 834 | def test_dash_v(self): 835 | self.assertEqual(self.cog.main(["argv0", "-v"]), 0) 836 | output = self.output.getvalue() 837 | self.assertEqual("Cog version %s\n" % __version__, output) 838 | 839 | def produces_help(self, args): 840 | self.new_cog() 841 | argv = ["argv0"] + args.split() 842 | self.assertEqual(self.cog.main(argv), 0) 843 | self.assertEqual(usage, self.output.getvalue()) 844 | 845 | def test_dash_h(self): 846 | # -h, --help, or -? anywhere on the command line should just print help. 847 | self.produces_help("-h") 848 | self.produces_help("--help") 849 | self.produces_help("-?") 850 | self.produces_help("fooey.txt -h") 851 | self.produces_help("fooey.txt --help") 852 | self.produces_help("-o -r @fooey.txt -? @booey.txt") 853 | 854 | def test_dash_o_and_dash_r(self): 855 | d = { 856 | "cogfile.txt": """\ 857 | # Please run cog 858 | """ 859 | } 860 | 861 | make_files(d) 862 | with self.assertRaisesRegex( 863 | CogUsageError, r"^Can't use -o with -r \(they are opposites\)$" 864 | ): 865 | self.cog.callable_main(["argv0", "-o", "foo", "-r", "cogfile.txt"]) 866 | 867 | def test_dash_z(self): 868 | d = { 869 | "test.cog": """\ 870 | // This is my C++ file. 871 | //[[[cog 872 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 873 | for fn in fnames: 874 | cog.outl("void %s();" % fn) 875 | //]]] 876 | """, 877 | "test.out": """\ 878 | // This is my C++ file. 879 | //[[[cog 880 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 881 | for fn in fnames: 882 | cog.outl("void %s();" % fn) 883 | //]]] 884 | void DoSomething(); 885 | void DoAnotherThing(); 886 | void DoLastThing(); 887 | """, 888 | } 889 | 890 | make_files(d) 891 | with self.assertRaisesRegex( 892 | CogError, r"^test.cog\(6\): Missing '\[\[\[end\]\]\]' before end of file.$" 893 | ): 894 | self.cog.callable_main(["argv0", "-r", "test.cog"]) 895 | self.new_cog() 896 | self.cog.callable_main(["argv0", "-r", "-z", "test.cog"]) 897 | self.assertFilesSame("test.cog", "test.out") 898 | 899 | def test_bad_dash_d(self): 900 | with self.assertRaisesRegex(CogUsageError, r"^-D takes a name=value argument$"): 901 | self.cog.callable_main(["argv0", "-Dfooey", "cog.txt"]) 902 | with self.assertRaisesRegex(CogUsageError, r"^-D takes a name=value argument$"): 903 | self.cog.callable_main(["argv0", "-D", "fooey", "cog.txt"]) 904 | 905 | def test_bad_markers(self): 906 | with self.assertRaisesRegex( 907 | CogUsageError, 908 | r"^--markers requires 3 values separated by spaces, could not parse 'X'$", 909 | ): 910 | self.cog.callable_main(["argv0", "--markers=X"]) 911 | with self.assertRaisesRegex( 912 | CogUsageError, 913 | r"^--markers requires 3 values separated by spaces, could not parse 'A B C D'$", 914 | ): 915 | self.cog.callable_main(["argv0", "--markers=A B C D"]) 916 | 917 | 918 | class TestMain(TestCaseWithTempDir): 919 | def setUp(self): 920 | super().setUp() 921 | self.old_argv = sys.argv[:] 922 | self.old_stderr = sys.stderr 923 | sys.stderr = io.StringIO() 924 | 925 | def tearDown(self): 926 | sys.stderr = self.old_stderr 927 | sys.argv = self.old_argv 928 | sys.modules.pop("mycode", None) 929 | super().tearDown() 930 | 931 | def test_main_function(self): 932 | sys.argv = ["argv0", "-Z"] 933 | ret = main() 934 | self.assertEqual(ret, 2) 935 | stderr = sys.stderr.getvalue() 936 | self.assertEqual(stderr, "option -Z not recognized\n(for help use --help)\n") 937 | 938 | files = { 939 | "test.cog": """\ 940 | //[[[cog 941 | def func(): 942 | import mycode 943 | mycode.boom() 944 | //]]] 945 | //[[[end]]] 946 | ----- 947 | //[[[cog 948 | func() 949 | //]]] 950 | //[[[end]]] 951 | """, 952 | "mycode.py": """\ 953 | def boom(): 954 | [][0] 955 | """, 956 | } 957 | 958 | def test_error_report(self): 959 | self.check_error_report() 960 | 961 | def test_error_report_with_prologue(self): 962 | self.check_error_report("-p", "#1\n#2") 963 | 964 | def check_error_report(self, *args): 965 | """Check that the error report is right.""" 966 | make_files(self.files) 967 | sys.argv = ["argv0"] + list(args) + ["-r", "test.cog"] 968 | main() 969 | expected = reindent_block("""\ 970 | Traceback (most recent call last): 971 | File "test.cog", line 9, in 972 | func() 973 | File "test.cog", line 4, in func 974 | mycode.boom() 975 | File "MYCODE", line 2, in boom 976 | [][0] 977 | IndexError: list index out of range 978 | """) 979 | expected = expected.replace("MYCODE", os.path.abspath("mycode.py")) 980 | assert expected == sys.stderr.getvalue() 981 | 982 | def test_error_in_prologue(self): 983 | make_files(self.files) 984 | sys.argv = ["argv0", "-p", "import mycode; mycode.boom()", "-r", "test.cog"] 985 | main() 986 | expected = reindent_block("""\ 987 | Traceback (most recent call last): 988 | File "", line 1, in 989 | import mycode; mycode.boom() 990 | File "MYCODE", line 2, in boom 991 | [][0] 992 | IndexError: list index out of range 993 | """) 994 | expected = expected.replace("MYCODE", os.path.abspath("mycode.py")) 995 | assert expected == sys.stderr.getvalue() 996 | 997 | 998 | class TestFileHandling(TestCaseWithTempDir): 999 | def test_simple(self): 1000 | d = { 1001 | "test.cog": """\ 1002 | // This is my C++ file. 1003 | //[[[cog 1004 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1005 | for fn in fnames: 1006 | cog.outl("void %s();" % fn) 1007 | //]]] 1008 | //[[[end]]] 1009 | """, 1010 | "test.out": """\ 1011 | // This is my C++ file. 1012 | //[[[cog 1013 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1014 | for fn in fnames: 1015 | cog.outl("void %s();" % fn) 1016 | //]]] 1017 | void DoSomething(); 1018 | void DoAnotherThing(); 1019 | void DoLastThing(); 1020 | //[[[end]]] 1021 | """, 1022 | } 1023 | 1024 | make_files(d) 1025 | self.cog.callable_main(["argv0", "-r", "test.cog"]) 1026 | self.assertFilesSame("test.cog", "test.out") 1027 | output = self.output.getvalue() 1028 | self.assertIn("(changed)", output) 1029 | 1030 | def test_print_output(self): 1031 | d = { 1032 | "test.cog": """\ 1033 | // This is my C++ file. 1034 | //[[[cog 1035 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1036 | for fn in fnames: 1037 | print("void %s();" % fn) 1038 | //]]] 1039 | //[[[end]]] 1040 | """, 1041 | "test.out": """\ 1042 | // This is my C++ file. 1043 | //[[[cog 1044 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1045 | for fn in fnames: 1046 | print("void %s();" % fn) 1047 | //]]] 1048 | void DoSomething(); 1049 | void DoAnotherThing(); 1050 | void DoLastThing(); 1051 | //[[[end]]] 1052 | """, 1053 | } 1054 | 1055 | make_files(d) 1056 | self.cog.callable_main(["argv0", "-rP", "test.cog"]) 1057 | self.assertFilesSame("test.cog", "test.out") 1058 | output = self.output.getvalue() 1059 | self.assertIn("(changed)", output) 1060 | 1061 | def test_wildcards(self): 1062 | d = { 1063 | "test.cog": """\ 1064 | // This is my C++ file. 1065 | //[[[cog 1066 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1067 | for fn in fnames: 1068 | cog.outl("void %s();" % fn) 1069 | //]]] 1070 | //[[[end]]] 1071 | """, 1072 | "test2.cog": """\ 1073 | // This is my C++ file. 1074 | //[[[cog 1075 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1076 | for fn in fnames: 1077 | cog.outl("void %s();" % fn) 1078 | //]]] 1079 | //[[[end]]] 1080 | """, 1081 | "test.out": """\ 1082 | // This is my C++ file. 1083 | //[[[cog 1084 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1085 | for fn in fnames: 1086 | cog.outl("void %s();" % fn) 1087 | //]]] 1088 | void DoSomething(); 1089 | void DoAnotherThing(); 1090 | void DoLastThing(); 1091 | //[[[end]]] 1092 | """, 1093 | "not_this_one.cog": """\ 1094 | // This is my C++ file. 1095 | //[[[cog 1096 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1097 | for fn in fnames: 1098 | cog.outl("void %s();" % fn) 1099 | //]]] 1100 | //[[[end]]] 1101 | """, 1102 | "not_this_one.out": """\ 1103 | // This is my C++ file. 1104 | //[[[cog 1105 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1106 | for fn in fnames: 1107 | cog.outl("void %s();" % fn) 1108 | //]]] 1109 | //[[[end]]] 1110 | """, 1111 | } 1112 | 1113 | make_files(d) 1114 | self.cog.callable_main(["argv0", "-r", "t*.cog"]) 1115 | self.assertFilesSame("test.cog", "test.out") 1116 | self.assertFilesSame("test2.cog", "test.out") 1117 | self.assertFilesSame("not_this_one.cog", "not_this_one.out") 1118 | output = self.output.getvalue() 1119 | self.assertIn("(changed)", output) 1120 | 1121 | def test_output_file(self): 1122 | # -o sets the output file. 1123 | d = { 1124 | "test.cog": """\ 1125 | // This is my C++ file. 1126 | //[[[cog 1127 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1128 | for fn in fnames: 1129 | cog.outl("void %s();" % fn) 1130 | //]]] 1131 | //[[[end]]] 1132 | """, 1133 | "test.out": """\ 1134 | // This is my C++ file. 1135 | //[[[cog 1136 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 1137 | for fn in fnames: 1138 | cog.outl("void %s();" % fn) 1139 | //]]] 1140 | void DoSomething(); 1141 | void DoAnotherThing(); 1142 | void DoLastThing(); 1143 | //[[[end]]] 1144 | """, 1145 | } 1146 | 1147 | make_files(d) 1148 | self.cog.callable_main(["argv0", "-o", "in/a/dir/test.cogged", "test.cog"]) 1149 | self.assertFilesSame("in/a/dir/test.cogged", "test.out") 1150 | 1151 | def test_at_file(self): 1152 | d = { 1153 | "one.cog": """\ 1154 | //[[[cog 1155 | cog.outl("hello world") 1156 | //]]] 1157 | //[[[end]]] 1158 | """, 1159 | "one.out": """\ 1160 | //[[[cog 1161 | cog.outl("hello world") 1162 | //]]] 1163 | hello world 1164 | //[[[end]]] 1165 | """, 1166 | "two.cog": """\ 1167 | //[[[cog 1168 | cog.outl("goodbye cruel world") 1169 | //]]] 1170 | //[[[end]]] 1171 | """, 1172 | "two.out": """\ 1173 | //[[[cog 1174 | cog.outl("goodbye cruel world") 1175 | //]]] 1176 | goodbye cruel world 1177 | //[[[end]]] 1178 | """, 1179 | "cogfiles.txt": """\ 1180 | # Please run cog 1181 | one.cog 1182 | 1183 | two.cog 1184 | """, 1185 | } 1186 | 1187 | make_files(d) 1188 | self.cog.callable_main(["argv0", "-r", "@cogfiles.txt"]) 1189 | self.assertFilesSame("one.cog", "one.out") 1190 | self.assertFilesSame("two.cog", "two.out") 1191 | output = self.output.getvalue() 1192 | self.assertIn("(changed)", output) 1193 | 1194 | def test_nested_at_file(self): 1195 | d = { 1196 | "one.cog": """\ 1197 | //[[[cog 1198 | cog.outl("hello world") 1199 | //]]] 1200 | //[[[end]]] 1201 | """, 1202 | "one.out": """\ 1203 | //[[[cog 1204 | cog.outl("hello world") 1205 | //]]] 1206 | hello world 1207 | //[[[end]]] 1208 | """, 1209 | "two.cog": """\ 1210 | //[[[cog 1211 | cog.outl("goodbye cruel world") 1212 | //]]] 1213 | //[[[end]]] 1214 | """, 1215 | "two.out": """\ 1216 | //[[[cog 1217 | cog.outl("goodbye cruel world") 1218 | //]]] 1219 | goodbye cruel world 1220 | //[[[end]]] 1221 | """, 1222 | "cogfiles.txt": """\ 1223 | # Please run cog 1224 | one.cog 1225 | @cogfiles2.txt 1226 | """, 1227 | "cogfiles2.txt": """\ 1228 | # This one too, please. 1229 | two.cog 1230 | """, 1231 | } 1232 | 1233 | make_files(d) 1234 | self.cog.callable_main(["argv0", "-r", "@cogfiles.txt"]) 1235 | self.assertFilesSame("one.cog", "one.out") 1236 | self.assertFilesSame("two.cog", "two.out") 1237 | output = self.output.getvalue() 1238 | self.assertIn("(changed)", output) 1239 | 1240 | def test_at_file_with_args(self): 1241 | d = { 1242 | "both.cog": """\ 1243 | //[[[cog 1244 | cog.outl("one: %s" % ('one' in globals())) 1245 | cog.outl("two: %s" % ('two' in globals())) 1246 | //]]] 1247 | //[[[end]]] 1248 | """, 1249 | "one.out": """\ 1250 | //[[[cog 1251 | cog.outl("one: %s" % ('one' in globals())) 1252 | cog.outl("two: %s" % ('two' in globals())) 1253 | //]]] 1254 | one: True // ONE 1255 | two: False // ONE 1256 | //[[[end]]] 1257 | """, 1258 | "two.out": """\ 1259 | //[[[cog 1260 | cog.outl("one: %s" % ('one' in globals())) 1261 | cog.outl("two: %s" % ('two' in globals())) 1262 | //]]] 1263 | one: False // TWO 1264 | two: True // TWO 1265 | //[[[end]]] 1266 | """, 1267 | "cogfiles.txt": """\ 1268 | # Please run cog 1269 | both.cog -o in/a/dir/both.one -s ' // ONE' -D one=x 1270 | both.cog -o in/a/dir/both.two -s ' // TWO' -D two=x 1271 | """, 1272 | } 1273 | 1274 | make_files(d) 1275 | self.cog.callable_main(["argv0", "@cogfiles.txt"]) 1276 | self.assertFilesSame("in/a/dir/both.one", "one.out") 1277 | self.assertFilesSame("in/a/dir/both.two", "two.out") 1278 | 1279 | def test_at_file_with_bad_arg_combo(self): 1280 | d = { 1281 | "both.cog": """\ 1282 | //[[[cog 1283 | cog.outl("one: %s" % ('one' in globals())) 1284 | cog.outl("two: %s" % ('two' in globals())) 1285 | //]]] 1286 | //[[[end]]] 1287 | """, 1288 | "cogfiles.txt": """\ 1289 | # Please run cog 1290 | both.cog 1291 | both.cog -d # This is bad: -r and -d 1292 | """, 1293 | } 1294 | 1295 | make_files(d) 1296 | with self.assertRaisesRegex( 1297 | CogUsageError, 1298 | r"^Can't use -d with -r \(or you would delete all your source!\)$", 1299 | ): 1300 | self.cog.callable_main(["argv0", "-r", "@cogfiles.txt"]) 1301 | 1302 | def test_at_file_with_tricky_filenames(self): 1303 | def fix_backslashes(files_txt): 1304 | """Make the contents of a files.txt sensitive to the platform.""" 1305 | if sys.platform != "win32": 1306 | files_txt = files_txt.replace("\\", "/") 1307 | return files_txt 1308 | 1309 | d = { 1310 | "one 1.cog": """\ 1311 | //[[[cog cog.outl("hello world") ]]] 1312 | """, 1313 | "one.out": """\ 1314 | //[[[cog cog.outl("hello world") ]]] 1315 | hello world //xxx 1316 | """, 1317 | "subdir": { 1318 | "subback.cog": """\ 1319 | //[[[cog cog.outl("down deep with backslashes") ]]] 1320 | """, 1321 | "subfwd.cog": """\ 1322 | //[[[cog cog.outl("down deep with slashes") ]]] 1323 | """, 1324 | }, 1325 | "subback.out": """\ 1326 | //[[[cog cog.outl("down deep with backslashes") ]]] 1327 | down deep with backslashes //yyy 1328 | """, 1329 | "subfwd.out": """\ 1330 | //[[[cog cog.outl("down deep with slashes") ]]] 1331 | down deep with slashes //zzz 1332 | """, 1333 | "cogfiles.txt": fix_backslashes("""\ 1334 | # Please run cog 1335 | 'one 1.cog' -s ' //xxx' 1336 | subdir\\subback.cog -s ' //yyy' 1337 | subdir/subfwd.cog -s ' //zzz' 1338 | """), 1339 | } 1340 | 1341 | make_files(d) 1342 | self.cog.callable_main(["argv0", "-z", "-r", "@cogfiles.txt"]) 1343 | self.assertFilesSame("one 1.cog", "one.out") 1344 | self.assertFilesSame("subdir/subback.cog", "subback.out") 1345 | self.assertFilesSame("subdir/subfwd.cog", "subfwd.out") 1346 | 1347 | def test_amp_file(self): 1348 | d = { 1349 | "code": { 1350 | "files_to_cog": """\ 1351 | # A locally resolved file name. 1352 | test.cog 1353 | """, 1354 | "test.cog": """\ 1355 | //[[[cog 1356 | import myampsubmodule 1357 | //]]] 1358 | //[[[end]]] 1359 | """, 1360 | "test.out": """\ 1361 | //[[[cog 1362 | import myampsubmodule 1363 | //]]] 1364 | Hello from myampsubmodule 1365 | //[[[end]]] 1366 | """, 1367 | "myampsubmodule.py": """\ 1368 | import cog 1369 | cog.outl("Hello from myampsubmodule") 1370 | """, 1371 | } 1372 | } 1373 | 1374 | make_files(d) 1375 | print(os.path.abspath("code/test.out")) 1376 | self.cog.callable_main(["argv0", "-r", "&code/files_to_cog"]) 1377 | self.assertFilesSame("code/test.cog", "code/test.out") 1378 | 1379 | def run_with_verbosity(self, verbosity): 1380 | d = { 1381 | "unchanged.cog": """\ 1382 | //[[[cog 1383 | cog.outl("hello world") 1384 | //]]] 1385 | hello world 1386 | //[[[end]]] 1387 | """, 1388 | "changed.cog": """\ 1389 | //[[[cog 1390 | cog.outl("goodbye cruel world") 1391 | //]]] 1392 | //[[[end]]] 1393 | """, 1394 | "cogfiles.txt": """\ 1395 | unchanged.cog 1396 | changed.cog 1397 | """, 1398 | } 1399 | 1400 | make_files(d) 1401 | self.cog.callable_main( 1402 | ["argv0", "-r", "--verbosity=" + verbosity, "@cogfiles.txt"] 1403 | ) 1404 | output = self.output.getvalue() 1405 | return output 1406 | 1407 | def test_verbosity0(self): 1408 | output = self.run_with_verbosity("0") 1409 | self.assertEqual(output, "") 1410 | 1411 | def test_verbosity1(self): 1412 | output = self.run_with_verbosity("1") 1413 | self.assertEqual(output, "Cogging changed.cog (changed)\n") 1414 | 1415 | def test_verbosity2(self): 1416 | output = self.run_with_verbosity("2") 1417 | self.assertEqual( 1418 | output, "Cogging unchanged.cog\nCogging changed.cog (changed)\n" 1419 | ) 1420 | 1421 | def test_change_dir(self): 1422 | # The code can change directories, cog will move us back. 1423 | d = { 1424 | "sub": { 1425 | "data.txt": "Hello!", 1426 | }, 1427 | "test.cog": """\ 1428 | //[[[cog 1429 | import os 1430 | os.chdir("sub") 1431 | cog.outl(open("data.txt").read()) 1432 | //]]] 1433 | //[[[end]]] 1434 | """, 1435 | "test.out": """\ 1436 | //[[[cog 1437 | import os 1438 | os.chdir("sub") 1439 | cog.outl(open("data.txt").read()) 1440 | //]]] 1441 | Hello! 1442 | //[[[end]]] 1443 | """, 1444 | } 1445 | 1446 | make_files(d) 1447 | self.cog.callable_main(["argv0", "-r", "test.cog"]) 1448 | self.assertFilesSame("test.cog", "test.out") 1449 | output = self.output.getvalue() 1450 | self.assertIn("(changed)", output) 1451 | 1452 | 1453 | class CogTestLineEndings(TestCaseWithTempDir): 1454 | """Tests for -U option (force LF line-endings in output).""" 1455 | 1456 | lines_in = [ 1457 | "Some text.", 1458 | "//[[[cog", 1459 | 'cog.outl("Cog text")', 1460 | "//]]]", 1461 | "gobbledegook.", 1462 | "//[[[end]]]", 1463 | "epilogue.", 1464 | "", 1465 | ] 1466 | 1467 | lines_out = [ 1468 | "Some text.", 1469 | "//[[[cog", 1470 | 'cog.outl("Cog text")', 1471 | "//]]]", 1472 | "Cog text", 1473 | "//[[[end]]]", 1474 | "epilogue.", 1475 | "", 1476 | ] 1477 | 1478 | def test_output_native_eol(self): 1479 | make_files({"infile": "\n".join(self.lines_in)}) 1480 | self.cog.callable_main(["argv0", "-o", "outfile", "infile"]) 1481 | self.assertFileContent("outfile", os.linesep.join(self.lines_out)) 1482 | 1483 | def test_output_lf_eol(self): 1484 | make_files({"infile": "\n".join(self.lines_in)}) 1485 | self.cog.callable_main(["argv0", "-U", "-o", "outfile", "infile"]) 1486 | self.assertFileContent("outfile", "\n".join(self.lines_out)) 1487 | 1488 | def test_replace_native_eol(self): 1489 | make_files({"test.cog": "\n".join(self.lines_in)}) 1490 | self.cog.callable_main(["argv0", "-r", "test.cog"]) 1491 | self.assertFileContent("test.cog", os.linesep.join(self.lines_out)) 1492 | 1493 | def test_replace_lf_eol(self): 1494 | make_files({"test.cog": "\n".join(self.lines_in)}) 1495 | self.cog.callable_main(["argv0", "-U", "-r", "test.cog"]) 1496 | self.assertFileContent("test.cog", "\n".join(self.lines_out)) 1497 | 1498 | 1499 | class CogTestCharacterEncoding(TestCaseWithTempDir): 1500 | def test_simple(self): 1501 | d = { 1502 | "test.cog": b"""\ 1503 | // This is my C++ file. 1504 | //[[[cog 1505 | cog.outl("// Unicode: \xe1\x88\xb4 (U+1234)") 1506 | //]]] 1507 | //[[[end]]] 1508 | """, 1509 | "test.out": b"""\ 1510 | // This is my C++ file. 1511 | //[[[cog 1512 | cog.outl("// Unicode: \xe1\x88\xb4 (U+1234)") 1513 | //]]] 1514 | // Unicode: \xe1\x88\xb4 (U+1234) 1515 | //[[[end]]] 1516 | """.replace(b"\n", os.linesep.encode()), 1517 | } 1518 | 1519 | make_files(d) 1520 | self.cog.callable_main(["argv0", "-r", "test.cog"]) 1521 | self.assertFilesSame("test.cog", "test.out") 1522 | output = self.output.getvalue() 1523 | self.assertIn("(changed)", output) 1524 | 1525 | def test_file_encoding_option(self): 1526 | d = { 1527 | "test.cog": b"""\ 1528 | // \xca\xee\xe4\xe8\xf0\xe2\xea\xe0 Windows 1529 | //[[[cog 1530 | cog.outl("\xd1\xfa\xe5\xf8\xfc \xe5\xf9\xb8 \xfd\xf2\xe8\xf5 \xec\xff\xe3\xea\xe8\xf5 \xf4\xf0\xe0\xed\xf6\xf3\xe7\xf1\xea\xe8\xf5 \xe1\xf3\xeb\xee\xea \xe4\xe0 \xe2\xfb\xef\xe5\xe9 \xf7\xe0\xfe") 1531 | //]]] 1532 | //[[[end]]] 1533 | """, 1534 | "test.out": b"""\ 1535 | // \xca\xee\xe4\xe8\xf0\xe2\xea\xe0 Windows 1536 | //[[[cog 1537 | cog.outl("\xd1\xfa\xe5\xf8\xfc \xe5\xf9\xb8 \xfd\xf2\xe8\xf5 \xec\xff\xe3\xea\xe8\xf5 \xf4\xf0\xe0\xed\xf6\xf3\xe7\xf1\xea\xe8\xf5 \xe1\xf3\xeb\xee\xea \xe4\xe0 \xe2\xfb\xef\xe5\xe9 \xf7\xe0\xfe") 1538 | //]]] 1539 | \xd1\xfa\xe5\xf8\xfc \xe5\xf9\xb8 \xfd\xf2\xe8\xf5 \xec\xff\xe3\xea\xe8\xf5 \xf4\xf0\xe0\xed\xf6\xf3\xe7\xf1\xea\xe8\xf5 \xe1\xf3\xeb\xee\xea \xe4\xe0 \xe2\xfb\xef\xe5\xe9 \xf7\xe0\xfe 1540 | //[[[end]]] 1541 | """.replace(b"\n", os.linesep.encode()), 1542 | } 1543 | 1544 | make_files(d) 1545 | self.cog.callable_main(["argv0", "-n", "cp1251", "-r", "test.cog"]) 1546 | self.assertFilesSame("test.cog", "test.out") 1547 | output = self.output.getvalue() 1548 | self.assertIn("(changed)", output) 1549 | 1550 | 1551 | class TestCaseWithImports(TestCaseWithTempDir): 1552 | """Automatic resetting of sys.modules for tests that import modules. 1553 | 1554 | When running tests which import modules, the sys.modules list 1555 | leaks from one test to the next. This test case class scrubs 1556 | the list after each run to keep the tests isolated from each other. 1557 | 1558 | """ 1559 | 1560 | def setUp(self): 1561 | super().setUp() 1562 | self.sysmodulekeys = list(sys.modules) 1563 | 1564 | def tearDown(self): 1565 | modstoscrub = [ 1566 | modname for modname in sys.modules if modname not in self.sysmodulekeys 1567 | ] 1568 | for modname in modstoscrub: 1569 | del sys.modules[modname] 1570 | super().tearDown() 1571 | 1572 | 1573 | class CogIncludeTests(TestCaseWithImports): 1574 | dincludes = { 1575 | "test.cog": """\ 1576 | //[[[cog 1577 | import mymodule 1578 | //]]] 1579 | //[[[end]]] 1580 | """, 1581 | "test.out": """\ 1582 | //[[[cog 1583 | import mymodule 1584 | //]]] 1585 | Hello from mymodule 1586 | //[[[end]]] 1587 | """, 1588 | "test2.out": """\ 1589 | //[[[cog 1590 | import mymodule 1591 | //]]] 1592 | Hello from mymodule in inc2 1593 | //[[[end]]] 1594 | """, 1595 | "include": { 1596 | "mymodule.py": """\ 1597 | import cog 1598 | cog.outl("Hello from mymodule") 1599 | """ 1600 | }, 1601 | "inc2": { 1602 | "mymodule.py": """\ 1603 | import cog 1604 | cog.outl("Hello from mymodule in inc2") 1605 | """ 1606 | }, 1607 | "inc3": { 1608 | "someothermodule.py": """\ 1609 | import cog 1610 | cog.outl("This is some other module.") 1611 | """ 1612 | }, 1613 | } 1614 | 1615 | def test_need_include_path(self): 1616 | # Try it without the -I, to see that an ImportError happens. 1617 | make_files(self.dincludes) 1618 | msg = "(ImportError|ModuleNotFoundError): No module named '?mymodule'?" 1619 | with self.assertRaisesRegex(CogUserException, msg): 1620 | self.cog.callable_main(["argv0", "-r", "test.cog"]) 1621 | 1622 | def test_include_path(self): 1623 | # Test that -I adds include directories properly. 1624 | make_files(self.dincludes) 1625 | self.cog.callable_main(["argv0", "-r", "-I", "include", "test.cog"]) 1626 | self.assertFilesSame("test.cog", "test.out") 1627 | 1628 | def test_two_include_paths(self): 1629 | # Test that two -I's add include directories properly. 1630 | make_files(self.dincludes) 1631 | self.cog.callable_main( 1632 | ["argv0", "-r", "-I", "include", "-I", "inc2", "test.cog"] 1633 | ) 1634 | self.assertFilesSame("test.cog", "test.out") 1635 | 1636 | def test_two_include_paths2(self): 1637 | # Test that two -I's add include directories properly. 1638 | make_files(self.dincludes) 1639 | self.cog.callable_main( 1640 | ["argv0", "-r", "-I", "inc2", "-I", "include", "test.cog"] 1641 | ) 1642 | self.assertFilesSame("test.cog", "test2.out") 1643 | 1644 | def test_useless_include_path(self): 1645 | # Test that the search will continue past the first directory. 1646 | make_files(self.dincludes) 1647 | self.cog.callable_main( 1648 | ["argv0", "-r", "-I", "inc3", "-I", "include", "test.cog"] 1649 | ) 1650 | self.assertFilesSame("test.cog", "test.out") 1651 | 1652 | def test_sys_path_is_unchanged(self): 1653 | d = { 1654 | "bad.cog": """\ 1655 | //[[[cog cog.error("Oh no!") ]]] 1656 | //[[[end]]] 1657 | """, 1658 | "good.cog": """\ 1659 | //[[[cog cog.outl("Oh yes!") ]]] 1660 | //[[[end]]] 1661 | """, 1662 | } 1663 | 1664 | make_files(d) 1665 | # Is it unchanged just by creating a cog engine? 1666 | oldsyspath = sys.path[:] 1667 | self.new_cog() 1668 | self.assertEqual(oldsyspath, sys.path) 1669 | # Is it unchanged for a successful run? 1670 | self.new_cog() 1671 | self.cog.callable_main(["argv0", "-r", "good.cog"]) 1672 | self.assertEqual(oldsyspath, sys.path) 1673 | # Is it unchanged for a successful run with includes? 1674 | self.new_cog() 1675 | self.cog.callable_main(["argv0", "-r", "-I", "xyzzy", "good.cog"]) 1676 | self.assertEqual(oldsyspath, sys.path) 1677 | # Is it unchanged for a successful run with two includes? 1678 | self.new_cog() 1679 | self.cog.callable_main(["argv0", "-r", "-I", "xyzzy", "-I", "quux", "good.cog"]) 1680 | self.assertEqual(oldsyspath, sys.path) 1681 | # Is it unchanged for a failed run? 1682 | self.new_cog() 1683 | with self.assertRaisesRegex(CogError, r"^Oh no!$"): 1684 | self.cog.callable_main(["argv0", "-r", "bad.cog"]) 1685 | self.assertEqual(oldsyspath, sys.path) 1686 | # Is it unchanged for a failed run with includes? 1687 | self.new_cog() 1688 | with self.assertRaisesRegex(CogError, r"^Oh no!$"): 1689 | self.cog.callable_main(["argv0", "-r", "-I", "xyzzy", "bad.cog"]) 1690 | self.assertEqual(oldsyspath, sys.path) 1691 | # Is it unchanged for a failed run with two includes? 1692 | self.new_cog() 1693 | with self.assertRaisesRegex(CogError, r"^Oh no!$"): 1694 | self.cog.callable_main( 1695 | ["argv0", "-r", "-I", "xyzzy", "-I", "quux", "bad.cog"] 1696 | ) 1697 | self.assertEqual(oldsyspath, sys.path) 1698 | 1699 | def test_sub_directories(self): 1700 | # Test that relative paths on the command line work, with includes. 1701 | 1702 | d = { 1703 | "code": { 1704 | "test.cog": """\ 1705 | //[[[cog 1706 | import mysubmodule 1707 | //]]] 1708 | //[[[end]]] 1709 | """, 1710 | "test.out": """\ 1711 | //[[[cog 1712 | import mysubmodule 1713 | //]]] 1714 | Hello from mysubmodule 1715 | //[[[end]]] 1716 | """, 1717 | "mysubmodule.py": """\ 1718 | import cog 1719 | cog.outl("Hello from mysubmodule") 1720 | """, 1721 | } 1722 | } 1723 | 1724 | make_files(d) 1725 | # We should be able to invoke cog without the -I switch, and it will 1726 | # auto-include the current directory 1727 | self.cog.callable_main(["argv0", "-r", "code/test.cog"]) 1728 | self.assertFilesSame("code/test.cog", "code/test.out") 1729 | 1730 | 1731 | class CogTestsInFiles(TestCaseWithTempDir): 1732 | def test_warn_if_no_cog_code(self): 1733 | # Test that the -e switch warns if there is no Cog code. 1734 | d = { 1735 | "with.cog": """\ 1736 | //[[[cog 1737 | cog.outl("hello world") 1738 | //]]] 1739 | hello world 1740 | //[[[end]]] 1741 | """, 1742 | "without.cog": """\ 1743 | There's no cog 1744 | code in this file. 1745 | """, 1746 | } 1747 | 1748 | make_files(d) 1749 | self.cog.callable_main(["argv0", "-e", "with.cog"]) 1750 | output = self.output.getvalue() 1751 | self.assertNotIn("Warning", output) 1752 | self.new_cog() 1753 | self.cog.callable_main(["argv0", "-e", "without.cog"]) 1754 | output = self.output.getvalue() 1755 | self.assertIn("Warning: no cog code found in without.cog", output) 1756 | self.new_cog() 1757 | self.cog.callable_main(["argv0", "without.cog"]) 1758 | output = self.output.getvalue() 1759 | self.assertNotIn("Warning", output) 1760 | 1761 | def test_file_name_props(self): 1762 | d = { 1763 | "cog1.txt": """\ 1764 | //[[[cog 1765 | cog.outl("This is %s in, %s out" % (cog.inFile, cog.outFile)) 1766 | //]]] 1767 | this is cog1.txt in, cog1.txt out 1768 | [[[end]]] 1769 | """, 1770 | "cog1.out": """\ 1771 | //[[[cog 1772 | cog.outl("This is %s in, %s out" % (cog.inFile, cog.outFile)) 1773 | //]]] 1774 | This is cog1.txt in, cog1.txt out 1775 | [[[end]]] 1776 | """, 1777 | "cog1out.out": """\ 1778 | //[[[cog 1779 | cog.outl("This is %s in, %s out" % (cog.inFile, cog.outFile)) 1780 | //]]] 1781 | This is cog1.txt in, cog1out.txt out 1782 | [[[end]]] 1783 | """, 1784 | } 1785 | 1786 | make_files(d) 1787 | self.cog.callable_main(["argv0", "-r", "cog1.txt"]) 1788 | self.assertFilesSame("cog1.txt", "cog1.out") 1789 | self.new_cog() 1790 | self.cog.callable_main(["argv0", "-o", "cog1out.txt", "cog1.txt"]) 1791 | self.assertFilesSame("cog1out.txt", "cog1out.out") 1792 | 1793 | def test_globals_dont_cross_files(self): 1794 | # Make sure that global values don't get shared between files. 1795 | d = { 1796 | "one.cog": """\ 1797 | //[[[cog s = "This was set in one.cog" ]]] 1798 | //[[[end]]] 1799 | //[[[cog cog.outl(s) ]]] 1800 | //[[[end]]] 1801 | """, 1802 | "one.out": """\ 1803 | //[[[cog s = "This was set in one.cog" ]]] 1804 | //[[[end]]] 1805 | //[[[cog cog.outl(s) ]]] 1806 | This was set in one.cog 1807 | //[[[end]]] 1808 | """, 1809 | "two.cog": """\ 1810 | //[[[cog 1811 | try: 1812 | cog.outl(s) 1813 | except NameError: 1814 | cog.outl("s isn't set!") 1815 | //]]] 1816 | //[[[end]]] 1817 | """, 1818 | "two.out": """\ 1819 | //[[[cog 1820 | try: 1821 | cog.outl(s) 1822 | except NameError: 1823 | cog.outl("s isn't set!") 1824 | //]]] 1825 | s isn't set! 1826 | //[[[end]]] 1827 | """, 1828 | "cogfiles.txt": """\ 1829 | # Please run cog 1830 | one.cog 1831 | 1832 | two.cog 1833 | """, 1834 | } 1835 | 1836 | make_files(d) 1837 | self.cog.callable_main(["argv0", "-r", "@cogfiles.txt"]) 1838 | self.assertFilesSame("one.cog", "one.out") 1839 | self.assertFilesSame("two.cog", "two.out") 1840 | output = self.output.getvalue() 1841 | self.assertIn("(changed)", output) 1842 | 1843 | def test_remove_generated_output(self): 1844 | d = { 1845 | "cog1.txt": """\ 1846 | //[[[cog 1847 | cog.outl("This line was generated.") 1848 | //]]] 1849 | This line was generated. 1850 | //[[[end]]] 1851 | This line was not. 1852 | """, 1853 | "cog1.out": """\ 1854 | //[[[cog 1855 | cog.outl("This line was generated.") 1856 | //]]] 1857 | //[[[end]]] 1858 | This line was not. 1859 | """, 1860 | "cog1.out2": """\ 1861 | //[[[cog 1862 | cog.outl("This line was generated.") 1863 | //]]] 1864 | This line was generated. 1865 | //[[[end]]] 1866 | This line was not. 1867 | """, 1868 | } 1869 | 1870 | make_files(d) 1871 | # Remove generated output. 1872 | self.cog.callable_main(["argv0", "-r", "-x", "cog1.txt"]) 1873 | self.assertFilesSame("cog1.txt", "cog1.out") 1874 | self.new_cog() 1875 | # Regenerate the generated output. 1876 | self.cog.callable_main(["argv0", "-r", "cog1.txt"]) 1877 | self.assertFilesSame("cog1.txt", "cog1.out2") 1878 | self.new_cog() 1879 | # Remove the generated output again. 1880 | self.cog.callable_main(["argv0", "-r", "-x", "cog1.txt"]) 1881 | self.assertFilesSame("cog1.txt", "cog1.out") 1882 | 1883 | def test_msg_call(self): 1884 | infile = """\ 1885 | #[[[cog 1886 | cog.msg("Hello there!") 1887 | #]]] 1888 | #[[[end]]] 1889 | """ 1890 | infile = reindent_block(infile) 1891 | self.assertEqual(self.cog.process_string(infile), infile) 1892 | output = self.output.getvalue() 1893 | self.assertEqual(output, "Message: Hello there!\n") 1894 | 1895 | def test_error_message_has_no_traceback(self): 1896 | # Test that a Cog error is printed to stderr with no traceback. 1897 | 1898 | d = { 1899 | "cog1.txt": """\ 1900 | //[[[cog 1901 | cog.outl("This line was newly") 1902 | cog.outl("generated by cog") 1903 | cog.outl("blah blah.") 1904 | //]]] 1905 | Xhis line was newly 1906 | generated by cog 1907 | blah blah. 1908 | //[[[end]]] (sum: qFQJguWta5) 1909 | """, 1910 | } 1911 | 1912 | make_files(d) 1913 | stderr = io.StringIO() 1914 | self.cog.set_output(stderr=stderr) 1915 | self.cog.main(["argv0", "-c", "-r", "cog1.txt"]) 1916 | self.assertEqual(self.output.getvalue(), "Cogging cog1.txt\n") 1917 | self.assertEqual( 1918 | stderr.getvalue(), 1919 | "cog1.txt(9): Output has been edited! Delete old checksum to unprotect.\n", 1920 | ) 1921 | 1922 | def test_dash_d(self): 1923 | d = { 1924 | "test.cog": """\ 1925 | --[[[cog cog.outl("Defined fooey as " + fooey) ]]] 1926 | --[[[end]]] 1927 | """, 1928 | "test.kablooey": """\ 1929 | --[[[cog cog.outl("Defined fooey as " + fooey) ]]] 1930 | Defined fooey as kablooey 1931 | --[[[end]]] 1932 | """, 1933 | "test.einstein": """\ 1934 | --[[[cog cog.outl("Defined fooey as " + fooey) ]]] 1935 | Defined fooey as e=mc2 1936 | --[[[end]]] 1937 | """, 1938 | } 1939 | 1940 | make_files(d) 1941 | self.cog.callable_main(["argv0", "-r", "-D", "fooey=kablooey", "test.cog"]) 1942 | self.assertFilesSame("test.cog", "test.kablooey") 1943 | make_files(d) 1944 | self.cog.callable_main(["argv0", "-r", "-Dfooey=kablooey", "test.cog"]) 1945 | self.assertFilesSame("test.cog", "test.kablooey") 1946 | make_files(d) 1947 | self.cog.callable_main(["argv0", "-r", "-Dfooey=e=mc2", "test.cog"]) 1948 | self.assertFilesSame("test.cog", "test.einstein") 1949 | make_files(d) 1950 | self.cog.callable_main( 1951 | ["argv0", "-r", "-Dbar=quux", "-Dfooey=kablooey", "test.cog"] 1952 | ) 1953 | self.assertFilesSame("test.cog", "test.kablooey") 1954 | make_files(d) 1955 | self.cog.callable_main( 1956 | ["argv0", "-r", "-Dfooey=kablooey", "-Dbar=quux", "test.cog"] 1957 | ) 1958 | self.assertFilesSame("test.cog", "test.kablooey") 1959 | make_files(d) 1960 | self.cog.callable_main( 1961 | ["argv0", "-r", "-Dfooey=gooey", "-Dfooey=kablooey", "test.cog"] 1962 | ) 1963 | self.assertFilesSame("test.cog", "test.kablooey") 1964 | 1965 | def test_output_to_stdout(self): 1966 | d = { 1967 | "test.cog": """\ 1968 | --[[[cog cog.outl('Hey there!') ]]] 1969 | --[[[end]]] 1970 | """ 1971 | } 1972 | 1973 | make_files(d) 1974 | stderr = io.StringIO() 1975 | self.cog.set_output(stderr=stderr) 1976 | self.cog.callable_main(["argv0", "test.cog"]) 1977 | output = self.output.getvalue() 1978 | outerr = stderr.getvalue() 1979 | self.assertEqual( 1980 | output, "--[[[cog cog.outl('Hey there!') ]]]\nHey there!\n--[[[end]]]\n" 1981 | ) 1982 | self.assertEqual(outerr, "") 1983 | 1984 | def test_read_from_stdin(self): 1985 | stdin = io.StringIO("--[[[cog cog.outl('Wow') ]]]\n--[[[end]]]\n") 1986 | 1987 | def restore_stdin(old_stdin): 1988 | sys.stdin = old_stdin 1989 | 1990 | self.addCleanup(restore_stdin, sys.stdin) 1991 | sys.stdin = stdin 1992 | 1993 | stderr = io.StringIO() 1994 | self.cog.set_output(stderr=stderr) 1995 | self.cog.callable_main(["argv0", "-"]) 1996 | output = self.output.getvalue() 1997 | outerr = stderr.getvalue() 1998 | self.assertEqual(output, "--[[[cog cog.outl('Wow') ]]]\nWow\n--[[[end]]]\n") 1999 | self.assertEqual(outerr, "") 2000 | 2001 | def test_suffix_output_lines(self): 2002 | d = { 2003 | "test.cog": """\ 2004 | Hey there. 2005 | ;[[[cog cog.outl('a\\nb\\n \\nc') ]]] 2006 | ;[[[end]]] 2007 | Good bye. 2008 | """, 2009 | "test.out": """\ 2010 | Hey there. 2011 | ;[[[cog cog.outl('a\\nb\\n \\nc') ]]] 2012 | a (foo) 2013 | b (foo) 2014 | """ # These three trailing spaces are important. 2015 | # The suffix is not applied to completely blank lines. 2016 | """ 2017 | c (foo) 2018 | ;[[[end]]] 2019 | Good bye. 2020 | """, 2021 | } 2022 | 2023 | make_files(d) 2024 | self.cog.callable_main(["argv0", "-r", "-s", " (foo)", "test.cog"]) 2025 | self.assertFilesSame("test.cog", "test.out") 2026 | 2027 | def test_empty_suffix(self): 2028 | d = { 2029 | "test.cog": """\ 2030 | ;[[[cog cog.outl('a\\nb\\nc') ]]] 2031 | ;[[[end]]] 2032 | """, 2033 | "test.out": """\ 2034 | ;[[[cog cog.outl('a\\nb\\nc') ]]] 2035 | a 2036 | b 2037 | c 2038 | ;[[[end]]] 2039 | """, 2040 | } 2041 | 2042 | make_files(d) 2043 | self.cog.callable_main(["argv0", "-r", "-s", "", "test.cog"]) 2044 | self.assertFilesSame("test.cog", "test.out") 2045 | 2046 | def test_hellish_suffix(self): 2047 | d = { 2048 | "test.cog": """\ 2049 | ;[[[cog cog.outl('a\\n\\nb') ]]] 2050 | """, 2051 | "test.out": """\ 2052 | ;[[[cog cog.outl('a\\n\\nb') ]]] 2053 | a /\\n*+([)]>< 2054 | 2055 | b /\\n*+([)]>< 2056 | """, 2057 | } 2058 | 2059 | make_files(d) 2060 | self.cog.callable_main(["argv0", "-z", "-r", "-s", r" /\n*+([)]><", "test.cog"]) 2061 | self.assertFilesSame("test.cog", "test.out") 2062 | 2063 | def test_prologue(self): 2064 | d = { 2065 | "test.cog": """\ 2066 | Some text. 2067 | //[[[cog cog.outl(str(math.sqrt(2))[:12])]]] 2068 | //[[[end]]] 2069 | epilogue. 2070 | """, 2071 | "test.out": """\ 2072 | Some text. 2073 | //[[[cog cog.outl(str(math.sqrt(2))[:12])]]] 2074 | 1.4142135623 2075 | //[[[end]]] 2076 | epilogue. 2077 | """, 2078 | } 2079 | 2080 | make_files(d) 2081 | self.cog.callable_main(["argv0", "-r", "-p", "import math", "test.cog"]) 2082 | self.assertFilesSame("test.cog", "test.out") 2083 | 2084 | def test_threads(self): 2085 | # Test that the implicitly imported cog module is actually different for 2086 | # different threads. 2087 | numthreads = 20 2088 | 2089 | d = {} 2090 | for i in range(numthreads): 2091 | d[f"f{i}.cog"] = ( 2092 | "x\n" * i 2093 | + "[[[cog\n" 2094 | + f"assert cog.firstLineNum == int(FIRST) == {i + 1}\n" 2095 | + "]]]\n" 2096 | + "[[[end]]]\n" 2097 | ) 2098 | make_files(d) 2099 | 2100 | results = [] 2101 | 2102 | def thread_main(num): 2103 | try: 2104 | ret = Cog().main( 2105 | ["cog.py", "-r", "-D", f"FIRST={num + 1}", f"f{num}.cog"] 2106 | ) 2107 | assert ret == 0 2108 | except Exception as exc: # pragma: no cover (only happens on test failure) 2109 | results.append(exc) 2110 | else: 2111 | results.append(None) 2112 | 2113 | ts = [ 2114 | threading.Thread(target=thread_main, args=(i,)) for i in range(numthreads) 2115 | ] 2116 | for t in ts: 2117 | t.start() 2118 | for t in ts: 2119 | t.join() 2120 | assert results == [None] * numthreads 2121 | 2122 | 2123 | class CheckTests(TestCaseWithTempDir): 2124 | def run_check(self, args, status=0): 2125 | actual_status = self.cog.main(["argv0", "--check"] + args) 2126 | print(self.output.getvalue()) 2127 | self.assertEqual(status, actual_status) 2128 | 2129 | def assert_made_files_unchanged(self, d): 2130 | for name, content in d.items(): 2131 | content = reindent_block(content) 2132 | if os.name == "nt": 2133 | content = content.replace("\n", "\r\n") 2134 | self.assertFileContent(name, content) 2135 | 2136 | def test_check_no_cog(self): 2137 | d = { 2138 | "hello.txt": """\ 2139 | Hello. 2140 | """, 2141 | } 2142 | make_files(d) 2143 | self.run_check(["hello.txt"], status=0) 2144 | self.assertEqual(self.output.getvalue(), "Checking hello.txt\n") 2145 | self.assert_made_files_unchanged(d) 2146 | 2147 | def test_check_good(self): 2148 | d = { 2149 | "unchanged.cog": """\ 2150 | //[[[cog 2151 | cog.outl("hello world") 2152 | //]]] 2153 | hello world 2154 | //[[[end]]] 2155 | """, 2156 | } 2157 | make_files(d) 2158 | self.run_check(["unchanged.cog"], status=0) 2159 | self.assertEqual(self.output.getvalue(), "Checking unchanged.cog\n") 2160 | self.assert_made_files_unchanged(d) 2161 | 2162 | def test_check_bad(self): 2163 | d = { 2164 | "changed.cog": """\ 2165 | //[[[cog 2166 | cog.outl("goodbye world") 2167 | //]]] 2168 | hello world 2169 | //[[[end]]] 2170 | """, 2171 | } 2172 | make_files(d) 2173 | self.run_check(["changed.cog"], status=5) 2174 | self.assertEqual( 2175 | self.output.getvalue(), "Checking changed.cog (changed)\nCheck failed\n" 2176 | ) 2177 | self.assert_made_files_unchanged(d) 2178 | 2179 | def test_check_bad_with_diff(self): 2180 | d = { 2181 | "skittering.cog": """\ 2182 | //[[[cog 2183 | for i in range(5): cog.outl(f"number {i}") 2184 | cog.outl("goodbye world") 2185 | //]]] 2186 | number 0 2187 | number 1 2188 | number 2 2189 | number 3 2190 | number 4 2191 | hello world 2192 | //[[[end]]] 2193 | """, 2194 | } 2195 | make_files(d) 2196 | self.run_check(["--diff", "skittering.cog"], status=5) 2197 | output = """\ 2198 | Checking skittering.cog (changed) 2199 | --- current skittering.cog 2200 | +++ changed skittering.cog 2201 | @@ -7,5 +7,5 @@ 2202 | number 2 2203 | number 3 2204 | number 4 2205 | -hello world 2206 | +goodbye world 2207 | //[[[end]]] 2208 | Check failed 2209 | """ 2210 | self.assertEqual(self.output.getvalue(), reindent_block(output)) 2211 | self.assert_made_files_unchanged(d) 2212 | 2213 | def test_check_mixed(self): 2214 | d = { 2215 | "unchanged.cog": """\ 2216 | //[[[cog 2217 | cog.outl("hello world") 2218 | //]]] 2219 | hello world 2220 | //[[[end]]] 2221 | """, 2222 | "changed.cog": """\ 2223 | //[[[cog 2224 | cog.outl("goodbye world") 2225 | //]]] 2226 | hello world 2227 | //[[[end]]] 2228 | """, 2229 | } 2230 | make_files(d) 2231 | for verbosity, output in [ 2232 | ("0", "Check failed\n"), 2233 | ("1", "Checking changed.cog (changed)\nCheck failed\n"), 2234 | ( 2235 | "2", 2236 | "Checking unchanged.cog\nChecking changed.cog (changed)\nCheck failed\n", 2237 | ), 2238 | ]: 2239 | self.new_cog() 2240 | self.run_check( 2241 | ["--verbosity=%s" % verbosity, "unchanged.cog", "changed.cog"], status=5 2242 | ) 2243 | self.assertEqual(self.output.getvalue(), output) 2244 | self.assert_made_files_unchanged(d) 2245 | 2246 | def test_check_with_good_checksum(self): 2247 | d = { 2248 | "good.txt": """\ 2249 | //[[[cog 2250 | cog.outl("This line was newly") 2251 | cog.outl("generated by cog") 2252 | cog.outl("blah blah.") 2253 | //]]] 2254 | This line was newly 2255 | generated by cog 2256 | blah blah. 2257 | //[[[end]]] (checksum: a8540982e5ad6b95c9e9a184b26f4346) 2258 | """, 2259 | } 2260 | make_files(d) 2261 | # Have to use -c with --check if there are checksums in the file. 2262 | self.run_check(["-c", "good.txt"], status=0) 2263 | self.assertEqual(self.output.getvalue(), "Checking good.txt\n") 2264 | self.assert_made_files_unchanged(d) 2265 | 2266 | def test_check_with_bad_checksum(self): 2267 | d = { 2268 | "bad.txt": """\ 2269 | //[[[cog 2270 | cog.outl("This line was newly") 2271 | cog.outl("generated by cog") 2272 | cog.outl("blah blah.") 2273 | //]]] 2274 | This line was newly 2275 | generated by cog 2276 | blah blah. 2277 | //[[[end]]] (checksum: a9999999e5ad6b95c9e9a184b26f4346) 2278 | """, 2279 | } 2280 | make_files(d) 2281 | # Have to use -c with --check if there are checksums in the file. 2282 | self.run_check(["-c", "bad.txt"], status=1) 2283 | self.assertEqual( 2284 | self.output.getvalue(), 2285 | "Checking bad.txt\nbad.txt(9): Output has been edited! Delete old checksum to unprotect.\n", 2286 | ) 2287 | self.assert_made_files_unchanged(d) 2288 | 2289 | def test_check_with_good_sum(self): 2290 | d = { 2291 | "good.txt": """\ 2292 | //[[[cog 2293 | cog.outl("This line was newly") 2294 | cog.outl("generated by cog") 2295 | cog.outl("blah blah.") 2296 | //]]] 2297 | This line was newly 2298 | generated by cog 2299 | blah blah. 2300 | //[[[end]]] (sum: qFQJguWta5) 2301 | """, 2302 | } 2303 | make_files(d) 2304 | # Have to use -c with --check if there are checksums in the file. 2305 | self.run_check(["-c", "good.txt"], status=0) 2306 | self.assertEqual(self.output.getvalue(), "Checking good.txt\n") 2307 | self.assert_made_files_unchanged(d) 2308 | 2309 | def test_check_with_bad_sum(self): 2310 | d = { 2311 | "bad.txt": """\ 2312 | //[[[cog 2313 | cog.outl("This line was newly") 2314 | cog.outl("generated by cog") 2315 | cog.outl("blah blah.") 2316 | //]]] 2317 | This line was newly 2318 | generated by cog 2319 | blah blah. 2320 | //[[[end]]] (sum: qZmZmeWta5) 2321 | """, 2322 | } 2323 | make_files(d) 2324 | # Have to use -c with --check if there are checksums in the file. 2325 | self.run_check(["-c", "bad.txt"], status=1) 2326 | self.assertEqual( 2327 | self.output.getvalue(), 2328 | "Checking bad.txt\nbad.txt(9): Output has been edited! Delete old checksum to unprotect.\n", 2329 | ) 2330 | self.assert_made_files_unchanged(d) 2331 | 2332 | 2333 | class WritabilityTests(TestCaseWithTempDir): 2334 | d = { 2335 | "test.cog": """\ 2336 | //[[[cog 2337 | for fn in ['DoSomething', 'DoAnotherThing', 'DoLastThing']: 2338 | cog.outl("void %s();" % fn) 2339 | //]]] 2340 | //[[[end]]] 2341 | """, 2342 | "test.out": """\ 2343 | //[[[cog 2344 | for fn in ['DoSomething', 'DoAnotherThing', 'DoLastThing']: 2345 | cog.outl("void %s();" % fn) 2346 | //]]] 2347 | void DoSomething(); 2348 | void DoAnotherThing(); 2349 | void DoLastThing(); 2350 | //[[[end]]] 2351 | """, 2352 | } 2353 | 2354 | if os.name == "nt": 2355 | # for Windows 2356 | cmd_w_args = "attrib -R %s" 2357 | cmd_w_asterisk = "attrib -R *" 2358 | else: 2359 | # for unix-like 2360 | cmd_w_args = "chmod +w %s" 2361 | cmd_w_asterisk = "chmod +w *" 2362 | 2363 | def setUp(self): 2364 | super().setUp() 2365 | make_files(self.d) 2366 | self.testcog = os.path.join(self.tempdir, "test.cog") 2367 | os.chmod(self.testcog, stat.S_IREAD) # Make the file readonly. 2368 | assert not os.access(self.testcog, os.W_OK) 2369 | 2370 | def tearDown(self): 2371 | os.chmod(self.testcog, stat.S_IWRITE) # Make the file writable again. 2372 | super().tearDown() 2373 | 2374 | def test_readonly_no_command(self): 2375 | with self.assertRaisesRegex(CogError, "^Can't overwrite test.cog$"): 2376 | self.cog.callable_main(["argv0", "-r", "test.cog"]) 2377 | assert not os.access(self.testcog, os.W_OK) 2378 | 2379 | def test_readonly_with_command(self): 2380 | self.cog.callable_main(["argv0", "-r", "-w", self.cmd_w_args, "test.cog"]) 2381 | self.assertFilesSame("test.cog", "test.out") 2382 | assert os.access(self.testcog, os.W_OK) 2383 | 2384 | def test_readonly_with_command_with_no_slot(self): 2385 | self.cog.callable_main(["argv0", "-r", "-w", self.cmd_w_asterisk, "test.cog"]) 2386 | self.assertFilesSame("test.cog", "test.out") 2387 | assert os.access(self.testcog, os.W_OK) 2388 | 2389 | def test_readonly_with_ineffectual_command(self): 2390 | with self.assertRaisesRegex(CogError, "^Couldn't make test.cog writable$"): 2391 | self.cog.callable_main(["argv0", "-r", "-w", "echo %s", "test.cog"]) 2392 | assert not os.access(self.testcog, os.W_OK) 2393 | 2394 | 2395 | class ChecksumTests(TestCaseWithTempDir): 2396 | def test_create_checksum_output(self): 2397 | d = { 2398 | "cog1.txt": """\ 2399 | //[[[cog 2400 | cog.outl("This line was generated.") 2401 | //]]] 2402 | This line was generated. 2403 | //[[[end]]] what 2404 | This line was not. 2405 | """, 2406 | "cog1.out": """\ 2407 | //[[[cog 2408 | cog.outl("This line was generated.") 2409 | //]]] 2410 | This line was generated. 2411 | //[[[end]]] (sum: itsT+1m5lq) what 2412 | This line was not. 2413 | """, 2414 | } 2415 | 2416 | make_files(d) 2417 | self.cog.callable_main(["argv0", "-r", "-c", "cog1.txt"]) 2418 | self.assertFilesSame("cog1.txt", "cog1.out") 2419 | 2420 | def test_check_checksum_output(self): 2421 | d = { 2422 | "cog1.txt": """\ 2423 | //[[[cog 2424 | cog.outl("This line was newly") 2425 | cog.outl("generated by cog") 2426 | cog.outl("blah blah.") 2427 | //]]] 2428 | This line was generated. 2429 | //[[[end]]] (sum: itsT+1m5lq) end 2430 | """, 2431 | "cog1.out": """\ 2432 | //[[[cog 2433 | cog.outl("This line was newly") 2434 | cog.outl("generated by cog") 2435 | cog.outl("blah blah.") 2436 | //]]] 2437 | This line was newly 2438 | generated by cog 2439 | blah blah. 2440 | //[[[end]]] (sum: qFQJguWta5) end 2441 | """, 2442 | } 2443 | 2444 | make_files(d) 2445 | self.cog.callable_main(["argv0", "-r", "-c", "cog1.txt"]) 2446 | self.assertFilesSame("cog1.txt", "cog1.out") 2447 | 2448 | def test_check_old_checksum_format(self): 2449 | # Test that old checksum format can still be read 2450 | d = { 2451 | "cog1.txt": """\ 2452 | //[[[cog 2453 | cog.outl("This line was newly") 2454 | cog.outl("generated by cog") 2455 | cog.outl("blah blah.") 2456 | //]]] 2457 | This line was generated. 2458 | //[[[end]]] (checksum: 8adb13fb59b996a1c7f0065ea9f3d893) end 2459 | """, 2460 | "cog1.out": """\ 2461 | //[[[cog 2462 | cog.outl("This line was newly") 2463 | cog.outl("generated by cog") 2464 | cog.outl("blah blah.") 2465 | //]]] 2466 | This line was newly 2467 | generated by cog 2468 | blah blah. 2469 | //[[[end]]] (sum: qFQJguWta5) end 2470 | """, 2471 | } 2472 | 2473 | make_files(d) 2474 | self.cog.callable_main(["argv0", "-r", "-c", "cog1.txt"]) 2475 | self.assertFilesSame("cog1.txt", "cog1.out") 2476 | 2477 | def test_remove_checksum_output(self): 2478 | d = { 2479 | "cog1.txt": """\ 2480 | //[[[cog 2481 | cog.outl("This line was newly") 2482 | cog.outl("generated by cog") 2483 | cog.outl("blah blah.") 2484 | //]]] 2485 | This line was generated. 2486 | //[[[end]]] (sum: itsT+1m5lq) fooey 2487 | """, 2488 | "cog1.out": """\ 2489 | //[[[cog 2490 | cog.outl("This line was newly") 2491 | cog.outl("generated by cog") 2492 | cog.outl("blah blah.") 2493 | //]]] 2494 | This line was newly 2495 | generated by cog 2496 | blah blah. 2497 | //[[[end]]] fooey 2498 | """, 2499 | } 2500 | 2501 | make_files(d) 2502 | self.cog.callable_main(["argv0", "-r", "cog1.txt"]) 2503 | self.assertFilesSame("cog1.txt", "cog1.out") 2504 | 2505 | def test_tampered_checksum_output(self): 2506 | d = { 2507 | "cog1.txt": """\ 2508 | //[[[cog 2509 | cog.outl("This line was newly") 2510 | cog.outl("generated by cog") 2511 | cog.outl("blah blah.") 2512 | //]]] 2513 | Xhis line was newly 2514 | generated by cog 2515 | blah blah. 2516 | //[[[end]]] (sum: qFQJguWta5) 2517 | """, 2518 | "cog2.txt": """\ 2519 | //[[[cog 2520 | cog.outl("This line was newly") 2521 | cog.outl("generated by cog") 2522 | cog.outl("blah blah.") 2523 | //]]] 2524 | This line was newly 2525 | generated by cog 2526 | blah blah! 2527 | //[[[end]]] (sum: qFQJguWta5) 2528 | """, 2529 | "cog3.txt": """\ 2530 | //[[[cog 2531 | cog.outl("This line was newly") 2532 | cog.outl("generated by cog") 2533 | cog.outl("blah blah.") 2534 | //]]] 2535 | 2536 | This line was newly 2537 | generated by cog 2538 | blah blah. 2539 | //[[[end]]] (sum: qFQJguWta5) 2540 | """, 2541 | "cog4.txt": """\ 2542 | //[[[cog 2543 | cog.outl("This line was newly") 2544 | cog.outl("generated by cog") 2545 | cog.outl("blah blah.") 2546 | //]]] 2547 | This line was newly 2548 | generated by cog 2549 | blah blah.. 2550 | //[[[end]]] (sum: qFQJguWta5) 2551 | """, 2552 | "cog5.txt": """\ 2553 | //[[[cog 2554 | cog.outl("This line was newly") 2555 | cog.outl("generated by cog") 2556 | cog.outl("blah blah.") 2557 | //]]] 2558 | This line was newly 2559 | generated by cog 2560 | blah blah. 2561 | extra 2562 | //[[[end]]] (sum: qFQJguWta5) 2563 | """, 2564 | "cog6.txt": """\ 2565 | //[[[cog 2566 | cog.outl("This line was newly") 2567 | cog.outl("generated by cog") 2568 | cog.outl("blah blah.") 2569 | //]]] 2570 | //[[[end]]] (sum: qFQJguWta5) 2571 | """, 2572 | } 2573 | 2574 | make_files(d) 2575 | with self.assertRaisesRegex( 2576 | CogError, 2577 | r"^cog1.txt\(9\): Output has been edited! Delete old checksum to unprotect.$", 2578 | ): 2579 | self.cog.callable_main(["argv0", "-c", "cog1.txt"]) 2580 | with self.assertRaisesRegex( 2581 | CogError, 2582 | r"^cog2.txt\(9\): Output has been edited! Delete old checksum to unprotect.$", 2583 | ): 2584 | self.cog.callable_main(["argv0", "-c", "cog2.txt"]) 2585 | with self.assertRaisesRegex( 2586 | CogError, 2587 | r"^cog3.txt\(10\): Output has been edited! Delete old checksum to unprotect.$", 2588 | ): 2589 | self.cog.callable_main(["argv0", "-c", "cog3.txt"]) 2590 | with self.assertRaisesRegex( 2591 | CogError, 2592 | r"^cog4.txt\(9\): Output has been edited! Delete old checksum to unprotect.$", 2593 | ): 2594 | self.cog.callable_main(["argv0", "-c", "cog4.txt"]) 2595 | with self.assertRaisesRegex( 2596 | CogError, 2597 | r"^cog5.txt\(10\): Output has been edited! Delete old checksum to unprotect.$", 2598 | ): 2599 | self.cog.callable_main(["argv0", "-c", "cog5.txt"]) 2600 | with self.assertRaisesRegex( 2601 | CogError, 2602 | r"^cog6.txt\(6\): Output has been edited! Delete old checksum to unprotect.$", 2603 | ): 2604 | self.cog.callable_main(["argv0", "-c", "cog6.txt"]) 2605 | 2606 | def test_argv_isnt_modified(self): 2607 | argv = ["argv0", "-v"] 2608 | orig_argv = argv[:] 2609 | self.cog.callable_main(argv) 2610 | self.assertEqual(argv, orig_argv) 2611 | 2612 | 2613 | class CustomMarkerTests(TestCaseWithTempDir): 2614 | def test_customer_markers(self): 2615 | d = { 2616 | "test.cog": """\ 2617 | //{{ 2618 | cog.outl("void %s();" % "MyFunction") 2619 | //}} 2620 | //{{end}} 2621 | """, 2622 | "test.out": """\ 2623 | //{{ 2624 | cog.outl("void %s();" % "MyFunction") 2625 | //}} 2626 | void MyFunction(); 2627 | //{{end}} 2628 | """, 2629 | } 2630 | 2631 | make_files(d) 2632 | self.cog.callable_main(["argv0", "-r", "--markers={{ }} {{end}}", "test.cog"]) 2633 | self.assertFilesSame("test.cog", "test.out") 2634 | 2635 | def test_truly_wacky_markers(self): 2636 | # Make sure the markers are properly re-escaped. 2637 | d = { 2638 | "test.cog": """\ 2639 | //**( 2640 | cog.outl("void %s();" % "MyFunction") 2641 | //**) 2642 | //**(end)** 2643 | """, 2644 | "test.out": """\ 2645 | //**( 2646 | cog.outl("void %s();" % "MyFunction") 2647 | //**) 2648 | void MyFunction(); 2649 | //**(end)** 2650 | """, 2651 | } 2652 | 2653 | make_files(d) 2654 | self.cog.callable_main( 2655 | ["argv0", "-r", "--markers=**( **) **(end)**", "test.cog"] 2656 | ) 2657 | self.assertFilesSame("test.cog", "test.out") 2658 | 2659 | def test_change_just_one_marker(self): 2660 | d = { 2661 | "test.cog": """\ 2662 | //**( 2663 | cog.outl("void %s();" % "MyFunction") 2664 | //]]] 2665 | //[[[end]]] 2666 | """, 2667 | "test.out": """\ 2668 | //**( 2669 | cog.outl("void %s();" % "MyFunction") 2670 | //]]] 2671 | void MyFunction(); 2672 | //[[[end]]] 2673 | """, 2674 | } 2675 | 2676 | make_files(d) 2677 | self.cog.callable_main( 2678 | ["argv0", "-r", "--markers=**( ]]] [[[end]]]", "test.cog"] 2679 | ) 2680 | self.assertFilesSame("test.cog", "test.out") 2681 | 2682 | 2683 | class BlakeTests(TestCaseWithTempDir): 2684 | # Blake Winton's contributions. 2685 | def test_delete_code(self): 2686 | # -o sets the output file. 2687 | d = { 2688 | "test.cog": """\ 2689 | // This is my C++ file. 2690 | //[[[cog 2691 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 2692 | for fn in fnames: 2693 | cog.outl("void %s();" % fn) 2694 | //]]] 2695 | Some Sample Code Here 2696 | //[[[end]]]Data Data 2697 | And Some More 2698 | """, 2699 | "test.out": """\ 2700 | // This is my C++ file. 2701 | void DoSomething(); 2702 | void DoAnotherThing(); 2703 | void DoLastThing(); 2704 | And Some More 2705 | """, 2706 | } 2707 | 2708 | make_files(d) 2709 | self.cog.callable_main(["argv0", "-d", "-o", "test.cogged", "test.cog"]) 2710 | self.assertFilesSame("test.cogged", "test.out") 2711 | 2712 | def test_delete_code_with_dash_r_fails(self): 2713 | d = { 2714 | "test.cog": """\ 2715 | // This is my C++ file. 2716 | """ 2717 | } 2718 | 2719 | make_files(d) 2720 | with self.assertRaisesRegex( 2721 | CogUsageError, 2722 | r"^Can't use -d with -r \(or you would delete all your source!\)$", 2723 | ): 2724 | self.cog.callable_main(["argv0", "-r", "-d", "test.cog"]) 2725 | 2726 | def test_setting_globals(self): 2727 | # Blake Winton contributed a way to set the globals that will be used in 2728 | # processFile(). 2729 | d = { 2730 | "test.cog": """\ 2731 | // This is my C++ file. 2732 | //[[[cog 2733 | for fn in fnames: 2734 | cog.outl("void %s();" % fn) 2735 | //]]] 2736 | Some Sample Code Here 2737 | //[[[end]]]""", 2738 | "test.out": """\ 2739 | // This is my C++ file. 2740 | void DoBlake(); 2741 | void DoWinton(); 2742 | void DoContribution(); 2743 | """, 2744 | } 2745 | 2746 | make_files(d) 2747 | globals = {} 2748 | globals["fnames"] = ["DoBlake", "DoWinton", "DoContribution"] 2749 | self.cog.options.delete_code = True 2750 | self.cog.process_file("test.cog", "test.cogged", globals=globals) 2751 | self.assertFilesSame("test.cogged", "test.out") 2752 | 2753 | 2754 | class ErrorCallTests(TestCaseWithTempDir): 2755 | def test_error_call_has_no_traceback(self): 2756 | # Test that cog.error() doesn't show a traceback. 2757 | d = { 2758 | "error.cog": """\ 2759 | //[[[cog 2760 | cog.error("Something Bad!") 2761 | //]]] 2762 | //[[[end]]] 2763 | """, 2764 | } 2765 | 2766 | make_files(d) 2767 | self.cog.main(["argv0", "-r", "error.cog"]) 2768 | output = self.output.getvalue() 2769 | self.assertEqual(output, "Cogging error.cog\nError: Something Bad!\n") 2770 | 2771 | def test_real_error_has_traceback(self): 2772 | # Test that a genuine error does show a traceback. 2773 | d = { 2774 | "error.cog": """\ 2775 | //[[[cog 2776 | raise RuntimeError("Hey!") 2777 | //]]] 2778 | //[[[end]]] 2779 | """, 2780 | } 2781 | 2782 | make_files(d) 2783 | self.cog.main(["argv0", "-r", "error.cog"]) 2784 | output = self.output.getvalue() 2785 | msg = "Actual output:\n" + output 2786 | self.assertTrue( 2787 | output.startswith("Cogging error.cog\nTraceback (most recent"), msg 2788 | ) 2789 | self.assertIn("RuntimeError: Hey!", output) 2790 | 2791 | 2792 | class HashHandlerTests(TestCase): 2793 | """Test cases for HashHandler functionality.""" 2794 | 2795 | def setUp(self): 2796 | self.handler = HashHandler("[[[end]]]") 2797 | 2798 | def test_validate_hash_with_base64_mismatch(self): 2799 | # Test the base64 validation branch with a mismatch 2800 | line = "//[[[end]]] (sum: wronghas12)" # 10 chars to match regex 2801 | expected_hash = "a8540982e5ad6b95c9e9a184b26f4346" 2802 | 2803 | with self.assertRaises(ValueError) as cm: 2804 | self.handler.validate_hash(line, expected_hash) 2805 | self.assertEqual( 2806 | str(cm.exception), 2807 | "Output has been edited! Delete old checksum to unprotect.", 2808 | ) 2809 | 2810 | def test_validate_hash_with_base64_match(self): 2811 | # Test the base64 validation branch with a match 2812 | line = "//[[[end]]] (sum: qFQJguWta5)" 2813 | expected_hash = "a8540982e5ad6b95c9e9a184b26f4346" 2814 | 2815 | # Should not raise an exception 2816 | result = self.handler.validate_hash(line, expected_hash) 2817 | self.assertTrue(result) 2818 | 2819 | 2820 | # Things not yet tested: 2821 | # - A bad -w command (currently fails silently). 2822 | -------------------------------------------------------------------------------- /cogapp/test_makefiles.py: -------------------------------------------------------------------------------- 1 | """Test the cogapp.makefiles modules""" 2 | 3 | import shutil 4 | import os 5 | import random 6 | import tempfile 7 | from unittest import TestCase 8 | 9 | from . import makefiles 10 | 11 | 12 | class SimpleTests(TestCase): 13 | def setUp(self): 14 | # Create a temporary directory. 15 | my_dir = "testmakefiles_tempdir_" + str(random.random())[2:] 16 | self.tempdir = os.path.join(tempfile.gettempdir(), my_dir) 17 | os.mkdir(self.tempdir) 18 | 19 | def tearDown(self): 20 | # Get rid of the temporary directory. 21 | shutil.rmtree(self.tempdir) 22 | 23 | def exists(self, dname, fname): 24 | return os.path.exists(os.path.join(dname, fname)) 25 | 26 | def check_files_exist(self, d, dname): 27 | for fname in d.keys(): 28 | assert self.exists(dname, fname) 29 | if isinstance(d[fname], dict): 30 | self.check_files_exist(d[fname], os.path.join(dname, fname)) 31 | 32 | def check_files_dont_exist(self, d, dname): 33 | for fname in d.keys(): 34 | assert not self.exists(dname, fname) 35 | 36 | def test_one_file(self): 37 | fname = "foo.txt" 38 | notfname = "not_here.txt" 39 | d = {fname: "howdy"} 40 | assert not self.exists(self.tempdir, fname) 41 | assert not self.exists(self.tempdir, notfname) 42 | 43 | makefiles.make_files(d, self.tempdir) 44 | assert self.exists(self.tempdir, fname) 45 | assert not self.exists(self.tempdir, notfname) 46 | 47 | makefiles.remove_files(d, self.tempdir) 48 | assert not self.exists(self.tempdir, fname) 49 | assert not self.exists(self.tempdir, notfname) 50 | 51 | def test_many_files(self): 52 | d = { 53 | "top1.txt": "howdy", 54 | "top2.txt": "hello", 55 | "sub": { 56 | "sub1.txt": "inside", 57 | "sub2.txt": "inside2", 58 | }, 59 | } 60 | 61 | self.check_files_dont_exist(d, self.tempdir) 62 | makefiles.make_files(d, self.tempdir) 63 | self.check_files_exist(d, self.tempdir) 64 | makefiles.remove_files(d, self.tempdir) 65 | self.check_files_dont_exist(d, self.tempdir) 66 | 67 | def test_overlapping(self): 68 | d1 = { 69 | "top1.txt": "howdy", 70 | "sub": { 71 | "sub1.txt": "inside", 72 | }, 73 | } 74 | 75 | d2 = { 76 | "top2.txt": "hello", 77 | "sub": { 78 | "sub2.txt": "inside2", 79 | }, 80 | } 81 | 82 | self.check_files_dont_exist(d1, self.tempdir) 83 | self.check_files_dont_exist(d2, self.tempdir) 84 | makefiles.make_files(d1, self.tempdir) 85 | makefiles.make_files(d2, self.tempdir) 86 | self.check_files_exist(d1, self.tempdir) 87 | self.check_files_exist(d2, self.tempdir) 88 | makefiles.remove_files(d1, self.tempdir) 89 | makefiles.remove_files(d2, self.tempdir) 90 | self.check_files_dont_exist(d1, self.tempdir) 91 | self.check_files_dont_exist(d2, self.tempdir) 92 | 93 | def test_contents(self): 94 | fname = "bar.txt" 95 | cont0 = "I am bar.txt" 96 | d = {fname: cont0} 97 | makefiles.make_files(d, self.tempdir) 98 | with open(os.path.join(self.tempdir, fname)) as fcont1: 99 | assert fcont1.read() == cont0 100 | 101 | def test_dedent(self): 102 | fname = "dedent.txt" 103 | d = { 104 | fname: """\ 105 | This is dedent.txt 106 | \tTabbed in. 107 | spaced in. 108 | OK. 109 | """, 110 | } 111 | makefiles.make_files(d, self.tempdir) 112 | with open(os.path.join(self.tempdir, fname)) as fcont: 113 | assert ( 114 | fcont.read() == "This is dedent.txt\n\tTabbed in.\n spaced in.\nOK.\n" 115 | ) 116 | -------------------------------------------------------------------------------- /cogapp/test_whiteutils.py: -------------------------------------------------------------------------------- 1 | """Test the cogapp.whiteutils module.""" 2 | 3 | from unittest import TestCase 4 | 5 | from .whiteutils import common_prefix, reindent_block, white_prefix 6 | 7 | 8 | class WhitePrefixTests(TestCase): 9 | """Test cases for cogapp.whiteutils.""" 10 | 11 | def test_single_line(self): 12 | self.assertEqual(white_prefix([""]), "") 13 | self.assertEqual(white_prefix([" "]), "") 14 | self.assertEqual(white_prefix(["x"]), "") 15 | self.assertEqual(white_prefix([" x"]), " ") 16 | self.assertEqual(white_prefix(["\tx"]), "\t") 17 | self.assertEqual(white_prefix([" x"]), " ") 18 | self.assertEqual(white_prefix([" \t \tx "]), " \t \t") 19 | 20 | def test_multi_line(self): 21 | self.assertEqual(white_prefix([" x", " x", " x"]), " ") 22 | self.assertEqual(white_prefix([" y", " y", " y"]), " ") 23 | self.assertEqual(white_prefix([" y", " y", " y"]), " ") 24 | 25 | def test_blank_lines_are_ignored(self): 26 | self.assertEqual(white_prefix([" x", " x", "", " x"]), " ") 27 | self.assertEqual(white_prefix(["", " x", " x", " x"]), " ") 28 | self.assertEqual(white_prefix([" x", " x", " x", ""]), " ") 29 | self.assertEqual(white_prefix([" x", " x", " ", " x"]), " ") 30 | 31 | def test_tab_characters(self): 32 | self.assertEqual(white_prefix(["\timport sys", "", "\tprint sys.argv"]), "\t") 33 | 34 | def test_decreasing_lengths(self): 35 | self.assertEqual(white_prefix([" x", " x", " x"]), " ") 36 | self.assertEqual(white_prefix([" x", " x", " x"]), " ") 37 | 38 | 39 | class ReindentBlockTests(TestCase): 40 | """Test cases for cogapp.reindentBlock.""" 41 | 42 | def test_non_term_line(self): 43 | self.assertEqual(reindent_block(""), "") 44 | self.assertEqual(reindent_block("x"), "x") 45 | self.assertEqual(reindent_block(" x"), "x") 46 | self.assertEqual(reindent_block(" x"), "x") 47 | self.assertEqual(reindent_block("\tx"), "x") 48 | self.assertEqual(reindent_block("x", " "), " x") 49 | self.assertEqual(reindent_block("x", "\t"), "\tx") 50 | self.assertEqual(reindent_block(" x", " "), " x") 51 | self.assertEqual(reindent_block(" x", "\t"), "\tx") 52 | self.assertEqual(reindent_block(" x", " "), " x") 53 | 54 | def test_single_line(self): 55 | self.assertEqual(reindent_block("\n"), "\n") 56 | self.assertEqual(reindent_block("x\n"), "x\n") 57 | self.assertEqual(reindent_block(" x\n"), "x\n") 58 | self.assertEqual(reindent_block(" x\n"), "x\n") 59 | self.assertEqual(reindent_block("\tx\n"), "x\n") 60 | self.assertEqual(reindent_block("x\n", " "), " x\n") 61 | self.assertEqual(reindent_block("x\n", "\t"), "\tx\n") 62 | self.assertEqual(reindent_block(" x\n", " "), " x\n") 63 | self.assertEqual(reindent_block(" x\n", "\t"), "\tx\n") 64 | self.assertEqual(reindent_block(" x\n", " "), " x\n") 65 | 66 | def test_real_block(self): 67 | self.assertEqual( 68 | reindent_block("\timport sys\n\n\tprint sys.argv\n"), 69 | "import sys\n\nprint sys.argv\n", 70 | ) 71 | 72 | 73 | class CommonPrefixTests(TestCase): 74 | """Test cases for cogapp.commonPrefix.""" 75 | 76 | def test_degenerate_cases(self): 77 | self.assertEqual(common_prefix([]), "") 78 | self.assertEqual(common_prefix([""]), "") 79 | self.assertEqual(common_prefix(["", "", "", "", ""]), "") 80 | self.assertEqual(common_prefix(["cat in the hat"]), "cat in the hat") 81 | 82 | def test_no_common_prefix(self): 83 | self.assertEqual(common_prefix(["a", "b"]), "") 84 | self.assertEqual(common_prefix(["a", "b", "c", "d", "e", "f"]), "") 85 | self.assertEqual(common_prefix(["a", "a", "a", "a", "a", "x"]), "") 86 | 87 | def test_usual_cases(self): 88 | self.assertEqual(common_prefix(["ab", "ac"]), "a") 89 | self.assertEqual(common_prefix(["aab", "aac"]), "aa") 90 | self.assertEqual(common_prefix(["aab", "aab", "aab", "aac"]), "aa") 91 | 92 | def test_blank_line(self): 93 | self.assertEqual(common_prefix(["abc", "abx", "", "aby"]), "") 94 | 95 | def test_decreasing_lengths(self): 96 | self.assertEqual(common_prefix(["abcd", "abc", "ab"]), "ab") 97 | -------------------------------------------------------------------------------- /cogapp/utils.py: -------------------------------------------------------------------------------- 1 | """Utilities for cog.""" 2 | 3 | import contextlib 4 | import functools 5 | import hashlib 6 | import os 7 | import sys 8 | 9 | 10 | # Support FIPS mode. We don't use MD5 for security. 11 | md5 = functools.partial(hashlib.md5, usedforsecurity=False) 12 | 13 | 14 | class Redirectable: 15 | """An object with its own stdout and stderr files.""" 16 | 17 | def __init__(self): 18 | self.stdout = sys.stdout 19 | self.stderr = sys.stderr 20 | 21 | def set_output(self, stdout=None, stderr=None): 22 | """Assign new files for standard out and/or standard error.""" 23 | if stdout: 24 | self.stdout = stdout 25 | if stderr: 26 | self.stderr = stderr 27 | 28 | def prout(self, s, end="\n"): 29 | print(s, file=self.stdout, end=end) 30 | 31 | def prerr(self, s, end="\n"): 32 | print(s, file=self.stderr, end=end) 33 | 34 | 35 | class NumberedFileReader: 36 | """A decorator for files that counts the readline()'s called.""" 37 | 38 | def __init__(self, f): 39 | self.f = f 40 | self.n = 0 41 | 42 | def readline(self): 43 | line = self.f.readline() 44 | if line: 45 | self.n += 1 46 | return line 47 | 48 | def linenumber(self): 49 | return self.n 50 | 51 | 52 | @contextlib.contextmanager 53 | def change_dir(new_dir): 54 | """Change directory, and then change back. 55 | 56 | Use as a context manager, it will return to the original 57 | directory at the end of the block. 58 | 59 | """ 60 | old_dir = os.getcwd() 61 | os.chdir(str(new_dir)) 62 | try: 63 | yield 64 | finally: 65 | os.chdir(old_dir) 66 | -------------------------------------------------------------------------------- /cogapp/whiteutils.py: -------------------------------------------------------------------------------- 1 | """Indentation utilities for Cog.""" 2 | 3 | import re 4 | 5 | 6 | def white_prefix(strings): 7 | """Find the whitespace prefix common to non-blank lines in `strings`.""" 8 | # Remove all blank lines from the list 9 | strings = [s for s in strings if s.strip() != ""] 10 | 11 | if not strings: 12 | return "" 13 | 14 | # Find initial whitespace chunk in the first line. 15 | # This is the best prefix we can hope for. 16 | pat = r"\s*" 17 | if isinstance(strings[0], bytes): 18 | pat = pat.encode("utf-8") 19 | prefix = re.match(pat, strings[0]).group(0) 20 | 21 | # Loop over the other strings, keeping only as much of 22 | # the prefix as matches each string. 23 | for s in strings: 24 | for i in range(len(prefix)): 25 | if prefix[i] != s[i]: 26 | prefix = prefix[:i] 27 | break 28 | return prefix 29 | 30 | 31 | def reindent_block(lines, new_indent=""): 32 | """Re-indent a block of text. 33 | 34 | Take a block of text as a string or list of lines. 35 | Remove any common whitespace indentation. 36 | Re-indent using `newIndent`, and return it as a single string. 37 | 38 | """ 39 | sep, nothing = "\n", "" 40 | if isinstance(lines, bytes): 41 | sep, nothing = b"\n", b"" 42 | if isinstance(lines, (bytes, str)): 43 | lines = lines.split(sep) 44 | old_indent = white_prefix(lines) 45 | out_lines = [] 46 | for line in lines: 47 | if old_indent: 48 | line = line.replace(old_indent, nothing, 1) 49 | if line and new_indent: 50 | line = new_indent + line 51 | out_lines.append(line) 52 | return sep.join(out_lines) 53 | 54 | 55 | def common_prefix(strings): 56 | """Find the longest string that is a prefix of all the strings.""" 57 | if not strings: 58 | return "" 59 | prefix = strings[0] 60 | for s in strings: 61 | if len(s) < len(prefix): 62 | prefix = prefix[: len(s)] 63 | if not prefix: 64 | return "" 65 | for i in range(len(prefix)): 66 | if prefix[i] != s[i]: 67 | prefix = prefix[:i] 68 | break 69 | return prefix 70 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | .. _changes: 2 | 3 | .. include:: ../CHANGELOG.rst 4 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | project = "cog" 10 | copyright = "2004–2025, Ned Batchelder" 11 | author = "Ned Batchelder" 12 | release = "3.5.0" 13 | 14 | # -- General configuration --------------------------------------------------- 15 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 16 | 17 | extensions = [] 18 | 19 | templates_path = ["_templates"] 20 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 21 | 22 | language = "en" 23 | 24 | # -- Options for HTML output ------------------------------------------------- 25 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 26 | 27 | html_theme = "alabaster" 28 | html_static_path = ["_static"] 29 | -------------------------------------------------------------------------------- /docs/design.rst: -------------------------------------------------------------------------------- 1 | Design 2 | ====== 3 | 4 | Cog is designed to be easy to run. It writes its results back into the 5 | original file while retaining the code it executed. This means cog can be run 6 | any number of times on the same file. Rather than have a source generator 7 | file, and a separate output file, typically cog is run with one file serving as 8 | both generator and output. 9 | 10 | Because the marker lines accommodate any language syntax, the markers can hide 11 | the cog Python code from the source file. This means cog files can be checked 12 | into source control without worrying about keeping the source files separate 13 | from the output files, without modifying build procedures, and so on. 14 | 15 | I experimented with using a templating engine for generating code, and found 16 | myself constantly struggling with white space in the generated output, and 17 | mentally converting from the Python code I could imagine, into its templating 18 | equivalent. The advantages of a templating system (that most of the code could 19 | be entered literally) were lost as the code generation tasks became more 20 | complex, and the generation process needed more logic. 21 | 22 | Cog lets you use the full power of Python for text generation, without a 23 | templating system dumbing down your tools for you. 24 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | === 2 | Cog 3 | === 4 | 5 | .. 6 | 7 | 8 | Created. 9 | Version 1.1. 10 | Minor edits for clarity. 11 | Updated to cog 1.11, added a See Also section, and fixed a sample. 12 | Updated to cog 1.12. 13 | Updated to cog 1.2. 14 | Updated to cog 1.3. 15 | Updated to cog 1.4. 16 | Added links to other Cog implementations. 17 | Added links to 2.0 beta 2. 18 | Updating for 2.0. 19 | Added PCG. 20 | Added an explicit mention of the license: MIT. 21 | Added links to 3rd-party packages. 22 | Clarified -D value types, and fixed a 3rd-party link. 23 | Tried to explain better about indentation, and fixed an incorrect parameter name. 24 | Added -U switch from Alexander Belchenko. 25 | Fixed the russian pointer to be to a current document. 26 | Removed handyxml, files are now at pypi. 27 | Python 3 is supported! 28 | Polish up Cog 2.3 29 | Version 2.4 30 | Version 3.0.0 31 | Version 3.2.0 32 | Version 3.3.0 33 | 34 | 35 | Cog is a file generation tool. It lets you use pieces of Python code 36 | as generators in your source files to generate whatever text you need. 37 | 38 | This page describes version 3.4.1, released March 7, 2024. 39 | 40 | 41 | What does it do? 42 | ================ 43 | 44 | Cog transforms files in a very simple way: it finds chunks of Python code 45 | embedded in them, executes the Python code, and inserts its output back into 46 | the original file. The file can contain whatever text you like around the 47 | Python code. It will usually be source code. 48 | 49 | For example, if you run this file through cog: 50 | 51 | .. code-block:: cpp 52 | 53 | // This is my C++ file. 54 | ... 55 | /*[[[cog 56 | import cog 57 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 58 | for fn in fnames: 59 | cog.outl("void %s();" % fn) 60 | ]]]*/ 61 | //[[[end]]] 62 | ... 63 | 64 | it will come out like this: 65 | 66 | .. code-block:: cpp 67 | 68 | // This is my C++ file. 69 | ... 70 | /*[[[cog 71 | import cog 72 | fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing'] 73 | for fn in fnames: 74 | cog.outl("void %s();" % fn) 75 | ]]]*/ 76 | void DoSomething(); 77 | void DoAnotherThing(); 78 | void DoLastThing(); 79 | //[[[end]]] 80 | ... 81 | 82 | Lines with triple square brackets are marker lines. The lines between 83 | ``[[[cog`` and ``]]]`` are the generator Python code. The lines between 84 | ``]]]`` and ``[[[end]]]`` are the output from the generator. 85 | 86 | Output is written with `cog.outl()`, or if you use the ``-P`` option, 87 | normal `print()` calls. 88 | 89 | When cog runs, it discards the last generated Python output, executes the 90 | generator Python code, and writes its generated output into the file. All text 91 | lines outside of the special markers are passed through unchanged. 92 | 93 | The cog marker lines can contain any text in addition to the triple square 94 | bracket tokens. This makes it possible to hide the generator Python code from 95 | the source file. In the sample above, the entire chunk of Python code is a C++ 96 | comment, so the Python code can be left in place while the file is treated as 97 | C++ code. 98 | 99 | 100 | Installation 101 | ============ 102 | 103 | Cog requires Python 3.7 or higher. 104 | 105 | Cog is installed in the usual way, except the installation name is "cogapp", 106 | not "cog": 107 | 108 | .. code-block:: bash 109 | 110 | $ python3 -m pip install cogapp 111 | 112 | You should now have a "cog" command you can run. 113 | 114 | See the :ref:`changelog ` for the history of changes. 115 | 116 | Cog is distributed under the `MIT license`_. Use it to spread goodness through 117 | the world. 118 | 119 | .. _MIT License: http://www.opensource.org/licenses/mit-license.php 120 | 121 | 122 | More 123 | ==== 124 | 125 | .. toctree:: 126 | :maxdepth: 1 127 | 128 | changes 129 | design 130 | source 131 | module 132 | running 133 | -------------------------------------------------------------------------------- /docs/module.rst: -------------------------------------------------------------------------------- 1 | The cog module 2 | ============== 3 | 4 | A synthetic module called ``cog`` provides functions you can call to produce 5 | output into your file. You don't need to use these functions: with the ``-P`` 6 | command-line option, your program's stdout writes to the output file, so 7 | `print()` will be enough. 8 | 9 | The module contents are: 10 | 11 | **cog.out** `(sOut='' [, dedent=False][, trimblanklines=False])` 12 | Writes text to the output. `sOut` is the string to write to the output. 13 | If `dedent` is True, then common initial white space is removed from the 14 | lines in `sOut` before adding them to the output. If `trimblanklines` is 15 | True, then an initial and trailing blank line are removed from `sOut` 16 | before adding them to the output. Together, these option arguments make it 17 | easier to use multi-line strings, and they only are useful for multi-line 18 | strings:: 19 | 20 | cog.out(""" 21 | These are lines I 22 | want to write into my source file. 23 | """, dedent=True, trimblanklines=True) 24 | 25 | **cog.outl** 26 | Same as **cog.out**, but adds a trailing newline. 27 | 28 | **cog.msg** `(msg)` 29 | Prints `msg` to stdout with a "Message: " prefix. 30 | 31 | **cog.error** `(msg)` 32 | Raises an exception with `msg` as the text. No traceback is included, so 33 | that non-Python programmers using your code generators won't be scared. 34 | 35 | **cog.inFile** 36 | An attribute, the path of the input file. 37 | 38 | **cog.outFile** 39 | An attribute, the path of the output file. 40 | 41 | **cog.firstLineNum** 42 | An attribute, the line number of the first line of Python code in the 43 | generator. This can be used to distinguish between two generators in the 44 | same input file, if needed. 45 | 46 | **cog.previous** 47 | An attribute, the text output of the previous run of this generator. This 48 | can be used for whatever purpose you like, including outputting again with 49 | **cog.out**. 50 | -------------------------------------------------------------------------------- /docs/running.rst: -------------------------------------------------------------------------------- 1 | Running cog 2 | =========== 3 | 4 | Cog is a command-line utility which takes arguments in standard form. 5 | 6 | .. {{{cog 7 | # Re-run this with `make cogdoc` 8 | # Here we use unconventional markers so the docs can use [[[ without 9 | # getting tangled up in the cog processing. 10 | 11 | import io 12 | import textwrap 13 | from cogapp import Cog 14 | 15 | print("\n.. code-block:: text\n") 16 | outf = io.StringIO() 17 | print("$ cog -h", file=outf) 18 | cog = Cog() 19 | cog.set_output(stdout=outf, stderr=outf) 20 | cog.main(["cog", "-h"]) 21 | print(textwrap.indent(outf.getvalue(), " ")) 22 | .. }}} 23 | 24 | .. code-block:: text 25 | 26 | $ cog -h 27 | cog - generate content with inlined Python code. 28 | 29 | cog [OPTIONS] [INFILE | @FILELIST | &FILELIST] ... 30 | 31 | INFILE is the name of an input file, '-' will read from stdin. 32 | FILELIST is the name of a text file containing file names or 33 | other @FILELISTs. 34 | 35 | For @FILELIST, paths in the file list are relative to the working 36 | directory where cog was called. For &FILELIST, paths in the file 37 | list are relative to the file list location. 38 | 39 | OPTIONS: 40 | -c Checksum the output to protect it against accidental change. 41 | -d Delete the generator code from the output file. 42 | -D name=val Define a global string available to your generator code. 43 | -e Warn if a file has no cog code in it. 44 | -I PATH Add PATH to the list of directories for data files and modules. 45 | -n ENCODING Use ENCODING when reading and writing files. 46 | -o OUTNAME Write the output to OUTNAME. 47 | -p PROLOGUE Prepend the generator source with PROLOGUE. Useful to insert an 48 | import line. Example: -p "import math" 49 | -P Use print() instead of cog.outl() for code output. 50 | -r Replace the input file with the output. 51 | -s STRING Suffix all generated output lines with STRING. 52 | -U Write the output with Unix newlines (only LF line-endings). 53 | -w CMD Use CMD if the output file needs to be made writable. 54 | A %s in the CMD will be filled with the filename. 55 | -x Excise all the generated output without running the generators. 56 | -z The end-output marker can be omitted, and is assumed at eof. 57 | -v Print the version of cog and exit. 58 | --check Check that the files would not change if run again. 59 | --diff With --check, show a diff of what failed the check. 60 | --markers='START END END-OUTPUT' 61 | The patterns surrounding cog inline instructions. Should 62 | include three values separated by spaces, the start, end, 63 | and end-output markers. Defaults to '[[[cog ]]] [[[end]]]'. 64 | --verbosity=VERBOSITY 65 | Control the amount of output. 2 (the default) lists all files, 66 | 1 lists only changed files, 0 lists no files. 67 | -h, --help Print this help. 68 | 69 | .. {{{end}}} (sum: aE5SIko6oj) 70 | 71 | In addition to running cog as a command on the command line, you can also 72 | invoke it as a module with the Python interpreter: 73 | 74 | .. code-block:: bash 75 | 76 | $ python3 -m cogapp [options] [arguments] 77 | 78 | Note that the Python module is called "cogapp". 79 | 80 | 81 | Input files 82 | ----------- 83 | 84 | Files on the command line are processed as input files. All input files are 85 | assumed to be UTF-8 encoded. Using a minus for a filename (``-``) will read the 86 | standard input. 87 | 88 | Files can also be listed in a text file named on the command line 89 | with an ``@``: 90 | 91 | .. code-block:: bash 92 | 93 | $ cog @files_to_cog.txt 94 | 95 | File names in the list file are relative to the current directory. You can also 96 | use ``&files_to_cog.txt`` and the file names will be relative to the location 97 | of the list file. 98 | 99 | These list files can be nested, and each line can contain switches as well as a 100 | file to process. For example, you can create a file cogfiles.txt: 101 | 102 | .. code-block:: text 103 | 104 | # These are the files I run through cog 105 | mycode.cpp 106 | myothercode.cpp 107 | myschema.sql -s " --**cogged**" 108 | readme.txt -s "" 109 | 110 | then invoke cog like this: 111 | 112 | .. code-block:: bash 113 | 114 | $ cog -s " //**cogged**" @cogfiles.txt 115 | 116 | Now cog will process four files, using C++ syntax for markers on all the C++ 117 | files, SQL syntax for the .sql file, and no markers at all on the readme.txt 118 | file. 119 | 120 | As another example, cogfiles2.txt could be: 121 | 122 | .. code-block:: text 123 | 124 | template.h -D thefile=data1.xml -o data1.h 125 | template.h -D thefile=data2.xml -o data2.h 126 | 127 | with cog invoked like this: 128 | 129 | .. code-block:: bash 130 | 131 | $ cog -D version=3.4.1 @cogfiles2.txt 132 | 133 | Cog will process template.h twice, creating both data1.h and data2.h. Both 134 | executions would define the variable version as "3.4.1", but the first run 135 | would have thefile equal to "data1.xml" and the second run would have thefile 136 | equal to "data2.xml". 137 | 138 | 139 | Overwriting files 140 | ----------------- 141 | 142 | The ``-r`` flag tells cog to write the output back to the input file. If the 143 | input file is not writable (for example, because it has not been checked out of 144 | a source control system), a command to make the file writable can be provided 145 | with ``-w``: 146 | 147 | .. code-block:: bash 148 | 149 | $ cog -r -w "p4 edit %s" @files_to_cog.txt 150 | 151 | 152 | Setting globals 153 | --------------- 154 | 155 | Global values can be set from the command line with the ``-D`` flag. For 156 | example, invoking Cog like this: 157 | 158 | .. code-block:: bash 159 | 160 | $ cog -D thefile=fooey.xml mycode.txt 161 | 162 | will run Cog over mycode.txt, but first define a global variable called thefile 163 | with a value of "fooey.xml". This variable can then be referenced in your 164 | generator code. You can provide multiple ``-D`` arguments on the command line, 165 | and all will be defined and available. 166 | 167 | The value is always interpreted as a Python string, to simplify the problem of 168 | quoting. This means that: 169 | 170 | .. code-block:: bash 171 | 172 | $ cog -D NUM_TO_DO=12 173 | 174 | will define ``NUM_TO_DO`` not as the integer ``12``, but as the string 175 | ``"12"``, which are different and not equal values in Python. Use 176 | `int(NUM_TO_DO)` to get the numeric value. 177 | 178 | 179 | Checksummed output 180 | ------------------ 181 | 182 | If cog is run with the ``-c`` flag, then generated output is accompanied by 183 | a checksum: 184 | 185 | .. code-block:: sql 186 | 187 | --[[[cog 188 | -- import cog 189 | -- for i in range(10): 190 | -- cog.out("%d " % i) 191 | --]]] 192 | 0 1 2 3 4 5 6 7 8 9 193 | --[[[end]]] (sum: vXcVMEUp9m) 194 | 195 | The checksum uses a compact base64 encoding to be less visually distracting. 196 | If the generated code is edited by a misguided developer, the next time cog 197 | is run, the checksum won't match, and cog will stop to avoid overwriting the 198 | edited code. 199 | 200 | Cog can also read files with the older hex checksum format: 201 | 202 | .. code-block:: sql 203 | 204 | --[[[end]]] (checksum: bd7715304529f66c4d3493e786bb0f1f) 205 | 206 | When such files are regenerated, the checksum will be updated to the new 207 | base64 format automatically. 208 | 209 | 210 | Continuous integration 211 | ---------------------- 212 | 213 | You can use the ``--check`` option to run cog just to check that the files 214 | would not change if run again. This is useful in continuous integration to 215 | check that your files have been updated properly. 216 | 217 | The ``--diff`` option will show a unified diff of the change that caused 218 | ``--check`` to fail. 219 | 220 | 221 | Output line suffixes 222 | -------------------- 223 | 224 | To make it easier to identify generated lines when grepping your source files, 225 | the ``-s`` switch provides a suffix which is appended to every non-blank text 226 | line generated by Cog. For example, with this input file (mycode.txt): 227 | 228 | .. code-block:: text 229 | 230 | [[[cog 231 | cog.outl('Three times:\n') 232 | for i in range(3): 233 | cog.outl('This is line %d' % i) 234 | ]]] 235 | [[[end]]] 236 | 237 | invoking cog like this: 238 | 239 | .. code-block:: bash 240 | 241 | $ cog -s " //(generated)" mycode.txt 242 | 243 | will produce this output: 244 | 245 | .. code-block:: text 246 | 247 | [[[cog 248 | cog.outl('Three times:\n') 249 | for i in range(3): 250 | cog.outl('This is line %d' % i) 251 | ]]] 252 | Three times: //(generated) 253 | 254 | This is line 0 //(generated) 255 | This is line 1 //(generated) 256 | This is line 2 //(generated) 257 | [[[end]]] 258 | 259 | 260 | Miscellaneous 261 | ------------- 262 | 263 | The ``-n`` option lets you tell cog what encoding to use when reading and 264 | writing files. 265 | 266 | The ``--verbose`` option lets you control how much cog should chatter about the 267 | files it is cogging. ``--verbose=2`` is the default: cog will name every file 268 | it considers, and whether it has changed. ``--verbose=1`` will only name the 269 | changed files. ``--verbose=0`` won't mention any files at all. 270 | 271 | The ``--markers`` option lets you control the syntax of the marker lines. The 272 | value must be a string with two spaces in it. The three markers are the three 273 | pieces separated by the spaces. The default value for markers is ``"[[[cog ]]] 274 | [[[end]]]"``. 275 | 276 | The ``-x`` flag tells cog to delete the old generated output without running 277 | the generators. This lets you remove all the generated output from a source 278 | file. 279 | 280 | The ``-d`` flag tells cog to delete the generators from the output file. This 281 | lets you generate content in a public file but not have to show the generator 282 | to your customers. 283 | 284 | The ``-U`` flag causes the output file to use pure Unix newlines rather than 285 | the platform's native line endings. You can use this on Windows to produce 286 | Unix-style output files. 287 | 288 | The ``-I`` flag adds a directory to the path used to find Python modules. 289 | 290 | The ``-p`` option specifies Python text to prepend to embedded generator 291 | source, which can keep common imports out of source files. 292 | 293 | The ``-z`` flag lets you omit the ``[[[end]]]`` marker line, and it will be 294 | assumed at the end of the file. 295 | -------------------------------------------------------------------------------- /docs/source.rst: -------------------------------------------------------------------------------- 1 | Writing the source files 2 | ======================== 3 | 4 | Source files to be run through cog are mostly just plain text that will be 5 | passed through untouched. The Python code in your source file is standard 6 | Python code. Any way you want to use Python to generate text to go into your 7 | file is fine. Each chunk of Python code (between the ``[[[cog`` and ``]]]`` 8 | lines) is called a *generator* and is executed in sequence. 9 | 10 | The output area for each generator (between the ``]]]`` and ``[[[end]]]`` 11 | lines) is deleted, and the output of running the Python code is inserted in its 12 | place. To accommodate all source file types, the format of the marker lines is 13 | irrelevant. If the line contains the special character sequence, the whole 14 | line is taken as a marker. Any of these lines mark the beginning of executable 15 | Python code: 16 | 17 | .. code-block:: text 18 | 19 | //[[[cog 20 | /* cog starts now: [[[cog */ 21 | -- [[[cog (this is cog Python code) 22 | #if 0 // [[[cog 23 | 24 | Cog can also be used in languages without multi-line comments. If the marker 25 | lines all have the same text before the triple brackets, and all the lines in 26 | the generator code also have this text as a prefix, then the prefixes are 27 | removed from all the generator lines before execution. For example, in a SQL 28 | file, this: 29 | 30 | .. code-block:: sql 31 | 32 | --[[[cog 33 | -- import cog 34 | -- for table in ['customers', 'orders', 'suppliers']: 35 | -- cog.outl("drop table %s;" % table) 36 | --]]] 37 | --[[[end]]] 38 | 39 | will produce this: 40 | 41 | .. code-block:: sql 42 | 43 | --[[[cog 44 | -- import cog 45 | -- for table in ['customers', 'orders', 'suppliers']: 46 | -- cog.outl("drop table %s;" % table) 47 | --]]] 48 | drop table customers; 49 | drop table orders; 50 | drop table suppliers; 51 | --[[[end]]] 52 | 53 | Finally, a compact form can be used for single-line generators. The begin-code 54 | marker and the end-code marker can appear on the same line, and all the text 55 | between them will be taken as a single Python line: 56 | 57 | .. code-block:: cpp 58 | 59 | // blah blah 60 | //[[[cog import MyModule as m; m.generateCode() ]]] 61 | //[[[end]]] 62 | 63 | You can also use this form to simply import a module. The top-level statements 64 | in the module can generate the code. 65 | 66 | If you have special requirements for the syntax of your file, you can use the 67 | ``--markers`` option to define new markers. 68 | 69 | If there are multiple generators in the same file, they are executed with the 70 | same globals dictionary, so it is as if they were all one Python module. 71 | 72 | Cog tries to do the right thing with white space. Your Python code can be 73 | block-indented to match the surrounding text in the source file, and cog will 74 | re-indent the output to fit as well. All of the output for a generator is 75 | collected as a block of text, a common whitespace prefix is removed, and then 76 | the block is indented to match the indentation of the cog generator. This means 77 | the left-most non-whitespace character in your output will have the same 78 | indentation as the begin-code marker line. Other lines in your output keep 79 | their relative indentation. 80 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "cogapp" 3 | description = "Cog: A content generator for executing Python snippets in source files." 4 | readme = "README.rst" 5 | authors = [ 6 | {name = "Ned Batchelder", email = "ned@nedbatchelder.com"}, 7 | ] 8 | license.text = "MIT" 9 | classifiers = [ 10 | "Development Status :: 5 - Production/Stable", 11 | "Environment :: Console", 12 | "Intended Audience :: Developers", 13 | "License :: OSI Approved :: MIT License", 14 | "Operating System :: OS Independent", 15 | "Programming Language :: Python :: 3.9", 16 | "Programming Language :: Python :: 3.10", 17 | "Programming Language :: Python :: 3.11", 18 | "Programming Language :: Python :: 3.12", 19 | "Programming Language :: Python :: 3.13", 20 | "Topic :: Software Development :: Code Generators", 21 | ] 22 | requires-python = ">= 3.9" 23 | dynamic = ["version"] 24 | 25 | [project.scripts] 26 | cog = "cogapp:main" 27 | 28 | [project.urls] 29 | "Documentation" = "https://cog.readthedocs.io/" 30 | "Code" = "http://github.com/nedbat/cog" 31 | "Issues" = "https://github.com/nedbat/cog/issues" 32 | "Funding" = "https://github.com/users/nedbat/sponsorship" 33 | "Mastodon" = "https://hachyderm.io/@nedbat" 34 | 35 | [tool.pytest.ini_options] 36 | addopts = "-q -rfe" 37 | 38 | [tool.setuptools] 39 | packages = ["cogapp"] 40 | 41 | [tool.setuptools.dynamic] 42 | version.attr = "cogapp.cogapp.__version__" 43 | 44 | [build-system] 45 | requires = ["setuptools"] 46 | build-backend = "setuptools.build_meta" 47 | -------------------------------------------------------------------------------- /requirements.pip: -------------------------------------------------------------------------------- 1 | build 2 | check-manifest 3 | coverage 4 | Sphinx 5 | tox 6 | tox-gh 7 | twine 8 | -------------------------------------------------------------------------------- /success/README.txt: -------------------------------------------------------------------------------- 1 | The source of the old Python Success Story about cog: 2 | https://www.python.org/about/success/cog/ 3 | -------------------------------------------------------------------------------- /success/cog-success.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | Cog: A Code Generation Tool Written in Python 3 | ============================================= 4 | 5 | :Category: Business 6 | :Keywords: cpython, code generation, utility, scripting, companion language 7 | :Title: Cog: A Code Generation Tool Written in Python 8 | :Author: Ned Batchelder 9 | :Date: $Date: 2004/05/25 21:12:37 $ 10 | :Websites: http://www.nedbatchelder.com/ 11 | :Website: http://www.kubisoftware.com/ 12 | :Summary: Cog, a general-purpose Python-based code generation tool, is used to speed development of a collaboration system written in C++. 13 | :Logo: images/batchelder-logo.gif 14 | 15 | Introduction 16 | ------------ 17 | 18 | `Cog`__ is a simple code generation tool written in Python. We use it or its 19 | results every day in the production of Kubi. 20 | 21 | __ http://www.nedbatchelder.com/code/cog 22 | 23 | `Kubi`__ is a collaboration system embodied in a handful of different products. 24 | We have a schema that describes the representation of customers' 25 | collaboration data: discussion topics, documents, calendar events, and so on. 26 | This data has to be handled in many ways: stored in a number of different 27 | data stores, shipped over the wire in an XML representation, manipulated in 28 | memory using traditional C++ objects, presented for debugging, and reasoned 29 | about to assess data validity, to name a few. 30 | 31 | __ http://www.kubisoftware.com/ 32 | 33 | We needed a way to describe this schema once and then reliably produce 34 | executable code from it. 35 | 36 | The Hard Way with C++ 37 | --------------------- 38 | 39 | Our first implementation of this schema involved a fractured collection of 40 | representations. The XML protocol module had tables describing the 41 | serialization and deserialization of XML streams. The storage modules had 42 | other tables describing the mapping from disk to memory structures. The 43 | validation module had its own tables containing rules about which properties 44 | had to be present on which items. The in-memory objects had getters and 45 | setters for each property. 46 | 47 | It worked, after a fashion, but was becoming unmanageable. Adding a new 48 | property to the schema required editing ten tables in different formats in 49 | as many source files, as well as adding getters and setters for the new 50 | property. There was no single authority in the code for the schema as a 51 | whole. Different aspects of the schema were represented in different 52 | ways in different files. 53 | 54 | We tried to simplify the mess using C++ macros. This worked to a degree, but 55 | was still difficult to manage. The schema representation was hampered by the 56 | simplistic nature of C++ macros, and the possibilities for expansion were 57 | extremely limited. 58 | 59 | The schema tables that could not be created with these primitive macros were 60 | still composed and edited by hand. Changing a property in the schema still 61 | meant touching a dozen files. This was tedious and error prone. Missing one 62 | place might introduce a bug that would go unnoticed for days. 63 | 64 | 65 | Searching for a Better Way 66 | -------------------------- 67 | 68 | It was becoming clear that we needed a better way to manage the property 69 | schema. Not only were the existing modifications difficult, but new areas of 70 | development were going to require new uses of the schema, and new kinds of 71 | modification that would be even more onerous. 72 | 73 | We'd been using C++ macros to try to turn a declarative description of the 74 | schema into executable code. The better way to do it is with code 75 | generation: a program that writes programs. We could use a tool to read the 76 | schema and generate the C++ code, then compile that generated code into the 77 | product. 78 | 79 | We needed a way to read the schema description file and output pieces of code 80 | that could be integrated into our C++ sources to be compiled with the rest of 81 | the product. 82 | 83 | Rather than write a program specific to our problem, I chose instead to write 84 | a general-purpose, although simple, code generator tool. It would solve the 85 | problem of managing small chunks of generator code sprinkled throughout a 86 | large collection of files. We could then use this general purpose tool to 87 | solve our specific generation problem. 88 | 89 | The tool I wrote is called Cog. Its requirements were: 90 | 91 | * We needed to be able to perform interesting computation on the schema to 92 | create the code we needed. Cog would have to provide a powerful language 93 | to write the code generators in. An existing language would make it easier 94 | for developers to begin using Cog. 95 | 96 | * I wanted developers to be able to change the schema, and then run the tool 97 | without having to understand the complexities of the code generation. Cog 98 | would have to make it simple to combine the generated chunks of code with 99 | the rest of the C++ source, and it should be simple to run Cog to generate 100 | the final code. 101 | 102 | * The tool shouldn't care about the language of the host file. We originally 103 | wanted to generate C++ files, but we were branching out into other 104 | languages. The generation process should be a pure text process, without 105 | regard to the eventual interpretation of that text. 106 | 107 | * Because the schema would change infrequently, the generation of code should 108 | be an edit-time activity, rather than a build-time activity. This avoided 109 | having to run the code generator as part of the build, and meant that the 110 | generated code would be available to our IDE and debugger. 111 | 112 | 113 | Code Generation with Python 114 | --------------------------- 115 | 116 | The language I chose for the code generators was, of course, Python. Its 117 | simplicity and power are perfect for the job of reading data files and 118 | producing code. To simplify the integration with the C++ code, the Python 119 | generators are inserted directly into the C++ file as comments. 120 | 121 | Cog reads a text file (C++ in our case), looking for specially-marked 122 | sections of text, that it will use as generators. It executes those sections 123 | as Python code, capturing the output. The output is then spliced into the 124 | file following the generator code. 125 | 126 | Because the generator code and its output are both kept in the file, there is 127 | no distinction between the input file and output file. Cog reads and writes 128 | the same file, and can be run over and over again without losing information. 129 | 130 | .. figure:: images/cog-web.png 131 | :alt: Cog's Processing Model 132 | 133 | *Cog processes text files, converting specially marked sections of the file 134 | into new content without disturbing the rest of the file or the sections 135 | that it executes to produce the generated content.* `Zoom in`__ 136 | 137 | __ images/cog.png 138 | 139 | In addition to executing Python generators, Cog itself is written in Python. 140 | Python's dynamic nature made it simple to execute the Python code Cog found, 141 | and its flexibility made it possible to execute it in a properly-constructed 142 | environment to get the desired semantics. Much of Cog's code is concerned 143 | with getting indentation correct: I wanted the author to be able to organize 144 | his generator code to look good in the host file, and produce generated code 145 | that looked good as well, without worrying about fiddly whitespace issues. 146 | 147 | Python's OS-level integration let me execute shell commands where needed. We 148 | use Perforce for source control, which keeps files read-only until they need 149 | to be edited. When running Cog, it may need to change files that the 150 | developer has not edited yet. It can execute a shell command to check out 151 | files that are read-only. 152 | 153 | Lastly, we used XML for our new property schema description, and Python's 154 | wide variety of XML processing libraries made parsing the XML a snap. 155 | 156 | 157 | An Example 158 | ---------- 159 | 160 | Here's a concrete but slightly contrived example. The properties are 161 | described in an XML file:: 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | We can write a C++ file with inlined Python code:: 172 | 173 | // SchemaPropEnum.h 174 | enum SchemaPropEnum { 175 | /* [[[cog 176 | import cog, handyxml 177 | for p in handyxml.xpath('Properties.xml', '//property'): 178 | cog.outl("Property%s," % p.name) 179 | ]]] */ 180 | // [[[end]]] 181 | }; 182 | 183 | After running this file through Cog, it looks like this:: 184 | 185 | // SchemaPropEnum.h 186 | enum SchemaPropEnum { 187 | /* [[[cog 188 | import cog, handyxml 189 | for p in handyxml.xpath('Properties.xml', '//property'): 190 | cog.outl("Property%s," % p.name) 191 | ]]] */ 192 | PropertyId, 193 | PropertyRevNum, 194 | PropertySubject, 195 | PropertyModDate, 196 | // [[[end]]] 197 | }; 198 | 199 | The lines with triple-brackets are marker lines that delimit the sections Cog 200 | cares about. The text between the **[[[cog and ]]]** lines is generator Python 201 | code. The text between **]]]** and **[[[end]]]** is the output from the last run of 202 | Cog (if any). For each chunk of generator code it finds, Cog will: 203 | 204 | 1. discard the output from the last run, 205 | 2. execute the generator code, 206 | 3. capture the output, from the cog.outl calls, and 207 | 4. insert the output back into the output section. 208 | 209 | 210 | How It Worked Out 211 | ----------------- 212 | 213 | In a word, great. We now have a powerful tool that lets us maintain a single 214 | XML file that describes our data schema. Developers changing the schema have 215 | a simple tool to run that generates code from the schema, producing output 216 | code in four different languages across 50 files. 217 | 218 | Where we once used a repetitive and aggravating process that was inadequate 219 | to our needs, we now have an automated process that lets developers express 220 | themselves and have Cog do the hard work. 221 | 222 | Python's flexibility and power were put to work in two ways: to develop Cog 223 | itself, and sprinkled throughout our C++ source code to give our developers a 224 | powerful tool to turn static data into running code. 225 | 226 | Although our product is built in C++, we've used Python to increase our 227 | productivity and expressive power, ease maintenance work, and automate 228 | error-prone tasks. Our shipping software is built every day with Python 229 | hard at work behind the scenes. 230 | 231 | More information, and Cog itself, is available at 232 | http://www.nedbatchelder.com/code/cog 233 | 234 | 235 | About the Author 236 | ---------------- 237 | 238 | *Ned Batchelder is a professional software developer who struggles along with 239 | C++, using Python to ease the friction every chance he gets. A previous 240 | project of his,* `Natsworld`__, *was the subject of an earlier Python Success Story.* 241 | 242 | __ /success&story=natsworld 243 | 244 | -------------------------------------------------------------------------------- /success/cog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nedbat/cog/315b06062f37598e7830d85c2851eabffe159dd4/success/cog.png -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # tox configuration for Cog. 2 | 3 | [tox] 4 | envlist = py39,py310,py311,py312,py313,coverage 5 | 6 | [testenv] 7 | deps = 8 | coverage 9 | pytest 10 | 11 | commands = 12 | coverage run -m pytest {posargs} 13 | 14 | usedevelop = True 15 | 16 | [testenv:coverage] 17 | skip_install = True 18 | commands = 19 | coverage combine -q 20 | coverage report -m 21 | 22 | [gh] 23 | python = 24 | 3.9 = py39 25 | 3.10 = py310 26 | 3.11 = py311 27 | 3.12 = py312 28 | 3.13 = py313 29 | --------------------------------------------------------------------------------