├── .bumpversion.cfg ├── .editorconfig ├── .github ├── FUNDING.yml └── workflows │ ├── pythonpackage.yml │ └── sphinx.yml ├── .gitignore ├── .readthedocs.yml ├── CHANGELOG.md ├── CHANGELOG.rst ├── CONTRIBUTING.rst ├── LICENSE ├── README.rst ├── docs ├── Makefile ├── README.md ├── changelog.rst ├── comparison.csv ├── conf.py ├── contributing.rst ├── docs │ └── dot ├── dot │ ├── makefile │ ├── pipeline-simple.svg │ ├── pipeline.dot │ └── pipeline.svg ├── example.csv ├── index.rst ├── installation.rst ├── make.bat ├── readme.rst ├── spelling_wordlist.txt └── usage.rst ├── environment.yml ├── makefile ├── pyproject.toml ├── src └── pantable │ ├── __init__.py │ ├── ast.py │ ├── cli │ ├── __init__.py │ ├── pantable.py │ ├── pantable2csv.py │ └── pantable2csvx.py │ ├── codeblock_to_table.py │ ├── io.py │ ├── table_to_codeblock.py │ └── util.py └── tests ├── __init__.py ├── ast_test.py ├── files ├── makefile ├── md │ └── tables.md ├── md_codeblock │ ├── comparison.md │ ├── csv_table_gbk.csv │ ├── csv_tables.csv │ ├── empty_csv.md │ ├── encoding.md │ ├── full_test.md │ ├── include_external_csv.md │ ├── include_external_csv_invalid_path.md │ ├── irregular_csv.md │ ├── issue-57.md │ ├── one_row_table.md │ ├── simple_test.md │ ├── testing_0_table_width.md │ └── testing_wrong_type.md ├── md_codeblock_idem_test.py ├── md_codeblock_reference │ ├── comparison.md │ ├── empty_csv.md │ ├── encoding.md │ ├── full_test.md │ ├── include_external_csv.md │ ├── include_external_csv_invalid_path.md │ ├── invalid_yaml.md │ ├── irregular_csv.md │ ├── issue-57.md │ ├── one_row_table.md │ ├── simple_test.md │ ├── testing_0_table_width.md │ └── testing_wrong_type.md ├── md_codeblock_test.py ├── md_reference │ └── tables.md ├── md_test.py ├── native │ ├── nordics.native │ ├── planets.native │ └── students.native ├── native_iden_test.py ├── native_reference │ ├── makefile │ ├── nordics.md │ ├── planets.md │ └── students.md └── native_test.py └── util_test.py /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.14.2 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:pyproject.toml] 7 | search = version = "{current_version}" 8 | replace = version = "{new_version}" 9 | 10 | [bumpversion:file:makefile] 11 | search = {current_version} 12 | replace = {new_version} 13 | 14 | [bumpversion:file:docs/README.md] 15 | search = v{current_version}. 16 | replace = v{new_version}. 17 | 18 | [bumpversion:file:README.rst] 19 | search = v{current_version}. 20 | replace = v{new_version}. 21 | 22 | [bumpversion:file:docs/conf.py] 23 | search = version = release = "{current_version}" 24 | replace = version = release = "{new_version}" 25 | 26 | [bumpversion:file:src/pantable/__init__.py] 27 | search = __version__ = '{current_version}' 28 | replace = __version__ = '{new_version}' 29 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # see https://editorconfig.org/ 2 | root = true 3 | 4 | [*] 5 | # Use Unix-style newlines for most files (except Windows files, see below). 6 | end_of_line = lf 7 | trim_trailing_whitespace = true 8 | indent_style = space 9 | insert_final_newline = true 10 | indent_size = 4 11 | charset = utf-8 12 | 13 | [*.{bat,cmd,ps1}] 14 | end_of_line = crlf 15 | 16 | [*.{yml,yaml}] 17 | indent_size = 2 18 | 19 | [*.tsv] 20 | indent_style = tab 21 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [ickc] 4 | -------------------------------------------------------------------------------- /.github/workflows/pythonpackage.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | pull_request: 9 | schedule: 10 | - cron: '37 11 * * 5' 11 | 12 | jobs: 13 | build-n-publish: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | # see setup.py for supported versions 20 | # here instead of having a matrix 21 | # we only test combinations in a round-robin fashion 22 | # make sure the versions are monotmonic increasing w.r.t. each other 23 | # other wise e.g. an older version of a dependency may not work well with a newer version of Python 24 | include: 25 | - python-version: "3.7" 26 | pandoc-version: "2.11.4" 27 | pyyaml-version: "pyyaml>=5,<6" 28 | numpy-version: "numpy>=1.16.6,<1.17" 29 | yamlloader-version: "yamlloader>=1,<1.1" 30 | - python-version: "pypy-3.7" 31 | pandoc-version: "2.13" 32 | pyyaml-version: "pyyaml>=5,<6" 33 | numpy-version: "numpy>=1.18.5,<1.19" 34 | yamlloader-version: "yamlloader>=1,<1.1" 35 | - python-version: "3.8" 36 | pandoc-version: "2.15" 37 | pyyaml-version: "pyyaml>=6,<7" 38 | numpy-version: "numpy>=1.20.3,<1.21" 39 | yamlloader-version: "yamlloader>=1.1,<2" 40 | - python-version: "3.9" 41 | pandoc-version: "2.16.2" 42 | pyyaml-version: "pyyaml>=6,<7" 43 | numpy-version: "numpy>=1.21.4,<1.22" 44 | yamlloader-version: "yamlloader>=1.1,<2" 45 | - python-version: "3.10" 46 | pandoc-version: "latest" 47 | pyyaml-version: "pyyaml>=6,<7" 48 | numpy-version: "numpy>=1.22,<1.23" 49 | yamlloader-version: "yamlloader>=1.1,<2" 50 | steps: 51 | - uses: actions/checkout@v2 52 | - name: Set up Python ${{ matrix.python-version }} 53 | uses: actions/setup-python@v2 54 | with: 55 | python-version: ${{ matrix.python-version }} 56 | - name: Install dependencies—pip 57 | run: | 58 | python -m pip install -U poetry setuptools "${{ matrix.pyyaml-version }}" "${{ matrix.numpy-version }}" "${{ matrix.yamlloader-version }}" 59 | python -m pip install .[extras,tests] 60 | # let coverage read generated setup.py instead of pyproject.toml 61 | make setup.py 62 | mv pyproject.toml .pyproject.toml 63 | - name: Install dependencies—pandoc 64 | run: | 65 | # pandoc 66 | [[ ${{ matrix.pandoc-version }} == "latest" ]] && url="https://github.com/jgm/pandoc/releases/latest" || url="https://github.com/jgm/pandoc/releases/tag/${{ matrix.pandoc-version }}" 67 | downloadUrl="https://github.com$(curl -L $url | grep -o '/jgm/pandoc/releases/download/.*-amd64\.deb')" 68 | wget --quiet "$downloadUrl" 69 | sudo dpkg -i "${downloadUrl##*/}" 70 | - name: Tests 71 | run: | 72 | make test 73 | coverage combine || true 74 | coverage report 75 | coverage xml 76 | coverage lcov 77 | - name: Coveralls Parallel 78 | uses: coverallsapp/github-action@master 79 | with: 80 | github-token: ${{ secrets.github_token }} 81 | flag-name: run-${{ matrix.test_number }} 82 | parallel: true 83 | path-to-lcov: ./coverage.lcov 84 | - name: Coverage—Codecov 85 | uses: codecov/codecov-action@v1 86 | with: 87 | file: ./coverage.xml 88 | # c.f. https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ 89 | - name: Prepare to publish 90 | if: ${{ startsWith(github.event.ref, 'refs/tags') && matrix.python-version == 3.10 }} 91 | run: | 92 | # undo the above. see "make editable" that already build the packages 93 | rm -f setup.py 94 | mv .pyproject.toml pyproject.toml 95 | - name: Publish distribution 📦 to PyPI 96 | if: ${{ startsWith(github.event.ref, 'refs/tags') && matrix.python-version == 3.10 }} 97 | uses: pypa/gh-action-pypi-publish@master 98 | with: 99 | password: ${{ secrets.pypi_password }} 100 | 101 | coveralls_finish: 102 | needs: build-n-publish 103 | runs-on: ubuntu-latest 104 | steps: 105 | - name: Coveralls Finished 106 | uses: coverallsapp/github-action@master 107 | with: 108 | github-token: ${{ secrets.github_token }} 109 | parallel-finished: true 110 | -------------------------------------------------------------------------------- /.github/workflows/sphinx.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Pages 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | python-version: 14 | - '3.10' 15 | pandoc-version: 16 | - latest 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install dependencies—pip 24 | run: | 25 | pip install -U poetry setuptools tox 26 | make editable EXTRAS=[docs] 27 | - name: Install dependencies—pandoc 28 | run: | 29 | # pandoc 30 | [[ ${{ matrix.pandoc-version }} == "latest" ]] && url="https://github.com/jgm/pandoc/releases/latest" || url="https://github.com/jgm/pandoc/releases/tag/${{ matrix.pandoc-version }}" 31 | downloadUrl="https://github.com$(curl -L $url | grep -o '/jgm/pandoc/releases/download/.*-amd64\.deb')" 32 | wget --quiet "$downloadUrl" 33 | sudo dpkg -i "${downloadUrl##*/}" 34 | - name: Make docs 35 | run: make html 36 | - name: Deploy 37 | uses: peaceiris/actions-gh-pages@v3 38 | with: 39 | github_token: ${{ secrets.GITHUB_TOKEN }} 40 | publish_dir: dist/docs 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | docs/api/ 2 | docs/dot/pipeline-simple.dot 3 | 4 | poetry.lock 5 | setup.py 6 | 7 | .ipynb_checkpoints 8 | *.ipynb 9 | 10 | *.py[cod] 11 | __pycache__ 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Packages 17 | *.egg 18 | *.egg-info 19 | dist 20 | build 21 | eggs 22 | .eggs 23 | parts 24 | bin 25 | var 26 | sdist 27 | wheelhouse 28 | develop-eggs 29 | .installed.cfg 30 | lib 31 | lib64 32 | venv*/ 33 | pyvenv*/ 34 | pip-wheel-metadata/ 35 | 36 | # Installer logs 37 | pip-log.txt 38 | 39 | # Unit test / coverage reports 40 | .coverage* 41 | .tox 42 | .pytest_cache/ 43 | nosetests.xml 44 | coverage.xml 45 | htmlcov 46 | 47 | # Translations 48 | *.mo 49 | 50 | # Buildout 51 | .mr.developer.cfg 52 | 53 | # IDE project files 54 | .project 55 | .pydevproject 56 | .idea 57 | .vscode 58 | *.iml 59 | *.komodoproject 60 | 61 | # Complexity 62 | output/*.html 63 | output/*/index.html 64 | 65 | # Sphinx 66 | docs/_build 67 | 68 | .DS_Store 69 | *~ 70 | .*.sw[po] 71 | .build 72 | .ve 73 | .env 74 | .cache 75 | .pytest 76 | .benchmarks 77 | .bootstrap 78 | .appveyor.token 79 | *.bak 80 | 81 | # Mypy Cache 82 | .mypy_cache/ 83 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/conf.py 5 | 6 | formats: 7 | - htmlzip 8 | - pdf 9 | - epub 10 | 11 | python: 12 | version: 3.8 13 | install: 14 | - method: pip 15 | path: . 16 | extra_requirements: 17 | - docs 18 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Revision history for Pantable 2 | 3 | - v0.14.2: Support pandoc 2.15–16 4 | - improve test matrix in GitHub Actions 5 | - update dependency constraints 6 | - v0.14.1: identical to v0.14.2 7 | - v0.14.0: Support pandoc 2.14 8 | - close #61 9 | - requires panflute >= 2.1 10 | - add `environment.yml` for an example conda environment running pantable 11 | - remove `pantable.util.PandocVersion` as it is merged upstream in `panflute.tools.PandocVersion`. 12 | - v0.13.6: Fix #57; update dependencies. 13 | - v0.13.5: Fix a division error when all widths are zero. 14 | - v0.13.4: Fix converting from native table that contains footnotes. See #58. 15 | - v0.13.3: packaging change only: allow Python4 16 | - v0.13.2: packaging change only: specified required versions for all dependencies including extras 17 | - v0.13.1: util.py: fix `iter_convert_texts_markdown_to_panflute` 18 | - v0.13:added pandoc 2.11.0.4+ & panflute 2+ support 19 | - pandoc 2.10 introduces a new table AST. This version provides complete support of all features supported in the pandoc AST. Hence, older pandoc versions are no longer supported. Install `pantable=0.12.4` if you need to use `pandoc<2.10`. 20 | - deprecated `pipe_tables`, `grid_tables`, `raw_markdown` options in pantable, which were introduced in v0.12. pantable v0.13 has a much better way to process markdown cells that these are no longer needed. 21 | - slight changes on markdown output which should be functionally identical. Both changes in pandoc and pantable cause this. See commit eadc6fb. 22 | - add `short-caption`, `alignment-cells`, `fancy_table`, `format`, `ms`, `ns_head`. See docs for details. 23 | - v0.12.4: Require panflute<2 explicitly 24 | - panflute 2 is released to support pandoc API 1.22. This release ensures that version control is correct when people specify pantable==0.12 in the future. 25 | - v0.12.3: Fixes test and CI; update on supported Python versions 26 | - migrate from Travis CI to GitHub Actions 27 | - supported Python versions are now 3.5-3.8, pypy3 28 | - minor update in README 29 | - v0.12.2: Add `grid_tables` 30 | - v0.12.1: add `include-encoding`, `csv-kwargs` 31 | - closes #36, #38. See doc for details on this new keys. 32 | - v0.12: Drop Python 2 support; enhance CSV with markdown performance 33 | - Dropping Python2 support, existing Python 2 users should be fine with pip>=9.0. See . 34 | 35 | - add `pipe_tables`, `raw_markdown` options. See README for details. This for example will speed up CSV with markdown cells a lot (trading correctness for speed though.) 36 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | .. This is auto-generated from `CHANGELOG.md`. Do not edit this file directly. 2 | 3 | :Date: November 30, 2021 4 | 5 | .. contents:: 6 | :depth: 3 7 | .. 8 | 9 | Revision history for Pantable 10 | ============================= 11 | 12 | - v0.14.2: Support pandoc 2.15–16 13 | 14 | - improve test matrix in GitHub Actions 15 | - update dependency constraints 16 | 17 | - v0.14.1: identical to v0.14.2 18 | - v0.14.0: Support pandoc 2.14 19 | 20 | - close #61 21 | - requires panflute >= 2.1 22 | - add ``environment.yml`` for an example conda environment running pantable 23 | - remove ``pantable.util.PandocVersion`` as it is merged upstream in ``panflute.tools.PandocVersion``. 24 | 25 | - v0.13.6: Fix #57; update dependencies. 26 | - v0.13.5: Fix a division error when all widths are zero. 27 | - v0.13.4: Fix converting from native table that contains footnotes. See #58. 28 | - v0.13.3: packaging change only: allow Python4 29 | - v0.13.2: packaging change only: specified required versions for all dependencies including extras 30 | - v0.13.1: util.py: fix ``iter_convert_texts_markdown_to_panflute`` 31 | - v0.13:added pandoc 2.11.0.4+ & panflute 2+ support 32 | 33 | - pandoc 2.10 introduces a new table AST. This version provides complete support of all features supported in the pandoc AST. Hence, older pandoc versions are no longer supported. Install ``pantable=0.12.4`` if you need to use ``pandoc<2.10``. 34 | - deprecated ``pipe_tables``, ``grid_tables``, ``raw_markdown`` options in pantable, which were introduced in v0.12. pantable v0.13 has a much better way to process markdown cells that these are no longer needed. 35 | - slight changes on markdown output which should be functionally identical. Both changes in pandoc and pantable cause this. See commit eadc6fb. 36 | - add ``short-caption``, ``alignment-cells``, ``fancy_table``, ``format``, ``ms``, ``ns_head``. See docs for details. 37 | 38 | - v0.12.4: Require panflute<2 explicitly 39 | 40 | - panflute 2 is released to support pandoc API 1.22. This release ensures that version control is correct when people specify pantable==0.12 in the future. 41 | 42 | - v0.12.3: Fixes test and CI; update on supported Python versions 43 | 44 | - migrate from Travis CI to GitHub Actions 45 | - supported Python versions are now 3.5-3.8, pypy3 46 | - minor update in README 47 | 48 | - v0.12.2: Add ``grid_tables`` 49 | - v0.12.1: add ``include-encoding``, ``csv-kwargs`` 50 | 51 | - closes #36, #38. See doc for details on this new keys. 52 | 53 | - v0.12: Drop Python 2 support; enhance CSV with markdown performance 54 | 55 | - Dropping Python2 support, existing Python 2 users should be fine with pip>=9.0. See https://python3statement.org/practicalities/. 56 | 57 | - add ``pipe_tables``, ``raw_markdown`` options. See README for details. This for example will speed up CSV with markdown cells a lot (trading correctness for speed though.) 58 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | Bug reports 9 | =========== 10 | 11 | When `reporting a bug `_ please include: 12 | 13 | * Your operating system name and version. 14 | * Any details about your local setup that might be helpful in troubleshooting. 15 | * Detailed steps to reproduce the bug. 16 | 17 | Documentation improvements 18 | ========================== 19 | 20 | pantable could always use more documentation, whether as part of the 21 | official pantable docs, in docstrings, or even on the web in blog posts, 22 | articles, and such. 23 | 24 | Feature requests and feedback 25 | ============================= 26 | 27 | The best way to send feedback is to file an issue at https://github.com/ickc/pantable/issues. 28 | 29 | If you are proposing a feature: 30 | 31 | * Explain in detail how it would work. 32 | * Keep the scope as narrow as possible, to make it easier to implement. 33 | * Remember that this is a volunteer-driven project, and that code contributions are welcome :) 34 | 35 | Development 36 | =========== 37 | 38 | To set up `pantable` for local development: 39 | 40 | 1. Fork `pantable `_ 41 | (look for the "Fork" button). 42 | 2. Clone your fork locally:: 43 | 44 | git clone git@github.com:YOURGITHUBNAME/pantable.git 45 | 46 | 3. Create a branch for local development:: 47 | 48 | git checkout -b name-of-your-bugfix-or-feature 49 | 50 | Now you can make your changes locally. 51 | 52 | 4. When you're done making changes run all the checks and docs builder with `tox `_ one command:: 53 | 54 | tox 55 | 56 | 5. Commit your changes and push your branch to GitHub:: 57 | 58 | git add . 59 | git commit -m "Your detailed description of your changes." 60 | git push origin name-of-your-bugfix-or-feature 61 | 62 | 6. Submit a pull request through the GitHub website. 63 | 64 | Pull Request Guidelines 65 | ----------------------- 66 | 67 | If you need some code review or feedback while you're developing the code just make the pull request. 68 | 69 | For merging, you should: 70 | 71 | 1. Include passing tests (run ``tox``). 72 | 2. Update documentation when there's new API, functionality etc. 73 | 3. Add a note to ``CHANGELOG.md`` about the changes. 74 | 75 | 76 | 77 | Tips 78 | ---- 79 | 80 | To run a subset of tests:: 81 | 82 | tox -e envname -- pytest -k test_myfeature 83 | 84 | To run all the test environments in *parallel*:: 85 | 86 | tox -p auto 87 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2016-2020, Kolen Cheung 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. This is auto-generated from `docs/README.md`. Do not edit this file directly. 2 | 3 | ======================================================================================== 4 | Pantable—A Python library for writing pandoc filters for tables with batteries included. 5 | ======================================================================================== 6 | 7 | :Date: January 25, 2022 8 | 9 | .. contents:: 10 | :depth: 3 11 | .. 12 | 13 | |Documentation Status| |image1| 14 | 15 | |GitHub Actions| |Coverage Status| |image2| |Codacy Badge| |Scrutinizer Status| |CodeClimate Quality Status| 16 | 17 | |Supported versions| |Supported implementations| |PyPI Wheel| |PyPI Package latest release| |GitHub Releases| |Development Status| |Downloads| |Commits since latest release| |License| 18 | 19 | |Conda Recipe| |Conda Downloads| |Conda Version| |Conda Platforms| 20 | 21 | |DOI| 22 | 23 | Introduction 24 | ============ 25 | 26 | Pantable is a Python library that maps the pandoc Table AST to an internal structure losslessly. This enables writing pandoc filters specifically manipulating tables in pandoc. 27 | 28 | This also comes with 3 pandoc filters, ``pantable``, ``pantable2csv``, ``pantable2csvx``. 29 | 30 | ``pantable`` is the main filter, introducing a syntax to include CSV table in markdown source. It supports all table features supported by the pandoc Table AST. 31 | 32 | ``pantable2csv`` complements ``pantable``, is the inverse of ``pantable``, which convert native pandoc tables into the CSV table format defined by ``pantable``. This is lossy as of pandoc 2.11+, which is supported since pantable 0.13. 33 | 34 | ``pantable2csvx`` (experimental, may drop in the future) is similar to ``pantable2csv``, but introduces an extra column with the ``fancy-table`` syntax defined below such that any general pandoc Table AST can be losslessly encoded in CSV format. 35 | 36 | Some example uses are: 37 | 38 | 1. You already have tables in CSV format. 39 | 40 | 2. You feel that directly editing markdown table is troublesome. You want a spreadsheet interface to edit, but want to convert it to native pandoc table for higher readability. And this process might go back and forth. 41 | 42 | 3. You want lower-level control on the table and column widths. 43 | 44 | 4. You want to use all table features supported by the pandoc’s internal AST table format, which is not possible in markdown for pandoc (as of writing.) 45 | 46 | A word on support 47 | ----------------- 48 | 49 | Note that the above is exactly how I use pantable personally. So you can count on the round-trip losslessness. ``pantable`` and ``pantable2csv`` should have robust support since it has been used for years. But since pandoc 2.11 the table AST has been majorly revised. Pantable 0.13 added support for this new AST by completely rewriting pantable, at the same time addresses some of the shortcoming of the original design. Part of the new design is to enable pantable as a library (see `Pantable as a library <#pantable-as-a-library>`__ below) so that its functionality can be extended, similar to how to write a pandoc filter to intercept the AST and modify it, you can intercept the internal structure of PanTable and modify it. 50 | 51 | However, since this library is completely rewritten as of v0.13, 52 | 53 | - ``pantable`` and ``pantable2csv`` as pandoc filters should be stable 54 | 55 | - there may be regression, please open an issue to report this 56 | 57 | - round-trip losslessness may break, please open an issue to report this 58 | - ``pantable2csvx`` as pandoc filter is experimental. API here might change in the future or may be dropped completed (e.g. replaces by something even more general) 59 | - Pantable as a library also is experimental, meaning that the API might be changed in the future. 60 | 61 | Installation 62 | ============ 63 | 64 | Pip 65 | --- 66 | 67 | To manage pantable using pip, open the command line and run 68 | 69 | - ``pip install pantable`` to install 70 | 71 | - ``pip install https://github.com/ickc/pantable/archive/master.zip`` to install the in-development version 72 | 73 | - ``pip install -U pantable`` to upgrade 74 | - ``pip uninstall pantable`` to remove 75 | 76 | You need a matching pandoc version for pantable to work flawlessly. See `Supported pandoc versions <#supported-pandoc-versions>`__ for details. Or, use the `Conda <#conda>`__ method to install below to have the pandoc version automatically managed for you. 77 | 78 | Conda 79 | ----- 80 | 81 | To manage pantable **with a matching pandoc version**, open the command line and run 82 | 83 | - ``conda install -c conda-forge pantable`` to install 84 | - ``conda update pantable`` to upgrade 85 | - ``conda remove pantable`` to remove 86 | 87 | You may also replace ``conda`` by ``mamba``, which is basically a drop-in replacement of the conda package manager. See `mamba-org/mamba: The Fast Cross-Platform Package Manager `__ for details. 88 | 89 | Note on versions 90 | ---------------- 91 | 92 | Supported Python versions 93 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 94 | 95 | pantable v0.12 drop Python 2 support. You need to ``pip install pantable<0.12`` if you need to run it on Python 2. 96 | 97 | To enforce using Python 3, depending on your system, you may need to specify ``python3`` and ``pip3`` explicitly. 98 | 99 | Check the badge above or ``setup.py`` for supported Python versions, ``setup.py`` further indicates support of pypy in additional of CPython. 100 | 101 | Supported pandoc versions 102 | ^^^^^^^^^^^^^^^^^^^^^^^^^ 103 | 104 | pandoc versioning semantics is `MAJOR.MAJOR.MINOR.PATCH `__ and panflute’s is MAJOR.MINOR.PATCH. Below we shows matching versions of pandoc that panflute supports, in descending order. Only major version is shown as long as the minor versions doesn’t matter. 105 | 106 | .. table:: Version Matching [1]_ 107 | 108 | +----------+------------------+---------------------------+-------------------------------+ 109 | | pantable | panflute version | supported pandoc versions | supported pandoc API versions | 110 | +==========+==================+===========================+===============================+ 111 | | 0.14.1 | 2.1.3 | 2.11.0.4–2.16.x | 1.22–1.22.1 | 112 | +----------+------------------+---------------------------+-------------------------------+ 113 | | 0.14 | 2.1 | 2.11.0.4—2.14.x | 1.22 | 114 | +----------+------------------+---------------------------+-------------------------------+ 115 | | 0.13 | 2.0 | 2.11.0.4—2.11.x | 1.22 | 116 | +----------+------------------+---------------------------+-------------------------------+ 117 | | - | not supported | 2.10 | 1.21 | 118 | +----------+------------------+---------------------------+-------------------------------+ 119 | | 0.12 | 1.12 | 2.7-2.9 | 1.17.5–1.20 | 120 | +----------+------------------+---------------------------+-------------------------------+ 121 | 122 | Note: pandoc 2.10 is short lived and 2.11 has minor API changes comparing to that, mainly for fixing its shortcomings. Please avoid using pandoc 2.10. 123 | 124 | To use pantable with pandoc < 2.10, install pantable 0.12 explicitly by ``pip install pantable~=0.12.4``. 125 | 126 | Pantable as pandoc filters 127 | ========================== 128 | 129 | ``pantable`` 130 | ------------ 131 | 132 | This allows CSV tables, optionally containing markdown syntax (disabled by default), to be put in markdown as a fenced code blocks. 133 | 134 | Example 135 | ------- 136 | 137 | Also see the README in `GitHub Pages `__. 138 | 139 | :: 140 | 141 | ```table 142 | --- 143 | caption: '*Awesome* **Markdown** Table' 144 | alignment: RC 145 | table-width: 2/3 146 | markdown: True 147 | --- 148 | First row,defaulted to be header row,can be disabled 149 | 1,cell can contain **markdown**,"It can be aribrary block element: 150 | 151 | - following standard markdown syntax 152 | - like this" 153 | 2,"Any markdown syntax, e.g.",E = mc^2^ 154 | ``` 155 | 156 | becomes 157 | 158 | .. table:: *Awesome* **Markdown** Table 159 | 160 | +---------------+-------------------------------+---------------------------------------+ 161 | | First row | defaulted to be header row | can be disabled | 162 | +===============+===============================+=======================================+ 163 | | 1 | cell can contain **markdown** | It can be aribrary block element: | 164 | | | | | 165 | | | | - following standard markdown syntax | 166 | | | | - like this | 167 | +---------------+-------------------------------+---------------------------------------+ 168 | | 2 | Any markdown syntax, e.g. | E = mc\ :sup:`2` | 169 | +---------------+-------------------------------+---------------------------------------+ 170 | 171 | (The equation might not work if you view this on PyPI.) 172 | 173 | Usage 174 | ----- 175 | 176 | .. code:: bash 177 | 178 | pandoc -F pantable -o README.html README.md 179 | 180 | Syntax 181 | ------ 182 | 183 | Fenced code blocks is used, with a class ``table``. See `Example <#example>`__. 184 | 185 | Optionally, YAML metadata block can be used within the fenced code block, following standard pandoc YAML metadata block syntax. 7 metadata keys are recognized: 186 | 187 | ``caption`` 188 | the caption of the table. Can be block-like. If omitted, no caption will be inserted. Interpreted as markdown only if ``markdown: true`` below. 189 | 190 | Default: disabled. 191 | 192 | ``short-caption`` 193 | the short-caption of the table. Must be inline-like element. Interpreted as markdown only if ``markdown: true`` below. 194 | 195 | Default: disabled. 196 | 197 | ``alignment`` 198 | alignment for columns: a string of characters among ``L,R,C,D``, case-insensitive, corresponds to Left-aligned, Right-aligned, Center-aligned, Default-aligned respectively. e.g. ``LCRD`` for a table with 4 columns. 199 | 200 | You can specify only the beginning that’s non-default. e.g. ``DLCR`` for a table with 8 columns is equivalent to ``DLCRDDDD``. 201 | 202 | Default: ``DDD...`` 203 | 204 | ``alignment-cells`` 205 | alignment per cell. One row per line. A string of characters among ``L,R,C,D``, case-insensitive, corresponds to Left-aligned, Right-aligned, Center-aligned, Default-aligned respectively. e.g. 206 | 207 | :: 208 | 209 | LCRD 210 | DRCL 211 | 212 | for a table with 4 columns, 2 rows. 213 | 214 | you can specify only the top left block that is not default, and the rest of the cells with be default to default automatically. e.g. 215 | 216 | :: 217 | 218 | DC 219 | LR 220 | 221 | for a table with 4 columns, 3 rows will be equivalent to 222 | 223 | :: 224 | 225 | DCDD 226 | LRDD 227 | DDDD 228 | 229 | Default: ``DDD...\n...`` 230 | 231 | ``width`` 232 | a list of relative width corresponding to the width of each columns. ``D`` means default width. e.g. 233 | 234 | .. code:: yaml 235 | 236 | - width 237 | - 0.1 238 | - 0.2 239 | - 0.3 240 | - 0.4 241 | - D 242 | 243 | Again, you can specify only the left ones that are non-default and it will be padded with defaults. 244 | 245 | Default: ``[D, D, D, ...]`` 246 | 247 | ``table-width`` 248 | the relative width of the table (e.g. relative to ``\linewidth``). If specified as a number, and if any of the column width in ``width`` is default, then auto-width will be performed such that the sum of ``width`` equals this number. 249 | 250 | Default: None 251 | 252 | ``header`` 253 | If it has a header row or not. 254 | 255 | Default: True 256 | 257 | ``markdown`` 258 | If CSV table cell contains markdown syntax or not. 259 | 260 | Default: False 261 | 262 | ``fancy_table`` 263 | if true, then the first column of the table will be interpreted as a special fancy-table syntax s.t. it encodes which rows are 264 | 265 | - table-header, 266 | - table-foot, 267 | - multiple table-bodies and 268 | - “body-head” within table-bodies. 269 | 270 | see example below. 271 | 272 | ``include`` 273 | the path to an CSV file, can be relative/absolute. If non-empty, override the CSV in the CodeBlock. 274 | 275 | Default: None 276 | 277 | ``include-encoding`` 278 | if specified, the file from ``include`` will be decoded according to this encoding, else assumed to be UTF-8. Hint: if you save the CSV file via Microsoft Excel, you may need to set this to ``utf-8-sig``. 279 | 280 | ``csv-kwargs`` 281 | If specified, should be a dictionary passed to ``csv.reader`` as options. e.g. 282 | 283 | .. code:: yaml 284 | 285 | --- 286 | csv-kwargs: 287 | dialect: unix 288 | key: value... 289 | ... 290 | 291 | ``format`` 292 | The file format from the data in code-block or include if specified. 293 | 294 | Default: ``csv`` for data from code-block, and infer from extension in include. 295 | 296 | Currently only ``csv`` is supported. 297 | 298 | ``ms`` 299 | (experimental, may drop in the future): a list of int that specifies the number of rows per row-block. e.g. ``[2, 6, 3, 4, 5, 1]`` means the table should have 21 rows, first 2 rows are table-head, last 1 row is table-foot, there are 2 table-bodies (indicated by ``6, 3, 4, 5`` in the middle) where the 1st body ``6, 3`` has 6 body-head and 3 “body-body”, and the 2nd body ``4, 5`` has 4 body-head and 5 “body-body”. 300 | 301 | If this is specified, ``header`` will be ignored. 302 | 303 | Default: None, which would be inferred from ``header``. 304 | 305 | ``ns_head`` 306 | (experimental, may drop in the future): a list of int that specifies the number of head columns per table-body. e.g. ``[1, 2]`` means the 1st table-body has 1 column of head, the 2nd table-body has 2 column of head 307 | 308 | Default: None 309 | 310 | ``pantable2csv`` 311 | ---------------- 312 | 313 | This one is the inverse of ``pantable``, a panflute filter to convert any native pandoc tables into the CSV table format used by pantable. 314 | 315 | Effectively, ``pantable`` forms a “CSV Reader”, and ``pantable2csv`` forms a “CSV Writer”. It allows you to convert back and forth between these 2 formats. 316 | 317 | For example, in the markdown source: 318 | 319 | :: 320 | 321 | +--------+---------------------+--------------------------+ 322 | | First | defaulted to be | can be disabled | 323 | | row | header row | | 324 | +========+=====================+==========================+ 325 | | 1 | cell can contain | It can be aribrary block | 326 | | | **markdown** | element: | 327 | | | | | 328 | | | | - following standard | 329 | | | | markdown syntax | 330 | | | | - like this | 331 | +--------+---------------------+--------------------------+ 332 | | 2 | Any markdown | $$E = mc^2$$ | 333 | | | syntax, e.g. | | 334 | +--------+---------------------+--------------------------+ 335 | 336 | : *Awesome* **Markdown** Table 337 | 338 | running ``pandoc -F pantable2csv -o output.md input.md``, it becomes 339 | 340 | :: 341 | 342 | ``` {.table} 343 | --- 344 | alignment: DDD 345 | caption: '*Awesome* **Markdown** Table' 346 | header: true 347 | markdown: true 348 | table-width: 0.8055555555555556 349 | width: [0.125, 0.3055555555555556, 0.375] 350 | --- 351 | First row,defaulted to be header row,can be disabled 352 | 1,cell can contain **markdown**,"It can be aribrary block element: 353 | 354 | - following standard markdown syntax 355 | - like this 356 | " 357 | 2,"Any markdown syntax, e.g.",$$E = mc^2$$ 358 | ``` 359 | 360 | ``pantable2csvx`` 361 | ----------------- 362 | 363 | (experimental, may drop in the future) 364 | 365 | Similar to ``pantable2csv``, but convert with ``fancy_table`` syntax s.t. any general Table in pandoc AST is in principle losslessly converted to a markdown-ish syntax in a CSV representation. 366 | 367 | e.g. 368 | 369 | .. code:: sh 370 | 371 | pandoc -F pantable2csvx -o tests/files/native_reference/planets.md tests/files/native/planets.native 372 | 373 | would turn the native Table from ``platnets.native``\ [2]_ to 374 | 375 | :: 376 | 377 | ``` {.table} 378 | --- 379 | caption: Data about the planets of our solar system. 380 | alignment: CCDRRRRRRRR 381 | ns-head: 382 | - 3 383 | markdown: true 384 | fancy-table: true 385 | ... 386 | ===,"(1, 2) 387 | ",,Name,Mass (10\^24kg),Diameter (km),Density (kg/m\^3),Gravity (m/s\^2),Length of day (hours),Distance from Sun (10\^6km),Mean temperature (C),Number of moons,Notes 388 | ,"(4, 2) 389 | Terrestrial planets",,Mercury,0.330,"4,879",5427,3.7,4222.6,57.9,167,0,Closest to the Sun 390 | ,,,Venus,4.87,"12,104",5243,8.9,2802.0,108.2,464,0, 391 | ,,,Earth,5.97,"12,756",5514,9.8,24.0,149.6,15,1,Our world 392 | ,,,Mars,0.642,"6,792",3933,3.7,24.7,227.9,-65,2,The red planet 393 | ,"(4, 1) 394 | Jovian planets","(2, 1) 395 | Gas giants",Jupiter,1898,"142,984",1326,23.1,9.9,778.6,-110,67,The largest planet 396 | ,,,Saturn,568,"120,536",687,9.0,10.7,1433.5,-140,62, 397 | ,,"(2, 1) 398 | Ice giants",Uranus,86.8,"51,118",1271,8.7,17.2,2872.5,-195,27, 399 | ,,,Neptune,102,"49,528",1638,11.0,16.1,4495.1,-200,14, 400 | ___,"(1, 2) 401 | Dwarf planets",,Pluto,0.0146,"2,370",2095,0.7,153.3,5906.4,-225,5,Declassified as a planet in 2006. 402 | ``` 403 | 404 | Pantable as a library 405 | ===================== 406 | 407 | (experimental, API may change in the future) 408 | 409 | Documentation here is sparse, partly because the upstream (pandoc) may change the table AST again. See `Crazy ideas: table structure from upstream GitHub `__. 410 | 411 | See the API docs in https://ickc.github.io/pantable/. 412 | 413 | For example, looking at the source of ``pantable`` as a pandoc filter, in ``codeblock_to_table.py``, you will see the main function doing the work is now 414 | 415 | .. code:: python 416 | 417 | pan_table_str = ( 418 | PanCodeBlock 419 | .from_yaml_filter(options=options, data=data, element=element, doc=doc) 420 | .to_pantablestr() 421 | ) 422 | if pan_table_str.table_width is not None: 423 | pan_table_str.auto_width() 424 | return ( 425 | pan_table_str 426 | .to_pantable() 427 | .to_panflute_ast() 428 | ) 429 | 430 | You can see another example from ``table_to_codeblock.py`` which is what ``pantable2csv`` and ``pantable2csvx`` called. 431 | 432 | Below is a diagram illustrating the API: 433 | 434 | .. figure:: docs/dot/pipeline-simple.svg 435 | :alt: Overview 436 | 437 | Overview 438 | 439 | Solid arrows are lossless conversions. Dashed arrows are lossy. 440 | 441 | You can see the pantable internal structure, ``PanTable`` is one-one correspondence to the pandoc Table AST. Similarly for ``PanCodeBlock``. 442 | 443 | It can then losslessly converts between PanTable and PanTableMarkdown, where everything in PanTableMarkdown is now markdown strings (whereas those in PanTable are panflute or panflute-like AST objects.) 444 | 445 | Lastly, it defines a one-one correspondence to PanCodeBlock with ``fancy_table`` syntax mentioned earlier. 446 | 447 | Below is the same diagram with the method names. You’d probably want to zoom into it to see it clearly. 448 | 449 | .. figure:: docs/dot/pipeline.svg 450 | :alt: Detailed w/ methods 451 | 452 | Detailed w/ methods 453 | 454 | Development 455 | =========== 456 | 457 | To run all the tests run ``tox``. GitHub Actions is used for CI too so if you fork this you can check if your commits passes there. 458 | 459 | Related Filters 460 | =============== 461 | 462 | (The table here is created in the beginning of pantable, which has since added more features. This is left here for historical reason and also as a credit to those before this.) 463 | 464 | The followings are pandoc filters written in Haskell that provide similar functionality. This filter is born after testing with theirs. 465 | 466 | - `baig/pandoc-csv2table: A Pandoc filter that renders CSV as Pandoc Markdown Tables. `__ 467 | - `mb21/pandoc-placetable: Pandoc filter to include CSV data (from file or URL) `__ 468 | - `sergiocorreia/panflute/csv-tables.py `__ 469 | 470 | .. table:: 471 | 472 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 473 | | | pandoc-csv2table | pandoc-placetable | panflute example | pantable | 474 | +=============+=========================================+========================+==========================+=======================================================+ 475 | | caption | caption | caption | title | caption | 476 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 477 | | aligns | aligns = LRCD | aligns = LRCD | | aligns = LRCD | 478 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 479 | | width | | widths = "0.5 0.2 0.3" | | width: [0.5, 0.2, 0.3] | 480 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 481 | | table-width | | | | table-width: 1.0 | 482 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 483 | | header | header = yes | no | header = yes | no | has_header: True | False | header: True | False | yes | NO | 484 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 485 | | markdown | | inlinemarkdown | | markdown: True | False | yes | NO | 486 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 487 | | source | source | file | source | include | 488 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 489 | | others | type = simple | multiline | grid | pipe | | | | 490 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 491 | | | | delimiter | | | 492 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 493 | | | | quotechar | | | 494 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 495 | | | | id (wrapped by div) | | | 496 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 497 | | Notes | | | | width are auto-calculated when width is not specified | 498 | +-------------+-----------------------------------------+------------------------+--------------------------+-------------------------------------------------------+ 499 | 500 | .. [1] 501 | For pandoc API verion, check https://hackage.haskell.org/package/pandoc for pandoc-types, which is the same thing. 502 | 503 | .. [2] 504 | copied from pandoc from `here `__, which was dual licensed as CC0 `here `__ 505 | 506 | .. |Documentation Status| image:: https://readthedocs.org/projects/pantable/badge/?version=latest 507 | :target: https://pantable.readthedocs.io/en/latest/?badge=latest&style=plastic 508 | .. |image1| image:: https://github.com/ickc/pantable/workflows/GitHub%20Pages/badge.svg 509 | :target: https://ickc.github.io/pantable 510 | .. |GitHub Actions| image:: https://github.com/ickc/pantable/workflows/Python%20package/badge.svg 511 | .. |Coverage Status| image:: https://codecov.io/gh/ickc/pantable/branch/master/graphs/badge.svg?branch=master 512 | :target: https://codecov.io/github/ickc/pantable 513 | .. |image2| image:: https://coveralls.io/repos/ickc/pantable/badge.svg?branch=master&service=github 514 | :target: https://coveralls.io/r/ickc/pantable 515 | .. |Codacy Badge| image:: https://app.codacy.com/project/badge/Grade/078ebc537c5747f68c1d4ad3d3594bbf 516 | :target: https://www.codacy.com/gh/ickc/pantable/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ickc/pantable&utm_campaign=Badge_Grade 517 | .. |Scrutinizer Status| image:: https://img.shields.io/scrutinizer/quality/g/ickc/pantable/master.svg 518 | :target: https://scrutinizer-ci.com/g/ickc/pantable/ 519 | .. |CodeClimate Quality Status| image:: https://codeclimate.com/github/ickc/pantable/badges/gpa.svg 520 | :target: https://codeclimate.com/github/ickc/pantable 521 | .. |Supported versions| image:: https://img.shields.io/pypi/pyversions/pantable.svg 522 | :target: https://pypi.org/project/pantable 523 | .. |Supported implementations| image:: https://img.shields.io/pypi/implementation/pantable.svg 524 | :target: https://pypi.org/project/pantable 525 | .. |PyPI Wheel| image:: https://img.shields.io/pypi/wheel/pantable.svg 526 | :target: https://pypi.org/project/pantable 527 | .. |PyPI Package latest release| image:: https://img.shields.io/pypi/v/pantable.svg 528 | :target: https://pypi.org/project/pantable 529 | .. |GitHub Releases| image:: https://img.shields.io/github/tag/ickc/pantable.svg?label=github+release 530 | :target: https://github.com/ickc/pantable/releases 531 | .. |Development Status| image:: https://img.shields.io/pypi/status/pantable.svg 532 | :target: https://pypi.python.org/pypi/pantable/ 533 | .. |Downloads| image:: https://img.shields.io/pypi/dm/pantable.svg 534 | :target: https://pypi.python.org/pypi/pantable/ 535 | .. |Commits since latest release| image:: https://img.shields.io/github/commits-since/ickc/pantable/v0.14.2.svg 536 | :target: https://github.com/ickc/pantable/compare/v0.14.2...master 537 | .. |License| image:: https://img.shields.io/pypi/l/pantable.svg 538 | .. |Conda Recipe| image:: https://img.shields.io/badge/recipe-pantable-green.svg 539 | :target: https://anaconda.org/conda-forge/pantable 540 | .. |Conda Downloads| image:: https://img.shields.io/conda/dn/conda-forge/pantable.svg 541 | :target: https://anaconda.org/conda-forge/pantable 542 | .. |Conda Version| image:: https://img.shields.io/conda/vn/conda-forge/pantable.svg 543 | :target: https://anaconda.org/conda-forge/pantable 544 | .. |Conda Platforms| image:: https://img.shields.io/conda/pn/conda-forge/pantable.svg 545 | :target: https://anaconda.org/conda-forge/pantable 546 | .. |DOI| image:: https://zenodo.org/badge/74008159.svg 547 | :target: https://zenodo.org/badge/latestdoi/74008159 548 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | fontsize: 11pt 3 | documentclass: memoir 4 | classoption: article 5 | geometry: inner=1in, outer=1in, top=1in, bottom=1.25in 6 | title: Pantable—A Python library for writing pandoc filters for tables with batteries included. 7 | ... 8 | 9 | 10 | [![Documentation Status](https://readthedocs.org/projects/pantable/badge/?version=latest)](https://pantable.readthedocs.io/en/latest/?badge=latest&style=plastic) 11 | [![Documentation Status](https://github.com/ickc/pantable/workflows/GitHub%20Pages/badge.svg)](https://ickc.github.io/pantable) 12 | 13 | ![GitHub Actions](https://github.com/ickc/pantable/workflows/Python%20package/badge.svg) 14 | [![Coverage Status](https://codecov.io/gh/ickc/pantable/branch/master/graphs/badge.svg?branch=master)](https://codecov.io/github/ickc/pantable) 15 | [![Coverage Status](https://coveralls.io/repos/github/ickc/pantable/badge.svg?branch=master)](https://coveralls.io/github/ickc/pantable?branch=master) 16 | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/078ebc537c5747f68c1d4ad3d3594bbf)](https://www.codacy.com/gh/ickc/pantable/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ickc/pantable&utm_campaign=Badge_Grade) 17 | [![Scrutinizer Status](https://img.shields.io/scrutinizer/quality/g/ickc/pantable/master.svg)](https://scrutinizer-ci.com/g/ickc/pantable/) 18 | [![CodeClimate Quality Status](https://codeclimate.com/github/ickc/pantable/badges/gpa.svg)](https://codeclimate.com/github/ickc/pantable) 19 | 20 | [![Supported versions](https://img.shields.io/pypi/pyversions/pantable.svg)](https://pypi.org/project/pantable) 21 | [![Supported implementations](https://img.shields.io/pypi/implementation/pantable.svg)](https://pypi.org/project/pantable) 22 | [![PyPI Wheel](https://img.shields.io/pypi/wheel/pantable.svg)](https://pypi.org/project/pantable) 23 | [![PyPI Package latest release](https://img.shields.io/pypi/v/pantable.svg)](https://pypi.org/project/pantable) 24 | [![GitHub Releases](https://img.shields.io/github/tag/ickc/pantable.svg?label=github+release)](https://github.com/ickc/pantable/releases) 25 | [![Development Status](https://img.shields.io/pypi/status/pantable.svg)](https://pypi.python.org/pypi/pantable/) 26 | [![Downloads](https://img.shields.io/pypi/dm/pantable.svg)](https://pypi.python.org/pypi/pantable/) 27 | [![Commits since latest release](https://img.shields.io/github/commits-since/ickc/pantable/v0.14.2.svg)](https://github.com/ickc/pantable/compare/v0.14.2...master) 28 | ![License](https://img.shields.io/pypi/l/pantable.svg) 29 | 30 | [![Conda Recipe](https://img.shields.io/badge/recipe-pantable-green.svg)](https://anaconda.org/conda-forge/pantable) 31 | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/pantable.svg)](https://anaconda.org/conda-forge/pantable) 32 | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/pantable.svg)](https://anaconda.org/conda-forge/pantable) 33 | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/pantable.svg)](https://anaconda.org/conda-forge/pantable) 34 | 35 | [![DOI](https://zenodo.org/badge/74008159.svg)](https://zenodo.org/badge/latestdoi/74008159) 36 | 37 | # Introduction 38 | 39 | Pantable is a Python library that maps the pandoc Table AST to an internal structure losslessly. This enables writing pandoc filters specifically manipulating tables in pandoc. 40 | 41 | This also comes with 3 pandoc filters, `pantable`, `pantable2csv`, `pantable2csvx`. 42 | 43 | `pantable` is the main filter, introducing a syntax to include CSV table in markdown source. It supports all table features supported by the pandoc Table AST. 44 | 45 | `pantable2csv` complements `pantable`, is the inverse of `pantable`, which convert native pandoc tables into the CSV table format defined by `pantable`. This is lossy as of pandoc 2.11+, which is supported since pantable 0.13. 46 | 47 | `pantable2csvx` (experimental, may drop in the future) is similar to `pantable2csv`, but introduces an extra column with the `fancy-table` syntax defined below such that any general pandoc Table AST can be losslessly encoded in CSV format. 48 | 49 | Some example uses are: 50 | 51 | 1. You already have tables in CSV format. 52 | 53 | 2. You feel that directly editing markdown table is troublesome. You want a spreadsheet interface to edit, but want to convert it to native pandoc table for higher readability. And this process might go back and forth. 54 | 55 | 3. You want lower-level control on the table and column widths. 56 | 57 | 4. You want to use all table features supported by the pandoc's internal AST table format, which is not possible in markdown for pandoc (as of writing.) 58 | 59 | ## A word on support 60 | 61 | Note that the above is exactly how I use pantable personally. So you can count on the round-trip losslessness. `pantable` and `pantable2csv` should have robust support since it has been used for years. But since pandoc 2.11 the table AST has been majorly revised. Pantable 0.13 added support for this new AST by completely rewriting pantable, at the same time addresses some of the shortcoming of the original design. Part of the new design is to enable pantable as a library (see [Pantable as a library] below) so that its functionality can be extended, similar to how to write a pandoc filter to intercept the AST and modify it, you can intercept the internal structure of PanTable and modify it. 62 | 63 | However, since this library is completely rewritten as of v0.13, 64 | 65 | - `pantable` and `pantable2csv` as pandoc filters should be stable 66 | - there may be regression, please open an issue to report this 67 | - round-trip losslessness may break, please open an issue to report this 68 | - `pantable2csvx` as pandoc filter is experimental. API here might change in the future or may be dropped completed (e.g. replaces by something even more general) 69 | - Pantable as a library also is experimental, meaning that the API might be changed in the future. 70 | 71 | # Installation 72 | 73 | ## Pip 74 | 75 | To manage pantable using pip, open the command line and run 76 | 77 | - `pip install pantable` to install 78 | - `pip install https://github.com/ickc/pantable/archive/master.zip` to install the in-development version 79 | - `pip install -U pantable` to upgrade 80 | - `pip uninstall pantable` to remove 81 | 82 | You need a matching pandoc version for pantable to work flawlessly. See [Supported pandoc versions] for details. Or, use the [Conda] method to install below to have the pandoc version automatically managed for you. 83 | 84 | ## Conda 85 | 86 | To manage pantable **with a matching pandoc version**, open the command line and run 87 | 88 | - `conda install -c conda-forge pantable` to install 89 | - `conda update pantable` to upgrade 90 | - `conda remove pantable` to remove 91 | 92 | You may also replace `conda` by `mamba`, which is basically a drop-in replacement of the conda package manager. See [mamba-org/mamba: The Fast Cross-Platform Package Manager](https://github.com/mamba-org/mamba) for details. 93 | 94 | ## Note on versions 95 | 96 | ### Supported Python versions 97 | 98 | pantable v0.12 drop Python 2 support. You need to `pip install pantable<0.12` if you need to run it on Python 2. 99 | 100 | To enforce using Python 3, depending on your system, you may need to specify `python3` and `pip3` explicitly. 101 | 102 | Check the badge above or `setup.py` for supported Python versions, `setup.py` further indicates support of pypy in additional of CPython. 103 | 104 | #### Supported pandoc versions 105 | 106 | pandoc versioning semantics is [MAJOR.MAJOR.MINOR.PATCH](https://pvp.haskell.org) and panflute's is MAJOR.MINOR.PATCH. Below we shows matching versions of pandoc that panflute supports, in descending order. Only major version is shown as long as the minor versions doesn't matter. 107 | 108 | | pantable | panflute version | supported pandoc versions | supported pandoc API versions | 109 | | -------- | ---------------- | ------------------------- | ----------------------------- | 110 | | 0.14.1-2 | 2.1.3 | 2.11.0.4–2.17.x | 1.22–1.22.1 | 111 | | 0.14 | 2.1 | 2.11.0.4—2.14.x | 1.22 | 112 | | 0.13 | 2.0 | 2.11.0.4—2.11.x | 1.22 | 113 | | - | not supported | 2.10 | 1.21 | 114 | | 0.12 | 1.12 | 2.7-2.9 | 1.17.5–1.20 | 115 | 116 | : Version Matching^[For pandoc API verion, check https://hackage.haskell.org/package/pandoc for pandoc-types, which is the same thing.] 117 | 118 | Note: pandoc 2.10 is short lived and 2.11 has minor API changes comparing to that, mainly for fixing its shortcomings. Please avoid using pandoc 2.10. 119 | 120 | To use pantable with pandoc < 2.10, install pantable 0.12 explicitly by `pip install pantable~=0.12.4`. 121 | 122 | # Pantable as pandoc filters 123 | 124 | ## `pantable` 125 | 126 | This allows CSV tables, optionally containing markdown syntax (disabled by default), to be put in markdown as a fenced code blocks. 127 | 128 | ## Example 129 | 130 | Also see the README in [GitHub Pages](https://ickc.github.io/pantable/). 131 | 132 | ~~~ 133 | ```table 134 | --- 135 | caption: '*Awesome* **Markdown** Table' 136 | alignment: RC 137 | table-width: 2/3 138 | markdown: True 139 | --- 140 | First row,defaulted to be header row,can be disabled 141 | 1,cell can contain **markdown**,"It can be aribrary block element: 142 | 143 | - following standard markdown syntax 144 | - like this" 145 | 2,"Any markdown syntax, e.g.",E = mc^2^ 146 | ``` 147 | ~~~ 148 | 149 | becomes 150 | 151 | ```table 152 | --- 153 | caption: '*Awesome* **Markdown** Table' 154 | alignment: RC 155 | table-width: 2/3 156 | markdown: True 157 | --- 158 | First row,defaulted to be header row,can be disabled 159 | 1,cell can contain **markdown**,"It can be aribrary block element: 160 | 161 | - following standard markdown syntax 162 | - like this" 163 | 2,"Any markdown syntax, e.g.",E = mc^2^ 164 | ``` 165 | 166 | (The equation might not work if you view this on PyPI.) 167 | 168 | ## Usage 169 | 170 | ```bash 171 | pandoc -F pantable -o README.html README.md 172 | ``` 173 | 174 | ## Syntax 175 | 176 | Fenced code blocks is used, with a class `table`. See [Example]. 177 | 178 | Optionally, YAML metadata block can be used within the fenced code block, following standard pandoc YAML metadata block syntax. 7 metadata keys are recognized: 179 | 180 | `caption` 181 | 182 | : the caption of the table. Can be block-like. If omitted, no caption will be inserted. 183 | Interpreted as markdown only if `markdown: true` below. 184 | 185 | Default: disabled. 186 | 187 | `short-caption` 188 | 189 | : the short-caption of the table. Must be inline-like element. 190 | Interpreted as markdown only if `markdown: true` below. 191 | 192 | Default: disabled. 193 | 194 | `alignment` 195 | 196 | : alignment for columns: 197 | a string of characters among `L,R,C,D`, case-insensitive, 198 | corresponds to Left-aligned, Right-aligned, 199 | Center-aligned, Default-aligned respectively. 200 | e.g. `LCRD` for a table with 4 columns. 201 | 202 | You can specify only the beginning that's non-default. 203 | e.g. `DLCR` for a table with 8 columns is equivalent to `DLCRDDDD`. 204 | 205 | Default: `DDD...` 206 | 207 | `alignment-cells` 208 | 209 | : alignment per cell. One row per line. 210 | A string of characters among `L,R,C,D`, case-insensitive, 211 | corresponds to Left-aligned, Right-aligned, 212 | Center-aligned, Default-aligned respectively. 213 | e.g. 214 | 215 | LCRD 216 | DRCL 217 | 218 | for a table with 4 columns, 2 rows. 219 | 220 | you can specify only the top left block that is not default, and the 221 | rest of the cells with be default to default automatically. 222 | e.g. 223 | 224 | DC 225 | LR 226 | 227 | for a table with 4 columns, 3 rows will be equivalent to 228 | 229 | DCDD 230 | LRDD 231 | DDDD 232 | 233 | Default: `DDD...\n...` 234 | 235 | `width` 236 | 237 | : a list of relative width corresponding to the width of each columns. 238 | `D` means default width. 239 | e.g. 240 | 241 | ```yaml 242 | - width 243 | - 0.1 244 | - 0.2 245 | - 0.3 246 | - 0.4 247 | - D 248 | ``` 249 | 250 | Again, you can specify only the left ones that are non-default and it will be padded 251 | with defaults. 252 | 253 | Default: `[D, D, D, ...]` 254 | 255 | `table-width` 256 | 257 | : the relative width of the table (e.g. relative to `\linewidth`). 258 | If specified as a number, and if any of the column width in `width` is default, then 259 | auto-width will be performed such that the sum of `width` equals this number. 260 | 261 | Default: None 262 | 263 | `header` 264 | 265 | : If it has a header row or not. 266 | 267 | Default: True 268 | 269 | `markdown` 270 | 271 | : If CSV table cell contains markdown syntax or not. 272 | 273 | Default: False 274 | 275 | `fancy_table` 276 | 277 | : if true, then the first column of the table will be interpreted as a special fancy-table 278 | syntax s.t. it encodes which rows are 279 | 280 | - table-header, 281 | - table-foot, 282 | - multiple table-bodies and 283 | - "body-head" within table-bodies. 284 | 285 | see example below. 286 | 287 | `include` 288 | : the path to an CSV file, can be relative/absolute. 289 | If non-empty, override the CSV in the CodeBlock. 290 | 291 | Default: None 292 | 293 | `include-encoding` 294 | 295 | : if specified, the file from `include` will be decoded according to this encoding, else assumed to be UTF-8. Hint: if you save the CSV file via Microsoft Excel, you may need to set this to `utf-8-sig`. 296 | 297 | `csv-kwargs` 298 | : If specified, should be a dictionary passed to `csv.reader` as options. e.g. 299 | 300 | ```yaml 301 | --- 302 | csv-kwargs: 303 | dialect: unix 304 | key: value... 305 | ... 306 | ``` 307 | 308 | `format` 309 | 310 | : The file format from the data in code-block or include if specified. 311 | 312 | Default: `csv` for data from code-block, and infer from extension in include. 313 | 314 | Currently only `csv` is supported. 315 | 316 | `ms` 317 | 318 | : (experimental, may drop in the future): a list of int that specifies the number of 319 | rows per row-block. 320 | e.g. `[2, 6, 3, 4, 5, 1]` means 321 | the table should have 21 rows, 322 | first 2 rows are table-head, 323 | last 1 row is table-foot, 324 | there are 2 table-bodies (indicated by `6, 3, 4, 5` in the middle) 325 | where the 1st body `6, 3` has 6 body-head and 3 "body-body", 326 | and the 2nd body `4, 5` has 4 body-head and 5 "body-body". 327 | 328 | If this is specified, `header` will be ignored. 329 | 330 | Default: None, which would be inferred from `header`. 331 | 332 | `ns_head` 333 | 334 | : (experimental, may drop in the future): a list of int that specifies the number of 335 | head columns per table-body. 336 | e.g. `[1, 2]` means 337 | the 1st table-body has 1 column of head, 338 | the 2nd table-body has 2 column of head 339 | 340 | Default: None 341 | 342 | ## `pantable2csv` 343 | 344 | This one is the inverse of `pantable`, a panflute filter to convert any native pandoc tables into the CSV table format used by pantable. 345 | 346 | Effectively, `pantable` forms a "CSV Reader", and `pantable2csv` forms a "CSV Writer". It allows you to convert back and forth between these 2 formats. 347 | 348 | For example, in the markdown source: 349 | 350 | ~~~ 351 | +--------+---------------------+--------------------------+ 352 | | First | defaulted to be | can be disabled | 353 | | row | header row | | 354 | +========+=====================+==========================+ 355 | | 1 | cell can contain | It can be aribrary block | 356 | | | **markdown** | element: | 357 | | | | | 358 | | | | - following standard | 359 | | | | markdown syntax | 360 | | | | - like this | 361 | +--------+---------------------+--------------------------+ 362 | | 2 | Any markdown | $$E = mc^2$$ | 363 | | | syntax, e.g. | | 364 | +--------+---------------------+--------------------------+ 365 | 366 | : *Awesome* **Markdown** Table 367 | ~~~ 368 | 369 | running `pandoc -F pantable2csv -o output.md input.md`{.bash}, it becomes 370 | 371 | ~~~ 372 | ``` {.table} 373 | --- 374 | alignment: DDD 375 | caption: '*Awesome* **Markdown** Table' 376 | header: true 377 | markdown: true 378 | table-width: 0.8055555555555556 379 | width: [0.125, 0.3055555555555556, 0.375] 380 | --- 381 | First row,defaulted to be header row,can be disabled 382 | 1,cell can contain **markdown**,"It can be aribrary block element: 383 | 384 | - following standard markdown syntax 385 | - like this 386 | " 387 | 2,"Any markdown syntax, e.g.",$$E = mc^2$$ 388 | ``` 389 | ~~~ 390 | 391 | ## `pantable2csvx` 392 | 393 | (experimental, may drop in the future) 394 | 395 | Similar to `pantable2csv`, but convert with `fancy_table` syntax s.t. any general Table in pandoc AST 396 | is in principle losslessly converted to a markdown-ish syntax in a CSV representation. 397 | 398 | e.g. 399 | 400 | ```sh 401 | pandoc -F pantable2csvx -o tests/files/native_reference/planets.md tests/files/native/planets.native 402 | ``` 403 | 404 | would turn the native Table from `platnets.native`^[copied from pandoc from [here](https://github.com/jgm/pandoc/blob/master/test/tables/planets.native), which was dual licensed as CC0 [here](https://github.com/sergiocorreia/panflute/pull/172#issuecomment-736252008)] to 405 | 406 | ~~~ 407 | ``` {.table} 408 | --- 409 | caption: Data about the planets of our solar system. 410 | alignment: CCDRRRRRRRR 411 | ns-head: 412 | - 3 413 | markdown: true 414 | fancy-table: true 415 | ... 416 | ===,"(1, 2) 417 | ",,Name,Mass (10\^24kg),Diameter (km),Density (kg/m\^3),Gravity (m/s\^2),Length of day (hours),Distance from Sun (10\^6km),Mean temperature (C),Number of moons,Notes 418 | ,"(4, 2) 419 | Terrestrial planets",,Mercury,0.330,"4,879",5427,3.7,4222.6,57.9,167,0,Closest to the Sun 420 | ,,,Venus,4.87,"12,104",5243,8.9,2802.0,108.2,464,0, 421 | ,,,Earth,5.97,"12,756",5514,9.8,24.0,149.6,15,1,Our world 422 | ,,,Mars,0.642,"6,792",3933,3.7,24.7,227.9,-65,2,The red planet 423 | ,"(4, 1) 424 | Jovian planets","(2, 1) 425 | Gas giants",Jupiter,1898,"142,984",1326,23.1,9.9,778.6,-110,67,The largest planet 426 | ,,,Saturn,568,"120,536",687,9.0,10.7,1433.5,-140,62, 427 | ,,"(2, 1) 428 | Ice giants",Uranus,86.8,"51,118",1271,8.7,17.2,2872.5,-195,27, 429 | ,,,Neptune,102,"49,528",1638,11.0,16.1,4495.1,-200,14, 430 | ___,"(1, 2) 431 | Dwarf planets",,Pluto,0.0146,"2,370",2095,0.7,153.3,5906.4,-225,5,Declassified as a planet in 2006. 432 | ``` 433 | ~~~ 434 | 435 | # Pantable as a library 436 | 437 | (experimental, API may change in the future) 438 | 439 | Documentation here is sparse, partly because the upstream (pandoc) may change the table AST again. See [Crazy ideas: table structure from upstream GitHub](https://github.com/jgm/pandoc-types/issues/86). 440 | 441 | See the API docs in . 442 | 443 | For example, looking at the source of `pantable` as a pandoc filter, in `codeblock_to_table.py`, you will see the main function doing the work is now 444 | 445 | ```python 446 | pan_table_str = ( 447 | PanCodeBlock 448 | .from_yaml_filter(options=options, data=data, element=element, doc=doc) 449 | .to_pantablestr() 450 | ) 451 | if pan_table_str.table_width is not None: 452 | pan_table_str.auto_width() 453 | return ( 454 | pan_table_str 455 | .to_pantable() 456 | .to_panflute_ast() 457 | ) 458 | ``` 459 | 460 | You can see another example from `table_to_codeblock.py` which is what `pantable2csv` and `pantable2csvx` called. 461 | 462 | Below is a diagram illustrating the API: 463 | 464 | ![Overview](docs/dot/pipeline-simple.svg) 465 | 466 | Solid arrows are lossless conversions. Dashed arrows are lossy. 467 | 468 | You can see the pantable internal structure, `PanTable` is one-one correspondence to the pandoc Table AST. Similarly for `PanCodeBlock`. 469 | 470 | It can then losslessly converts between PanTable and PanTableMarkdown, where everything in PanTableMarkdown is now markdown strings (whereas those in PanTable are panflute or panflute-like AST objects.) 471 | 472 | Lastly, it defines a one-one correspondence to PanCodeBlock with `fancy_table` syntax mentioned earlier. 473 | 474 | Below is the same diagram with the method names. You'd probably want to zoom into it to see it clearly. 475 | 476 | ![Detailed w/ methods](docs/dot/pipeline.svg) 477 | 478 | # Development 479 | 480 | To run all the tests run `tox`. GitHub Actions is used for CI too so if you fork this you can check if your commits passes there. 481 | 482 | # Related Filters 483 | 484 | (The table here is created in the beginning of pantable, which has since added more features. This is left here for historical reason and also as a credit to those before this.) 485 | 486 | The followings are pandoc filters written in Haskell that provide similar functionality. This filter is born after testing with theirs. 487 | 488 | - [baig/pandoc-csv2table: A Pandoc filter that renders CSV as Pandoc Markdown Tables.](https://github.com/baig/pandoc-csv2table) 489 | - [mb21/pandoc-placetable: Pandoc filter to include CSV data (from file or URL)](https://github.com/mb21/pandoc-placetable) 490 | - [sergiocorreia/panflute/csv-tables.py](https://github.com/sergiocorreia/panflute/blob/1ddcaba019b26f41f8c4f6f66a8c6540a9c5f31a/docs/source/csv-tables.py) 491 | 492 | ```table 493 | --- 494 | Caption: Comparison 495 | include: comparison.csv 496 | ... 497 | ``` 498 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /docs/comparison.csv: -------------------------------------------------------------------------------- 1 | ,pandoc-csv2table,pandoc-placetable,panflute example,pantable 2 | caption,caption,caption,title,caption 3 | aligns,aligns = LRCD,aligns = LRCD,,aligns = LRCD 4 | width,,"widths = ""0.5 0.2 0.3""",,"width: [0.5, 0.2, 0.3]" 5 | table-width,,,,table-width: 1.0 6 | header,header = yes | no,header = yes | no,has_header: True | False,header: True | False | yes | NO 7 | markdown,,inlinemarkdown,,markdown: True | False | yes | NO 8 | source,source,file,source,include 9 | others,type = simple | multiline | grid | pipe,,, 10 | ,,delimiter,, 11 | ,,quotechar,, 12 | ,,id (wrapped by div),, 13 | Notes,,,,width are auto-calculated when width is not specified -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | import sphinx_bootstrap_theme 2 | 3 | html_css_files = [ 4 | "https://cdn.jsdelivr.net/gh/ickc/markdown-latex-css/css/_table.min.css", 5 | "https://cdn.jsdelivr.net/gh/ickc/markdown-latex-css/fonts/fonts.min.css", 6 | ] 7 | 8 | extensions = [ 9 | "sphinx.ext.autodoc", 10 | "sphinx.ext.autosummary", 11 | "sphinx.ext.coverage", 12 | "sphinx.ext.doctest", 13 | "sphinx.ext.extlinks", 14 | "sphinx.ext.ifconfig", 15 | "sphinx.ext.napoleon", 16 | "sphinx.ext.todo", 17 | "sphinx.ext.viewcode", 18 | "sphinxcontrib.apidoc", 19 | ] 20 | source_suffix = ".rst" 21 | master_doc = "index" 22 | project = "pantable" 23 | year = "2016-2020" 24 | author = "Kolen Cheung" 25 | copyright = f"{year}, {author}" 26 | version = release = "0.14.2" 27 | 28 | pygments_style = "solarized-light" 29 | html_theme = "bootstrap" 30 | html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() 31 | html_theme_options = { 32 | "navbar_links": [("GitHub", "https://github.com/ickc/pantable/", True,)], 33 | "source_link_position": None, 34 | "bootswatch_theme": "readable", 35 | "bootstrap_version": "3", 36 | } 37 | 38 | html_use_smartypants = True 39 | html_last_updated_fmt = "%b %d, %Y" 40 | html_split_index = False 41 | html_short_title = f"{project}-{version}" 42 | 43 | napoleon_use_ivar = True 44 | napoleon_use_rtype = False 45 | napoleon_use_param = False 46 | 47 | # sphinxcontrib.apidoc 48 | apidoc_module_dir = '../src/pantable' 49 | apidoc_separate_modules = True 50 | apidoc_module_first = True 51 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/docs/dot: -------------------------------------------------------------------------------- 1 | ../dot -------------------------------------------------------------------------------- /docs/dot/makefile: -------------------------------------------------------------------------------- 1 | SHELL := /usr/bin/env bash 2 | 3 | # dot, neato, twopi, circo, fdp, sfdp, patchwork, osage 4 | DOT ?= dot 5 | 6 | FONT ?= "Latin Modern Roman" 7 | 8 | DOTs = $(wildcard *.dot) pipeline-simple.dot 9 | SVGs = $(patsubst %.dot,%.svg,$(DOTs)) 10 | 11 | svg: $(SVGs) 12 | 13 | # remove edge label 14 | pipeline-simple.dot: pipeline.dot 15 | printf "%s\n\n" "// This is auto-generated from \`$<\`. Do not edit this file directly." > $@ 16 | sed -E -e 's/->(.*)label="[^"]+",? ?/->\1/' -e 's/ \[\]//' -e 's/rankdir=LR/rankdir=TB/' $< >> $@ 17 | 18 | %.svg: %.dot 19 | $(DOT) -T svg $< -o $*.svg -Nfontname=$(FONT) -Efontname=$(FONT) -Gfontname=$(FONT) 20 | svgcleaner --multipass $*.svg $*.svg 21 | 22 | clean: 23 | rm -f $(SVGs) 24 | 25 | print-%: 26 | $(info $* = $($*)) 27 | -------------------------------------------------------------------------------- /docs/dot/pipeline-simple.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | %3 11 | 12 | 13 | 14 | Table 15 | 16 | AST-Table 17 | 18 | 19 | 20 | PanTable 21 | 22 | PanTable 23 | 24 | 25 | 26 | Table->PanTable 27 | 28 | 29 | 30 | 31 | 32 | CodeBlock 33 | 34 | AST-CodeBlock 35 | 36 | 37 | 38 | PanCodeBlock 39 | 40 | PanCodeBlock 41 | 42 | 43 | 44 | CodeBlock->PanCodeBlock 45 | 46 | 47 | 48 | 49 | 50 | PanTableStr 51 | 52 | PanTableStr 53 | 54 | 55 | 56 | PanTableStr->PanTable 57 | 58 | 59 | 60 | 61 | 62 | PanTableStr->PanCodeBlock 63 | 64 | 65 | 66 | 67 | 68 | PanTableMarkdown 69 | 70 | PanTableMarkdown 71 | 72 | 73 | 74 | PanTableMarkdown->PanTable 75 | 76 | 77 | 78 | 79 | 80 | PanTableMarkdown->PanCodeBlock 81 | 82 | 83 | 84 | 85 | 86 | PanTableMarkdown->PanCodeBlock 87 | 88 | 89 | 90 | 91 | 92 | PanTableText 93 | 94 | PanTableText 95 | 96 | 97 | 98 | PanTableText->PanTable 99 | 100 | 101 | 102 | 103 | 104 | PanTableText->PanCodeBlock 105 | 106 | 107 | 108 | 109 | 110 | PanTable->Table 111 | 112 | 113 | 114 | 115 | 116 | PanTable->PanTableStr 117 | 118 | 119 | 120 | 121 | 122 | PanTable->PanTableMarkdown 123 | 124 | 125 | 126 | 127 | 128 | PanCodeBlock->CodeBlock 129 | 130 | 131 | 132 | 133 | 134 | PanCodeBlock->PanTableMarkdown 135 | 136 | 137 | 138 | 139 | 140 | PanCodeBlock->PanTableText 141 | 142 | 143 | 144 | 145 | 146 | -------------------------------------------------------------------------------- /docs/dot/pipeline.dot: -------------------------------------------------------------------------------- 1 | digraph { 2 | rankdir=LR 3 | 4 | graph [fontname = "Latin Modern Roman"] 5 | node [fontname = "Latin Modern Roman" shape=box] 6 | edge [fontname = "Latin Modern Roman" penwidth=0.6] 7 | 8 | Table [label="AST-Table"] 9 | CodeBlock [label="AST-CodeBlock"] 10 | 11 | { 12 | rank=same 13 | PanTableStr 14 | PanTableMarkdown 15 | PanTableText 16 | } 17 | 18 | // PanTable methods 19 | Table -> PanTable [label="PanTable.from_panflute_ast"] 20 | PanTable -> Table [label="PanTable.to_panflute_ast" weight=1000000] 21 | PanTable -> PanTableMarkdown [label="PanTable.to_pantablemarkdown" weight=1000000] 22 | PanTable -> PanTableStr [label="PanTable.to_pantablestr" style=dashed] 23 | 24 | // PanTableMarkdown methods 25 | PanTableMarkdown -> PanTable [label="PanTableMarkdown.to_pantable"] 26 | PanTableMarkdown -> PanCodeBlock [label="PanTableMarkdown.to_pancodeblock(fancy_table=True)"] 27 | PanTableMarkdown -> PanCodeBlock [label="PanTableMarkdown.to_pancodeblock", style=dashed] 28 | 29 | // PanTableStr methods 30 | PanTableStr -> PanTable [label="PanTableStr.to_pantable"] 31 | PanTableStr -> PanCodeBlock [label="PanTableStr.to_pancodeblock", style=dashed] 32 | 33 | // PanTableText methods 34 | PanTableText -> PanTable [label="PanTableText.to_pantable"] 35 | PanTableText -> PanCodeBlock [label="PanTableText.to_pancodeblock"] 36 | 37 | // PanCodeBlock methods 38 | PanCodeBlock -> CodeBlock [label="PanCodeBlock.to_panflute_ast"] 39 | PanCodeBlock -> PanTableMarkdown [label="PanCodeBlock.to_pantablestr w/ PanTableOption.markdown=True" weight=1000000] 40 | PanCodeBlock -> PanTableText [label="PanCodeBlock.to_pantablestr w/ PanTableOption.markdown=False"] 41 | CodeBlock -> PanCodeBlock [label="PanCodeBlock.from_yaml_filter" weight=1000000] 42 | } 43 | -------------------------------------------------------------------------------- /docs/dot/pipeline.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | %3 11 | 12 | 13 | 14 | Table 15 | 16 | AST-Table 17 | 18 | 19 | 20 | PanTable 21 | 22 | PanTable 23 | 24 | 25 | 26 | Table->PanTable 27 | 28 | 29 | PanTable.from_panflute_ast 30 | 31 | 32 | 33 | CodeBlock 34 | 35 | AST-CodeBlock 36 | 37 | 38 | 39 | PanCodeBlock 40 | 41 | PanCodeBlock 42 | 43 | 44 | 45 | CodeBlock->PanCodeBlock 46 | 47 | 48 | PanCodeBlock.from_yaml_filter 49 | 50 | 51 | 52 | PanTableStr 53 | 54 | PanTableStr 55 | 56 | 57 | 58 | PanTableStr->PanTable 59 | 60 | 61 | PanTableStr.to_pantable 62 | 63 | 64 | 65 | PanTableStr->PanCodeBlock 66 | 67 | 68 | PanTableStr.to_pancodeblock 69 | 70 | 71 | 72 | PanTableMarkdown 73 | 74 | PanTableMarkdown 75 | 76 | 77 | 78 | PanTableMarkdown->PanTable 79 | 80 | 81 | PanTableMarkdown.to_pantable 82 | 83 | 84 | 85 | PanTableMarkdown->PanCodeBlock 86 | 87 | 88 | PanTableMarkdown.to_pancodeblock(fancy_table=True) 89 | 90 | 91 | 92 | PanTableMarkdown->PanCodeBlock 93 | 94 | 95 | PanTableMarkdown.to_pancodeblock 96 | 97 | 98 | 99 | PanTableText 100 | 101 | PanTableText 102 | 103 | 104 | 105 | PanTableText->PanTable 106 | 107 | 108 | PanTableText.to_pantable 109 | 110 | 111 | 112 | PanTableText->PanCodeBlock 113 | 114 | 115 | PanTableText.to_pancodeblock 116 | 117 | 118 | 119 | PanTable->Table 120 | 121 | 122 | PanTable.to_panflute_ast 123 | 124 | 125 | 126 | PanTable->PanTableStr 127 | 128 | 129 | PanTable.to_pantablestr 130 | 131 | 132 | 133 | PanTable->PanTableMarkdown 134 | 135 | 136 | PanTable.to_pantablemarkdown 137 | 138 | 139 | 140 | PanCodeBlock->CodeBlock 141 | 142 | 143 | PanCodeBlock.to_panflute_ast 144 | 145 | 146 | 147 | PanCodeBlock->PanTableMarkdown 148 | 149 | 150 | PanCodeBlock.to_pantablestr w/ PanTableOption.markdown=True 151 | 152 | 153 | 154 | PanCodeBlock->PanTableText 155 | 156 | 157 | PanCodeBlock.to_pantablestr w/ PanTableOption.markdown=False 158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /docs/example.csv: -------------------------------------------------------------------------------- 1 | First row,defaulted to be header row,can be disabled 2 | 1,cell can contain **markdown**,"It can be aribrary block element: - following standard markdown syntax - like this" 3 | 2,"Any markdown syntax, e.g.",$$E = mc^2$$ -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Contents 3 | ======== 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Contents: 8 | 9 | readme 10 | installation 11 | usage 12 | contributing 13 | changelog 14 | api/modules 15 | 16 | Indices and tables 17 | ================== 18 | 19 | * :ref:`genindex` 20 | * :ref:`modindex` 21 | * :ref:`search` 22 | 23 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | At the command line:: 6 | 7 | pip install pantable 8 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | builtin 2 | builtins 3 | classmethod 4 | staticmethod 5 | classmethods 6 | staticmethods 7 | args 8 | kwargs 9 | callstack 10 | Changelog 11 | Indices 12 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | Usage 3 | ===== 4 | 5 | To use pantable in a project:: 6 | 7 | import pantable 8 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | # this should be very similar to https://github.com/conda-forge/pantable-feedstock/blob/master/recipe/meta.yaml 2 | # run the following command to create a conda environment that is suitable for testing pantable 3 | # conda env create -f environment.yml 4 | # conda activate pantable 5 | # python -m ipykernel install --user --name pantable --display-name pantable 6 | # and then run this to install pantable in editable mode 7 | # make editable 8 | # update using 9 | # conda env update --name pantable --file environment.yml --prune 10 | name: pantable 11 | channels: 12 | - conda-forge 13 | dependencies: 14 | # host: 15 | - pip 16 | - python >=3.8 # for cached property 17 | - poetry >=1,<2 18 | # run: 19 | - numpy >=1.16,<2 20 | - panflute >=2.1.3,<3 21 | - pyyaml >=5,<7 22 | # run_constrained: 23 | - "pandoc >=2.11.2,<2.18" 24 | - coloredlogs >=14,<16 25 | - tabulate >=0.8,<0.9 26 | - yamlloader >=1,<2 27 | # tests: 28 | - coverage>=6.3,<7 29 | - coveralls 30 | - flake8 31 | - pytest 32 | - pytest-parallel >=0.1.1,<0.2 33 | # docs: 34 | - sphinx 35 | - sphinx_bootstrap_theme 36 | - sphinxcontrib-apidoc 37 | # dev: 38 | - tox 39 | - data-science-types 40 | - ipykernel 41 | - mypy 42 | - bandit 43 | - bump2version 44 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | SHELL = /usr/bin/env bash 2 | 3 | PANTABLELOGLEVEL ?= DEBUG 4 | python ?= python 5 | _python = PANTABLELOGLEVEL=$(PANTABLELOGLEVEL) $(python) 6 | pandoc ?= pandoc 7 | _pandoc = PANTABLELOGLEVEL=$(PANTABLELOGLEVEL) $(pandoc) 8 | PYTESTARGS ?= --workers auto 9 | # for bump2version, valid options are: major, minor, patch 10 | PART ?= patch 11 | 12 | pandocArgs = --toc -M date="`date "+%B %e, %Y"`" --filter=pantable --wrap=none 13 | 14 | RSTs = CHANGELOG.rst README.rst 15 | 16 | # Main Targets ################################################################# 17 | 18 | .PHONY: test docs-all docs html epub files dot clean Clean 19 | 20 | all: dot files editable 21 | $(MAKE) test docs-all 22 | 23 | test: 24 | $(_python) \ 25 | -m coverage run \ 26 | -m pytest -vv $(PYTESTARGS) tests 27 | coverage: test 28 | coverage combine 29 | coverage report 30 | coverage html 31 | 32 | docs-all: docs html epub 33 | docs: $(RSTs) 34 | html: dist/docs/ 35 | epub: dist/epub/pantable.epub 36 | 37 | files: 38 | cd tests/files; $(MAKE) 39 | dot: 40 | cd docs/dot; $(MAKE) 41 | 42 | clean: 43 | rm -f .coverage* docs/pantable*.rst docs/modules.rst docs/setup.rst setup.py 44 | rm -rf htmlcov pantable.egg-info .cache .idea dist docs/_build \ 45 | docs/_static docs/_templates .ipynb_checkpoints .mypy_cache \ 46 | .pytest_cache .tox 47 | find . -type f -name "*.py[co]" -delete \ 48 | -or -type d -name "__pycache__" -delete 49 | Clean: clean 50 | rm -f $(RSTs) 51 | 52 | # maintenance ################################################################## 53 | 54 | .PHONY: pypi pypiManual pep8 flake8 pylint 55 | # Deploy to PyPI 56 | ## by CI, properly git tagged 57 | pypi: 58 | git push origin v0.14.2 59 | ## Manually 60 | pypiManual: 61 | rm -rf dist 62 | tox -e check 63 | poetry build 64 | twine upload dist/* 65 | 66 | # check python styles 67 | pep8: 68 | pycodestyle . --ignore=E501 69 | flake8: 70 | flake8 . --ignore=E501 71 | pylint: 72 | pylint pantable 73 | 74 | print-%: 75 | $(info $* = $($*)) 76 | 77 | # docs ######################################################################### 78 | 79 | README.rst: docs/README.md 80 | printf \ 81 | "%s\n\n" \ 82 | ".. This is auto-generated from \`$<\`. Do not edit this file directly." \ 83 | > $@ 84 | cd $(> ../$@ 87 | 88 | %.rst: %.md 89 | printf \ 90 | "%s\n\n" \ 91 | ".. This is auto-generated from \`$<\`. Do not edit this file directly." \ 92 | > $@ 93 | $(_pandoc) $(pandocArgs) $< -s -t rst >> $@ 94 | 95 | dist/docs/: 96 | tox -e docs 97 | # depends on docs as the api doc is built there 98 | # didn't put this in tox as we should build this once every release 99 | # TODO: consider put this in tox and automate it in GH Actions 100 | dist/epub/pantable.epub: docs 101 | sphinx-build -E -b epub docs dist/epub 102 | # the badges and dots has svg files that LaTeX complains about 103 | # dist/pantable.pdf: docs 104 | # sphinx-build -E -b latex docs dist/pdf 105 | # cd dist/pdf; make 106 | # mv dist/pdf/pantable.pdf dist 107 | 108 | # poetry ####################################################################### 109 | 110 | setup.py: 111 | poetry build 112 | cd dist; tar -xf pantable-0.14.2.tar.gz pantable-0.14.2/setup.py 113 | mv dist/pantable-0.14.2/setup.py . 114 | rm -rf dist/pantable-0.14.2 115 | 116 | # since poetry doesn't support editable, we can build and extract the setup.py, 117 | # temporary remove pyproject.toml and ask pip to install from setup.py instead. 118 | editable: setup.py 119 | mv pyproject.toml .pyproject.toml 120 | $(_python) -m pip install --no-dependencies -e . 121 | mv .pyproject.toml pyproject.toml 122 | 123 | # releasing #################################################################### 124 | 125 | bump: 126 | bump2version $(PART) 127 | git push --follow-tags 128 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["poetry_core>=1.0.0"] 3 | build-backend = "poetry.core.masonry.api" 4 | 5 | [tool.poetry] 6 | name = "pantable" 7 | version = "0.14.2" 8 | description = "A Python library for writing pandoc filters for tables with batteries included." 9 | license = "BSD-3-Clause" 10 | keywords = [ 11 | "pandoc", 12 | "pandocfilters", 13 | "panflute", 14 | "markdown", 15 | "latex", 16 | "html", 17 | "csv", 18 | "table", 19 | ] 20 | classifiers = [ 21 | "Development Status :: 4 - Beta", 22 | "Environment :: Console", 23 | "Intended Audience :: End Users/Desktop", 24 | "Intended Audience :: Developers", 25 | "Topic :: Software Development :: Build Tools", 26 | "Topic :: Text Processing :: Filters", 27 | "License :: OSI Approved :: BSD License", 28 | "Operating System :: Unix", 29 | "Operating System :: POSIX", 30 | "Operating System :: Microsoft :: Windows", 31 | "Programming Language :: Python", 32 | "Programming Language :: Python :: 3", 33 | "Programming Language :: Python :: 3.7", 34 | "Programming Language :: Python :: 3.8", 35 | "Programming Language :: Python :: 3.9", 36 | "Programming Language :: Python :: Implementation :: CPython", 37 | "Programming Language :: Python :: Implementation :: PyPy", 38 | "Topic :: Utilities", 39 | ] 40 | homepage = "https://github.com/ickc/pantable" 41 | repository = "https://github.com/ickc/pantable" 42 | documentation = "https://ickc.github.io/pantable" 43 | authors = ["Kolen Cheung "] 44 | readme = "README.rst" 45 | packages = [ 46 | { include = "pantable", from = "src" }, 47 | ] 48 | include = [ 49 | 'CHANGELOG.rst', 50 | 'CONTRIBUTING.rst', 51 | 'LICENSE', 52 | 'README.rst', 53 | ] 54 | 55 | [tool.poetry.dependencies] 56 | python = ">=3.7" 57 | panflute = "^2.1.3" 58 | pyyaml = ">=5,<7" 59 | numpy = "^1.16" 60 | "backports.cached-property" = {python = "<3.8", version = "^1"} 61 | 62 | # extras 63 | coloredlogs = {optional = true, version = ">=14,<16"} 64 | tabulate = {optional = true, version = "^0.8"} 65 | yamlloader = {optional = true, version = "^1"} 66 | 67 | # tests 68 | coverage = { optional = true, version = "^6.3" } 69 | coveralls = {optional = true, version = "*"} 70 | flake8 = {optional = true, version = "*"} 71 | pytest = {optional = true, version = "*"} 72 | pytest-parallel = {optional = true, version = "^0.1.1"} 73 | 74 | # docs: sync this with tox.testenv.docs below 75 | sphinx = {optional = true, version = ">=3,<5"} 76 | sphinx-bootstrap-theme = {optional = true, version = "*"} 77 | sphinxcontrib-apidoc = {optional = true, version = "*"} 78 | 79 | [tool.poetry.dev-dependencies] 80 | tox = "*" 81 | data-science-types = "*" 82 | ipykernel = "*" 83 | mypy = "*" 84 | bandit = "*" 85 | bump2version = "*" 86 | 87 | [tool.poetry.extras] 88 | extras = [ 89 | "coloredlogs", 90 | "tabulate", 91 | "yamlloader", 92 | ] 93 | tests = [ 94 | "coverage", 95 | "coveralls", 96 | "flake8", 97 | "pytest", 98 | "pytest-parallel", 99 | ] 100 | docs = [ 101 | "sphinx", 102 | "sphinx-bootstrap-theme", 103 | "sphinxcontrib-apidoc", 104 | ] 105 | 106 | [tool.poetry.scripts] 107 | pantable = 'pantable.cli.pantable:main' 108 | pantable2csv = 'pantable.cli.pantable2csv:main' 109 | pantable2csvx = 'pantable.cli.pantable2csvx:main' 110 | 111 | [tool.coverage.paths] 112 | source = [ 113 | 'src', 114 | '*/site-packages', 115 | ] 116 | 117 | [tool.coverage.run] 118 | branch = true 119 | source = [ 120 | 'src', 121 | 'tests', 122 | ] 123 | parallel = true 124 | relative_files = true 125 | 126 | [tool.coverage.report] 127 | show_missing = true 128 | precision = 2 129 | 130 | [tool.pytest.ini_options] 131 | 132 | python_files = [ 133 | 'test_*.py', 134 | '*_test.py', 135 | 'tests.py', 136 | ] 137 | 138 | addopts = [ 139 | '-ra', 140 | '--strict-markers', 141 | '--doctest-modules', 142 | '--doctest-glob=\*.rst', 143 | '--tb=short', 144 | ] 145 | 146 | testpaths = [ 147 | 'tests', 148 | ] 149 | 150 | [tool.isort] 151 | line_length = 120 152 | known_first_party = 'pantable' 153 | default_section = 'THIRDPARTY' 154 | forced_separate = 'test_pantable' 155 | skip = [ 156 | '.tox', 157 | '.eggs', 158 | 'ci/templates', 159 | 'build', 160 | 'dist', 161 | ] 162 | 163 | [tool.tox] 164 | legacy_tox_ini = ''' 165 | [testenv:bootstrap] 166 | deps = 167 | jinja2 168 | matrix 169 | tox 170 | skip_install = true 171 | commands = 172 | python ci/bootstrap.py --no-env 173 | passenv = 174 | * 175 | ; a generative tox configuration, see: https://tox.readthedocs.io/en/latest/config.html#generative-envlist 176 | 177 | [tox] 178 | isolated_build = True 179 | envlist = 180 | clean, 181 | check, 182 | docs, 183 | {py37,py38,py39,pypy3}, 184 | report 185 | ignore_basepython_conflict = true 186 | 187 | [gh-actions] 188 | python = 189 | 3.7: py37 190 | 3.8: py38, mypy 191 | 3.9: py39 192 | pypy3: pypy3 193 | 194 | [testenv] 195 | basepython = 196 | pypy: {env:TOXPYTHON:pypy} 197 | pypy3: {env:TOXPYTHON:pypy3} 198 | py37: {env:TOXPYTHON:python3.7} 199 | py38: {env:TOXPYTHON:python3.8} 200 | py39: {env:TOXPYTHON:python3.9} 201 | {bootstrap,clean,check,report,docs,codecov,coveralls}: {env:TOXPYTHON:python3} 202 | .package: python3 203 | setenv = 204 | # for coverage to work properly 205 | PYTHONPATH={toxinidir}/src 206 | PYTHONUNBUFFERED=yes 207 | passenv = 208 | * 209 | usedevelop = false 210 | deps = 211 | pytest 212 | pytest-cov 213 | yamlloader 214 | commands = 215 | {posargs:pytest --cov --cov-branch --cov-report=term-missing --cov-report=xml -vv tests} 216 | 217 | [testenv:check] 218 | deps = 219 | docutils 220 | check-manifest 221 | flake8 222 | readme-renderer 223 | pygments 224 | isort 225 | skip_install = true 226 | commands = 227 | flake8 --ignore F821,E501 --max-line-length 140 --exclude '.tox,.eggs,ci/templates,build,dist,setup.py' 228 | isort --verbose --check-only --diff --filter-files . 229 | 230 | [testenv:docs] 231 | usedevelop = false 232 | deps = 233 | sphinx >=3.3,<5 234 | sphinx-bootstrap-theme 235 | sphinxcontrib-apidoc 236 | commands = 237 | sphinx-build {posargs:-E} -b dirhtml docs dist/docs 238 | sphinx-build -b linkcheck docs dist/docs 239 | 240 | [testenv:coveralls] 241 | deps = 242 | coveralls 243 | skip_install = true 244 | commands = 245 | coveralls [] 246 | 247 | [testenv:codecov] 248 | deps = 249 | codecov 250 | skip_install = true 251 | commands = 252 | codecov [] 253 | 254 | [testenv:report] 255 | deps = 256 | coverage 257 | toml 258 | skip_install = true 259 | commands = 260 | coverage report 261 | coverage html 262 | 263 | [testenv:clean] 264 | commands = coverage erase 265 | skip_install = true 266 | deps = 267 | coverage 268 | toml 269 | ''' 270 | -------------------------------------------------------------------------------- /src/pantable/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | 5 | try: 6 | from coloredlogs import ColoredFormatter as Formatter 7 | except ImportError: 8 | from logging import Formatter 9 | 10 | __version__ = '0.14.2' 11 | PY37 = sys.version_info.minor == 7 12 | 13 | logger = logging.getLogger(__name__) 14 | handler = logging.StreamHandler() 15 | logger.addHandler(handler) 16 | handler.setFormatter(Formatter('%(name)s %(levelname)s %(message)s')) 17 | try: 18 | level = os.environ.get('PANTABLELOGLEVEL', logging.WARNING) 19 | logger.setLevel(level=level) 20 | except ValueError: 21 | logger.setLevel(level=logging.WARNING) 22 | logger.error(f'Unknown PANTABLELOGLEVEL {level}, set to default WARNING.') 23 | -------------------------------------------------------------------------------- /src/pantable/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ickc/pantable/ba26525e3e9c6ddab6236276ec9a9ac3508e31f5/src/pantable/cli/__init__.py -------------------------------------------------------------------------------- /src/pantable/cli/pantable.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from panflute.io import run_filter 6 | from panflute.tools import yaml_filter 7 | 8 | from ..codeblock_to_table import codeblock_to_table 9 | 10 | if TYPE_CHECKING: 11 | from panflute.elements import Doc 12 | 13 | 14 | def main(doc: Doc | None = None): 15 | """a pandoc filter converting csv table in code block 16 | 17 | Fenced code block with class table will be parsed using 18 | panflute.yaml_filter with the fuction 19 | :func:`pantable.codeblock_to_table.codeblock_to_table` 20 | """ 21 | return run_filter( 22 | yaml_filter, 23 | doc=doc, 24 | tag="table", 25 | function=codeblock_to_table, 26 | strict_yaml=True, 27 | ) 28 | 29 | 30 | if __name__ == "__main__": 31 | main() 32 | -------------------------------------------------------------------------------- /src/pantable/cli/pantable2csv.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from panflute.io import run_filter 6 | 7 | from ..table_to_codeblock import table_to_codeblock 8 | 9 | if TYPE_CHECKING: 10 | from panflute.elements import Doc 11 | 12 | 13 | def main(doc: Doc | None = None): 14 | """Covert all tables to CSV table format defined in pantable 15 | 16 | - in code-block with class table 17 | - metadata in YAML 18 | - table in CSV 19 | """ 20 | return run_filter(table_to_codeblock, doc=doc) 21 | 22 | 23 | if __name__ == "__main__": 24 | main() 25 | -------------------------------------------------------------------------------- /src/pantable/cli/pantable2csvx.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from panflute.io import run_filter 6 | 7 | from ..table_to_codeblock import table_to_codeblock 8 | 9 | if TYPE_CHECKING: 10 | from panflute.elements import Doc 11 | 12 | 13 | def main(doc: Doc | None = None): 14 | """Covert all tables to CSV table format defined in pantable 15 | 16 | - in code-block with class table 17 | - metadata in YAML 18 | - table in CSV 19 | """ 20 | return run_filter(table_to_codeblock, doc=doc, fancy_table=True) 21 | 22 | 23 | if __name__ == "__main__": 24 | main() 25 | -------------------------------------------------------------------------------- /src/pantable/codeblock_to_table.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from logging import getLogger 4 | from typing import TYPE_CHECKING 5 | 6 | from .ast import PanCodeBlock 7 | from .util import EmptyTableError 8 | 9 | if TYPE_CHECKING: 10 | from typing import Optional, Union 11 | 12 | from panflute.elements import CodeBlock, Doc 13 | from panflute.table_elements import Table 14 | 15 | logger = getLogger('pantable') 16 | 17 | 18 | def codeblock_to_table( 19 | options: Optional[dict] = None, 20 | data: str = '', 21 | element: Optional[CodeBlock] = None, 22 | doc: Optional[Doc] = None, 23 | ) -> Union[Table, list, None]: 24 | try: 25 | pan_table_str = ( 26 | PanCodeBlock 27 | .from_yaml_filter(options=options, data=data, element=element, doc=doc) 28 | .to_pantablestr() 29 | ) 30 | if pan_table_str.table_width is not None: 31 | pan_table_str.auto_width() 32 | return ( 33 | pan_table_str 34 | .to_pantable() 35 | .to_panflute_ast() 36 | ) 37 | # delete element if table is empty (by returning []) 38 | # element unchanged if include is invalid (by returning None) 39 | except FileNotFoundError as e: 40 | logger.error(f'{e} Codeblock shown as is.') 41 | return None 42 | except EmptyTableError: 43 | logger.warning("table is empty. Deleted.") 44 | # [] means delete the current element 45 | return [] 46 | except ImportError as e: 47 | logger.error(f'Some modules cannot be imported, Codeblock shown as is: {e}') 48 | return None 49 | -------------------------------------------------------------------------------- /src/pantable/io.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import csv 4 | import io 5 | from logging import getLogger 6 | from pathlib import Path 7 | from typing import TYPE_CHECKING 8 | 9 | import numpy as np 10 | 11 | from .util import EmptyTableError 12 | 13 | if TYPE_CHECKING: 14 | from typing import List 15 | 16 | from .ast import PanTableOption 17 | 18 | logger = getLogger('pantable') 19 | 20 | 21 | def load_csv( 22 | data: str, 23 | options: PanTableOption, 24 | ) -> List[List[str]]: 25 | '''loading CSV table 26 | 27 | Note that this can emit EmptyTableError, FileNotFoundError 28 | ''' 29 | include = options.include 30 | # TODO: PY37 31 | # encoding = encoding_ if (encoding_ := options.include_encoding) else None 32 | encoding = options.include_encoding 33 | # default include_encoding is '' 34 | # default encoding below is None 35 | if not encoding: 36 | encoding = None 37 | try: 38 | with ( 39 | open(include, encoding=encoding, newline='') 40 | ) if include else ( 41 | io.StringIO(data, newline='') 42 | ) as f: 43 | table_list = list(csv.reader(f, **options.csv_kwargs)) 44 | if table_list: 45 | for row in table_list: 46 | if row: 47 | for i in row: 48 | if i.strip(): 49 | return table_list 50 | raise EmptyTableError 51 | except FileNotFoundError: 52 | raise FileNotFoundError(f'include path {include} not found.') 53 | 54 | 55 | def load_csv_array( 56 | data: str, 57 | options: PanTableOption, 58 | ) -> np.ndarray[np.str_]: 59 | '''loading CSV table in `numpy.ndarray` 60 | 61 | Note that this can emit EmptyTableError, FileNotFoundError 62 | ''' 63 | table_list = load_csv(data, options) 64 | m = len(table_list) 65 | n = max(len(row) for row in table_list) 66 | res = np.full((m, n), '', dtype=np.object_) 67 | for i, row in enumerate(table_list): 68 | for j, cell in enumerate(row): 69 | res[i, j] = cell 70 | return res 71 | 72 | 73 | def dump_csv( 74 | data: np.ndarray[np.str_], 75 | options: PanTableOption, 76 | ) -> str: 77 | '''dump data as CSV string 78 | ''' 79 | with io.StringIO(newline='') as f: 80 | writer = csv.writer(f, **options.csv_kwargs) 81 | writer.writerows(data) 82 | return f.getvalue() 83 | 84 | 85 | def dump_csv_io( 86 | data: np.ndarray[np.str_], 87 | options: PanTableOption, 88 | ) -> str: 89 | '''dump data as CSV 90 | 91 | it will mutate options.include if it is an invalid write path. 92 | ''' 93 | _include = options.include 94 | 95 | text = dump_csv(data, options) 96 | 97 | if _include: 98 | try: 99 | include = Path(_include) 100 | include.parent.mkdir(parents=True, exist_ok=True) 101 | with open(include, 'x', encoding=options.include_encoding, newline='') as f: 102 | f.write(text) 103 | return '' 104 | except (PermissionError, FileExistsError): 105 | logger.error(f'Data cannot be written to file {options.include}, Overriding include path to empty...') 106 | options.include = '' 107 | return text 108 | -------------------------------------------------------------------------------- /src/pantable/table_to_codeblock.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from panflute.elements import Table 6 | 7 | if TYPE_CHECKING: 8 | from typing import Optional 9 | 10 | from panflute.elements import Doc 11 | 12 | from .ast import PanTable 13 | 14 | 15 | def table_to_codeblock( 16 | element: Optional[Table] = None, 17 | doc: Optional[Doc] = None, 18 | format: str = 'csv', 19 | fancy_table: bool = False, 20 | include: str = '', 21 | csv_kwargs: Optional[dict] = None, 22 | ) -> Optional[PanTable]: 23 | """convert Table element and to csv table in code-block with class "table" in panflute AST""" 24 | if type(element) is Table: 25 | return ( 26 | PanTable 27 | .from_panflute_ast(element) 28 | .to_pantablemarkdown() 29 | # no options chosen here to match historical behavior 30 | .to_pancodeblock( 31 | format=format, 32 | fancy_table=fancy_table, 33 | include=include, 34 | csv_kwargs=csv_kwargs, 35 | ) 36 | .to_panflute_ast() 37 | ) 38 | return None 39 | -------------------------------------------------------------------------------- /src/pantable/util.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from functools import partial 4 | from logging import getLogger 5 | from typing import TYPE_CHECKING, Any, _SpecialForm, get_type_hints 6 | 7 | from . import PY37 8 | 9 | if not PY37: 10 | from typing import get_args, get_origin 11 | 12 | import numpy as np 13 | from panflute.elements import ListContainer, Para, Str 14 | from panflute.tools import convert_text, run_pandoc, yaml_filter 15 | 16 | if TYPE_CHECKING: 17 | from typing import Callable, Dict, Generator, Iterable, Iterator, List, Optional, Tuple 18 | 19 | from panflute.elements import Element 20 | 21 | logger = getLogger('pantable') 22 | 23 | 24 | class EmptyTableError(Exception): 25 | pass 26 | 27 | 28 | def convert_texts( 29 | texts: Iterable, 30 | input_format: str = 'markdown', 31 | output_format: str = 'panflute', 32 | standalone: bool = False, 33 | extra_args: Optional[List[str]] = None, 34 | ) -> List[list]: 35 | '''run convert_text on list of text''' 36 | try: 37 | from map_parallel import map_parallel 38 | 39 | _map_parallel = partial(map_parallel, mode='multithreading') 40 | except ImportError: 41 | logger.warning('Consider `pip install map_parallel` to speed up `convert_texts`.') 42 | 43 | def _map_parallel(f, arg): 44 | return list(map(f, arg)) 45 | 46 | _convert_text = partial( 47 | convert_text, 48 | input_format=input_format, 49 | output_format=output_format, 50 | standalone=standalone, 51 | extra_args=extra_args, 52 | ) 53 | return _map_parallel(_convert_text, texts) 54 | 55 | 56 | def iter_convert_texts_markdown_to_panflute( 57 | texts: Iterable[str], 58 | extra_args: Optional[List[str]] = None, 59 | ) -> Iterator[ListContainer]: 60 | '''a faster, specialized convert_texts 61 | ''' 62 | # put each text in a Div together 63 | text = '\n\n'.join( 64 | ( 65 | f'''::: PanTableDiv ::: 66 | 67 | {text} 68 | 69 | :::''' 70 | for text in texts 71 | ) 72 | ) 73 | pf = convert_text(text, input_format='markdown', output_format='panflute', extra_args=extra_args) 74 | return (elem.content for elem in pf) 75 | 76 | 77 | def iter_convert_texts_panflute_to_markdown( 78 | elems: Iterable[ListContainer], 79 | extra_args: Optional[List[str]] = None, 80 | seperator: str = np.random.randint(65, 91, size=256, dtype=np.uint8).view('S256')[0].decode(), 81 | ) -> Iterator[str]: 82 | '''a faster, specialized convert_texts 83 | 84 | :param list elems: must be list of ListContainer of Block. 85 | This is more restrictive than convert_texts which can also accept list of Block 86 | :param str seperator: a string for seperator in the temporary markdown output 87 | ''' 88 | def iter_seperator(elems: List[ListContainer], inserter: Para): 89 | '''insert between every element in a ListContainer''' 90 | for elem in elems: 91 | for i in elem: 92 | yield i 93 | yield inserter 94 | 95 | def iter_split_by_seperator(text: str, seperator: str) -> Iterator[str]: 96 | '''split the text into list by the seperator 97 | ''' 98 | temp = [] 99 | for line in text.splitlines(): 100 | if line != seperator: 101 | temp.append(line) 102 | else: 103 | res = '\n'.join(temp).strip() 104 | # reset for next yield 105 | temp = [] 106 | yield res 107 | 108 | inserter = Para(Str(seperator)) 109 | 110 | elems_inserted = ListContainer(*iter_seperator(elems, inserter)) 111 | # reference-location=block for footnotes, see issue #58 112 | texts_converted = convert_text(elems_inserted, input_format='panflute', output_format='markdown', extra_args=['--reference-location=block']) 113 | return iter_split_by_seperator(texts_converted, seperator) 114 | 115 | 116 | convert_texts_func: Dict[Tuple[str, str], Callable[[Iterable, Optional[List[str]]], Iterator]] = { 117 | ('markdown', 'panflute'): ( 118 | lambda *args, **kwargs: 119 | # this is just to convert returned value from 120 | # Iterator[ListContainer] to Iterator[list] 121 | # which is what convert_texts does 122 | map(list, iter_convert_texts_markdown_to_panflute(*args, **kwargs)) 123 | ), 124 | ('panflute', 'markdown'): iter_convert_texts_panflute_to_markdown, 125 | } 126 | 127 | 128 | def convert_texts_fast( 129 | texts: Iterable, 130 | input_format: str = 'markdown', 131 | output_format: str = 'panflute', 132 | extra_args: Optional[List[str]] = None, 133 | ) -> List[list]: 134 | '''a faster, specialized convert_texts 135 | 136 | should have identical result from convert_texts 137 | ''' 138 | try: 139 | return list( 140 | convert_texts_func[ 141 | (input_format, output_format) 142 | ]( 143 | texts, 144 | extra_args, 145 | ) 146 | ) 147 | except KeyError: 148 | logger.warning(f'Unsupported input/output format pair: {input_format}, {output_format}. Doing it slowly...') 149 | return convert_texts( 150 | texts, 151 | input_format, 152 | output_format, 153 | extra_args=extra_args, 154 | ) 155 | 156 | 157 | def eq_panflute_elem(elem1: Element, elem2: Element) -> bool: 158 | return repr(elem1) == repr(elem2) 159 | 160 | 161 | def eq_panflute_elems(elems1: List[Element], elems2: List[Element]) -> bool: 162 | if not len(elems1) == len(elems2): 163 | return False 164 | for elem1, elem2 in zip(elems1, elems2): 165 | if not eq_panflute_elem(elem1, elem2): 166 | return False 167 | return True 168 | 169 | 170 | def parse_markdown_codeblock(text: str) -> dict: 171 | '''parse markdown CodeBlock just as `panflute.yaml_filter` would 172 | 173 | useful for development to obtain the objects that the filter 174 | would see after passed to `panflute.yaml_filter` 175 | 176 | :param str text: must be a single codeblock of class table in markdown 177 | ''' 178 | 179 | def function(**kwargs): 180 | return kwargs 181 | 182 | doc = convert_text(text, standalone=True) 183 | return yaml_filter(doc.content[0], doc, tag='table', function=function, strict_yaml=True) 184 | 185 | 186 | if PY37: 187 | def _find_type_origin(type_hint: Any) -> Generator[Any, None, None]: 188 | if isinstance(type_hint, _SpecialForm): 189 | # case of Any, ClassVar, Final, Literal, 190 | # NoReturn, Optional, or Union without parameters 191 | yield Any 192 | return 193 | try: 194 | actual_type = type_hint.__origin__ 195 | except AttributeError: 196 | # In case of non-typing types (such as , for instance) 197 | actual_type = type_hint 198 | if isinstance(actual_type, _SpecialForm): 199 | # case of Union[…] or ClassVar[…] or … 200 | for origins in map(_find_type_origin, type_hint.__args__): 201 | yield from origins 202 | else: 203 | yield actual_type 204 | else: 205 | def _find_type_origin(type_hint: Any) -> Generator[Any, None, None]: 206 | if isinstance(type_hint, _SpecialForm): 207 | # case of Any, ClassVar, Final, Literal, 208 | # NoReturn, Optional, or Union without parameters 209 | yield Any 210 | return 211 | actual_type = get_origin(type_hint) or type_hint 212 | if isinstance(actual_type, _SpecialForm): 213 | # case of Union[…] or ClassVar[…] or … 214 | for origins in map(_find_type_origin, get_args(type_hint)): 215 | yield from origins 216 | else: 217 | yield actual_type 218 | 219 | 220 | def get_types(cls: Any) -> Dict[str, tuple]: 221 | '''returns all type hints in a Union 222 | 223 | c.f. https://stackoverflow.com/a/50622643 224 | ''' 225 | return { 226 | name: tuple( 227 | origin 228 | for origin in _find_type_origin(type_hint) 229 | if origin is not Any 230 | ) 231 | for name, type_hint in get_type_hints(cls).items() 232 | } 233 | 234 | 235 | def get_yaml_dumper(): 236 | try: 237 | from yamlloader.ordereddict.dumpers import CSafeDumper as Dumper 238 | except ImportError: 239 | try: 240 | from yamlloader.ordereddict.dumpers import SafeDumper as Dumper 241 | except ImportError: 242 | logger.warning('Try `pip install yamlloader` or `conda install yamlloader -c conda-forge` to preserve yaml dict ordering.') 243 | try: 244 | from yaml.cyaml import CSafeDumper as Dumper 245 | except ImportError: 246 | from yaml.dumper import SafeDumper as Dumper 247 | return Dumper 248 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ickc/pantable/ba26525e3e9c6ddab6236276ec9a9ac3508e31f5/tests/__init__.py -------------------------------------------------------------------------------- /tests/ast_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pytest import mark 3 | 4 | from pantable.ast import Align, PanTableOption 5 | 6 | 7 | @mark.parametrize('kwargs1,kwargs2', ( 8 | ( 9 | {'table_width': 3.}, 10 | {'table_width': 3}, 11 | ), 12 | ( 13 | {}, 14 | {'table_width': 'string'}, 15 | ), 16 | ( 17 | {'caption': '2'}, 18 | {'caption': 2}, 19 | ), 20 | ( 21 | {}, 22 | {'csv_kwargs': []}, 23 | ), 24 | ( 25 | {'table_width': '2/3'}, 26 | {'table_width': 2 / 3}, 27 | ), 28 | )) 29 | def test_pantableoption_type(kwargs1, kwargs2): 30 | assert PanTableOption(**kwargs1) == PanTableOption(**kwargs2) 31 | 32 | 33 | @mark.parametrize('kwargs1,kwargs2,shape', ( 34 | ( 35 | {'width': [1, 2, 'D']}, 36 | {'width': [1, 2, None]}, 37 | (3, 3), 38 | ), 39 | ( 40 | {'width': [1, 2, '2/3']}, 41 | {'width': [1, 2, 2 / 3]}, 42 | (3, 3), 43 | ), 44 | ( 45 | {'width': [1, 2, '2/3']}, 46 | {'width': [1, 2]}, 47 | (2, 2), 48 | ), 49 | ( 50 | { 51 | 'width': [1, 2, '2/3'], 52 | 'table_width': 3, 53 | }, { 54 | 'width': [1, 2, '2/3'], 55 | 'table_width': None, 56 | }, 57 | (3, 3), 58 | ), 59 | ( 60 | {'ms': [1, 1]}, 61 | {'ms': None}, 62 | (3, 3), 63 | ), 64 | ( 65 | {'ms': [1, 1, 0, 0, 1]}, 66 | {'ms': None}, 67 | (3, 3), 68 | ), 69 | ( 70 | {'ms': [-1, 1, 0, 3]}, 71 | {'ms': None}, 72 | (3, 3), 73 | ), 74 | ( 75 | {'ms': [1, 1, 0, 3]}, 76 | {'ms': None}, 77 | (3, 3), 78 | ), 79 | ( 80 | { 81 | 'ms': [1, 1, 1, 1, 3, 3], 82 | 'ns_head': [1], 83 | }, { 84 | 'ms': [1, 1, 1, 1, 3, 3], 85 | 'ns_head': None, 86 | }, 87 | (10, 10), 88 | ), 89 | ( 90 | { 91 | 'ms': [1, 1, 1, 1, 3, 3], 92 | 'ns_head': [11, 11], 93 | }, { 94 | 'ms': [1, 1, 1, 1, 3, 3], 95 | 'ns_head': None, 96 | }, 97 | (10, 10), 98 | ), 99 | )) 100 | def test_pantableoption_normalize(kwargs1, kwargs2, shape): 101 | op1 = PanTableOption(**kwargs1) 102 | op1.normalize(shape=shape) 103 | op2 = PanTableOption(**kwargs2) 104 | op2.normalize(shape=shape) 105 | assert op1 == op2 106 | 107 | 108 | @mark.parametrize('kwargs1,kwargs2', ( 109 | ( 110 | {'width': ['D', 'D', 'D']}, 111 | {'width': None}, 112 | ), 113 | ( 114 | {'width': [1 / 3, 2 / 3, 10 / 3]}, 115 | {'width': ['1/3', '2/3', '10/3']}, 116 | ), 117 | ( 118 | {'ms': [1, 0, 9, 0]}, 119 | {'header': True}, 120 | ), 121 | ( 122 | {'ms': [0, 0, 9, 0]}, 123 | {'header': False}, 124 | ), 125 | )) 126 | def test_pantableoption_simplify(kwargs1, kwargs2): 127 | op1 = PanTableOption(**kwargs1) 128 | op1.simplify() 129 | op2 = PanTableOption(**kwargs2) 130 | op2.simplify() 131 | assert op1 == op2 132 | 133 | 134 | @mark.parametrize('kwargs', ( 135 | {'ms': [2, 0, 9, 0]}, 136 | )) 137 | def test_pantableoption_simplify_2(kwargs): 138 | '''nothing to simplify cases 139 | ''' 140 | res = PanTableOption(**kwargs) 141 | res.simplify() 142 | for key, value in kwargs.items(): 143 | assert getattr(res, key) == value 144 | 145 | 146 | case_test = PanTableOption.from_kwargs(**{ 147 | 'caption': 'Some interesting...', 148 | 'unknown-key': 'path towards error', 149 | 'table-width': 0.5, 150 | }) 151 | 152 | 153 | def test_pantableoption_unknown_key(): 154 | assert case_test == PanTableOption.from_kwargs(**{ 155 | 'caption': 'Some interesting...', 156 | 'table-width': 0.5, 157 | }) 158 | 159 | 160 | def test_pantableoption_kwargs(): 161 | assert case_test == case_test.from_kwargs(**case_test.kwargs) 162 | 163 | 164 | ALIGN_DICT = { 165 | 'D': "AlignDefault", 166 | 'L': "AlignLeft", 167 | 'R': "AlignRight", 168 | 'C': "AlignCenter", 169 | } 170 | aligns_char = ['D', 'R', 'C', 'L'] 171 | 172 | 173 | def test_align_text_1D(): 174 | aligns_char_1D = np.array(aligns_char, dtype='S1') 175 | n = 4 176 | 177 | aligns = Align.from_aligns_char(aligns_char_1D) 178 | 179 | np.testing.assert_array_equal(aligns.aligns_char, aligns_char_1D) 180 | np.testing.assert_array_equal(aligns.aligns_idx, np.array([0, 2, 3, 1], dtype=np.int8)) 181 | 182 | aligns_text = aligns.aligns_text 183 | for i in range(n): 184 | assert aligns_text[i] == ALIGN_DICT[aligns_char_1D[i].decode()] 185 | 186 | aligns_string = aligns.aligns_string 187 | assert aligns_string == 'DRCL' 188 | 189 | assert Align.from_aligns_text(aligns_text) == aligns 190 | assert Align.from_aligns_string(aligns_string) == aligns 191 | 192 | 193 | def test_align_text_2D(): 194 | aligns_char_2D = np.array([aligns_char, list(reversed(aligns_char))], dtype='S1') 195 | m = 2 196 | n = 4 197 | 198 | aligns = Align.from_aligns_char(aligns_char_2D) 199 | 200 | np.testing.assert_array_equal(aligns.aligns_char, aligns_char_2D) 201 | np.testing.assert_array_equal(aligns.aligns_idx, np.array([ 202 | [0, 2, 3, 1], 203 | [1, 3, 2, 0], 204 | ], dtype=np.int8)) 205 | 206 | aligns_text = aligns.aligns_text 207 | for i in range(m): 208 | for j in range(n): 209 | assert aligns_text[i, j] == ALIGN_DICT[aligns_char_2D[i, j].decode()] 210 | 211 | aligns_string = aligns.aligns_string 212 | assert aligns_string == 'DRCL\nLCRD' 213 | 214 | assert Align.from_aligns_text(aligns_text) == aligns 215 | assert Align.from_aligns_string(aligns_string) == aligns 216 | -------------------------------------------------------------------------------- /tests/files/makefile: -------------------------------------------------------------------------------- 1 | SHELL = /usr/bin/env bash 2 | 3 | PANTABLELOGLEVEL ?= DEBUG 4 | pandoc ?= pandoc 5 | _pandoc = PANTABLELOGLEVEL=$(PANTABLELOGLEVEL) $(pandoc) 6 | 7 | MDFILES = $(wildcard md/*.md) 8 | MDFILESOUTPUT = $(patsubst md/%.md,md_reference/%.md,$(MDFILES)) 9 | MDCODEBLOCKFILES = $(wildcard md_codeblock/*.md) 10 | MDCODEBLOCKFILESOUTPUT = $(patsubst md_codeblock/%.md,md_codeblock_reference/%.md,$(MDCODEBLOCKFILES)) 11 | NATIVEFILES = $(wildcard native/*.native) 12 | NATIVEFILESOUTPUT = $(patsubst native/%.native,native_reference/%.md,$(NATIVEFILES)) 13 | 14 | HTML = $(patsubst native/%.native,native/%.html,$(NATIVEFILES)) 15 | 16 | all: $(MDFILESOUTPUT) $(MDCODEBLOCKFILESOUTPUT) $(NATIVEFILESOUTPUT) 17 | 18 | html: $(HTML) 19 | 20 | md_reference/%.md: md/%.md 21 | cd ../..; $(_pandoc) -F pantable2csv -o tests/files/$@ tests/files/$< 22 | 23 | md_codeblock_reference/%.md: md_codeblock/%.md 24 | cd ../..; $(_pandoc) -F pantable -o tests/files/$@ tests/files/$< 25 | 26 | native_reference/%.md: native/%.native 27 | cd ../..; $(_pandoc) -F pantable2csvx -o tests/files/$@ tests/files/$< 28 | 29 | native/%.html: native/%.native 30 | pandoc $< -s -o $@ 31 | -------------------------------------------------------------------------------- /tests/files/md/tables.md: -------------------------------------------------------------------------------- 1 | | 1 | 2 | 3 | 4 | 2 | |:--|--:|:-:|---| 3 | | 1 | 2 | 3 | 4 | 4 | 5 | : *abcd* 6 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/comparison.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | table-width: 1. 4 | --- 5 | ,pandoc-csv2table,pandoc-placetable,panflute example,my proposal 6 | type,type=simple|multiline|grid|pipe,,, 7 | header,header=yes|no,header=yes|no,header: True|False,header: True|False 8 | caption,caption,caption,title,caption 9 | source,source,file,source,include 10 | aligns,aligns=LRCD,aligns=LRCD,,alignment: LRCD 11 | width,,"widths=""0.5 0.2 0.3""",,"column-width: [0.5, 0.2, 0.3]" 12 | ,,inlinemarkdown,,markdown: True|False 13 | ,,delimiter,, 14 | ,,quotechar,, 15 | ,,id (wrapped by div),, 16 | ``` 17 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/csv_table_gbk.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ickc/pantable/ba26525e3e9c6ddab6236276ec9a9ac3508e31f5/tests/files/md_codeblock/csv_table_gbk.csv -------------------------------------------------------------------------------- /tests/files/md_codeblock/csv_tables.csv: -------------------------------------------------------------------------------- 1 | **_Fruit_**,~~Price~~,_Number_,`Advantages` 2 | *Bananas~1~*,$1.34,12~units~,"Benefits of eating bananas 3 | (**Note the appropriately 4 | rendered block markdown**): 5 | 6 | - _built-in wrapper_ 7 | - ~~**bright color**~~ 8 | 9 | " 10 | *Oranges~2~*,$2.10,5^10^~units~,"Benefits of eating oranges: 11 | 12 | - **cures** scurvy 13 | - `tasty`" -------------------------------------------------------------------------------- /tests/files/md_codeblock/empty_csv.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | ... 4 | ``` 5 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/encoding.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | include: tests/files/md_codeblock/csv_table_gbk.csv 4 | include-encoding: gbk 5 | ``` 6 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/full_test.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | caption: "*Great* Title" 4 | alignment: LRC 5 | width: 6 | - 0.1 7 | - 0.2 8 | - 0.3 9 | - 0.4 10 | header: False 11 | markdown: True 12 | ... 13 | **_Fruit_**,~~Price~~,_Number_,`Advantages` 14 | *Bananas~1~*,$1.34,12~units~,"Benefits of eating bananas 15 | (**Note the appropriately 16 | rendered block markdown**): 17 | 18 | - _built-in wrapper_ 19 | - ~~**bright color**~~ 20 | 21 | " 22 | *Oranges~2~*,$2.10,5^10^~units~,"Benefits of eating oranges: 23 | 24 | - **cures** scurvy 25 | - `tasty`" 26 | ``` 27 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/include_external_csv.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | caption: "*Great* Title" 4 | header: True 5 | markdown: True 6 | alignment: AlignLeft, AlignRight, AlignCenter, AlignDefault 7 | include: tests/files/md_codeblock/csv_tables.csv 8 | ... 9 | ``` 10 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/include_external_csv_invalid_path.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | caption: "*Great* Title" 4 | header: True 5 | markdown: True 6 | alignment: AlignLeft, AlignRight, AlignCenter, AlignDefault 7 | include: xyz/csv_tables.csv 8 | ... 9 | ``` 10 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/irregular_csv.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | markdown: False 4 | ... 5 | 2,3,4,5,6,7 6 | 1 7 | ``` 8 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/issue-57.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | alignment: DLRL 4 | markdown: true 5 | fancy-table: true 6 | ... 7 | ===,,**asdfsadf**,, 8 | ,**asdfasdf**,asdfasd,180,safgafg 9 | ,,asdfa,90,asgadsfg 10 | ,,asdfsadf,40,asgasfg 11 | ,,zxcvxczv,1,asgsafg 12 | ___,,zxcvxczv,1,zxcvzxv 13 | ,**xzcvxczv**,zxcvzcxv,100,sdfgasg 14 | ,,sagfsg,40,asfg 15 | ,,asdgfasfg,70,adsfgbbvv 16 | ,,asgsadg,30,adfgfdg 17 | ___,,Edsafdsag,1,asfgsafg 18 | ``` 19 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/one_row_table.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | 1,2 3 | ``` 4 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/simple_test.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | **_Fruit_**,~~Price~~,_Number_,`Advantages` 3 | *Bananas~1~*,$1.34,12~units~,"Benefits of eating bananas 4 | (**Note the appropriately 5 | rendered block markdown**): 6 | 7 | - _built-in wrapper_ 8 | - ~~**bright color**~~ 9 | 10 | " 11 | *Oranges~2~*,$2.10,5^10^~units~,"Benefits of eating oranges: 12 | 13 | - **cures** scurvy 14 | - `tasty`" 15 | ``` 16 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/testing_0_table_width.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | , 3 | , 4 | ``` 5 | -------------------------------------------------------------------------------- /tests/files/md_codeblock/testing_wrong_type.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | **_Fruit_**,~~Price~~,_Number_,`Advantages` 3 | *Bananas~1~*,$1.34,12~units~,"Benefits of eating bananas 4 | (**Note the appropriately 5 | rendered block markdown**): 6 | 7 | - _built-in wrapper_ 8 | - ~~**bright color**~~ 9 | 10 | --- 11 | caption: 0.1 12 | header: IDK 13 | markdown: true 14 | ... 15 | 16 | " 17 | *Oranges~2~*,$2.10,5^10^~units~,"Benefits of eating oranges: 18 | 19 | - **cures** scurvy 20 | - `tasty`" 21 | 22 | --- 23 | width: 24 | - -0.1 25 | - -0.2 26 | - -0.3 27 | - -0.4 28 | alignment: 0.1 29 | --- 30 | ``` 31 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_idem_test.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | from pathlib import Path 3 | from typing import Tuple 4 | 5 | from panflute import convert_text 6 | from pytest import mark 7 | 8 | from pantable.ast import PanCodeBlock 9 | from pantable.util import parse_markdown_codeblock 10 | 11 | logger = getLogger('pantable') 12 | 13 | EXT = 'md' 14 | PWD = Path(__file__).parent 15 | DIR = PWD / 'md_codeblock' 16 | 17 | 18 | def round_trip(text: str) -> str: 19 | kwargs = parse_markdown_codeblock(text) 20 | pan_codeblock = PanCodeBlock.from_yaml_filter(**kwargs) 21 | doc = pan_codeblock.to_panflute_ast() 22 | return convert_text(doc, input_format='panflute', output_format='markdown') 23 | 24 | 25 | def read(path: Path) -> Tuple[str, str, str]: 26 | '''test parsing markdown codeblock to PanCodeBlock 27 | ''' 28 | logger.info(f'Testing idempotence with {path}...') 29 | with open(path, 'r') as f: 30 | text = f.read() 31 | 32 | text_out = round_trip(text) 33 | text_idem = round_trip(text_out) 34 | 35 | return text_out, text_idem, text 36 | 37 | 38 | def read_io(name: str) -> Tuple[str, str, str]: 39 | path = DIR / f'{name}.{EXT}' 40 | return read(path) 41 | 42 | 43 | @mark.parametrize('name', (path.stem for path in DIR.glob(f'*.{EXT}'))) 44 | def test_md_codeblock_idem(name): 45 | res = read_io(name) 46 | assert res[0].strip() == res[1].strip() 47 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/comparison.md: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------------------------------------------------------------------------------------ 2 | pandoc-csv2table pandoc-placetable panflute example my proposal 3 | -------------- ------------------------------------ ------------------------ --------------------- --------------------------------- 4 | type type=simple\|multiline\|grid\|pipe 5 | 6 | header header=yes\|no header=yes\|no header: True\|False header: True\|False 7 | 8 | caption caption caption title caption 9 | 10 | source source file source include 11 | 12 | aligns aligns=LRCD aligns=LRCD alignment: LRCD 13 | 14 | width widths=\"0.5 0.2 0.3\" column-width: \[0.5, 0.2, 0.3\] 15 | 16 | inlinemarkdown markdown: True\|False 17 | 18 | delimiter 19 | 20 | quotechar 21 | 22 | id (wrapped by div) 23 | ------------------------------------------------------------------------------------------------------------------------------------ 24 | 25 | : 26 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/empty_csv.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/encoding.md: -------------------------------------------------------------------------------- 1 | 一 二 三 2 | ---- ---- ---- 3 | a b c 4 | 1 2 3 5 | 6 | : 7 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/full_test.md: -------------------------------------------------------------------------------- 1 | +------+-------------+--------------------+---------------------------+ 2 | | *** | ~~Price~~ | *Number* | `Advantages` | 3 | | Frui | | | | 4 | | t*** | | | | 5 | +:=====+============:+:==================:+===========================+ 6 | | *Ban | \$1.34 | 12~units~ | Benefits of eating | 7 | | anas | | | bananas (**Note the | 8 | | ~1~* | | | appropriately rendered | 9 | | | | | block markdown**): | 10 | | | | | | 11 | | | | | - *built-in wrapper* | 12 | | | | | - ~~**bright color**~~ | 13 | +------+-------------+--------------------+---------------------------+ 14 | | *Ora | \$2.10 | 5^10^~units~ | Benefits of eating | 15 | | nges | | | oranges: | 16 | | ~2~* | | | | 17 | | | | | - **cures** scurvy | 18 | | | | | - `tasty` | 19 | +------+-------------+--------------------+---------------------------+ 20 | 21 | : *Great* Title 22 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/include_external_csv.md: -------------------------------------------------------------------------------- 1 | +--------------+-----------+--------------+-------------------------+ 2 | | ***Fruit*** | ~~Price~~ | *Number* | `Advantages` | 3 | +:=============+:==========+==============+========================:+ 4 | | *Bananas~1~* | \$1.34 | 12~units~ | Benefits of eating | 5 | | | | | bananas (**Note the | 6 | | | | | appropriately rendered | 7 | | | | | block markdown**): | 8 | | | | | | 9 | | | | | - *built-in wrapper*\ | 10 | | | | | - ~~**bright | 11 | | | | | color**~~ | 12 | +--------------+-----------+--------------+-------------------------+ 13 | | *Oranges~2~* | \$2.10 | 5^10^~units~ | Benefits of eating | 14 | | | | | oranges: | 15 | | | | | | 16 | | | | | - **cures** scurvy | 17 | | | | | - `tasty` | 18 | +--------------+-----------+--------------+-------------------------+ 19 | 20 | : *Great* Title 21 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/include_external_csv_invalid_path.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | caption: "*Great* Title" 4 | header: True 5 | markdown: True 6 | alignment: AlignLeft, AlignRight, AlignCenter, AlignDefault 7 | include: xyz/csv_tables.csv 8 | ... 9 | ``` 10 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/invalid_yaml.md: -------------------------------------------------------------------------------- 1 | ``` {.table} 2 | --- 3 | caption: *unquoted* 4 | ... 5 | 1,2 6 | 3,4 7 | ``` 8 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/irregular_csv.md: -------------------------------------------------------------------------------- 1 | 2 3 4 5 6 7 2 | --- --- --- --- --- --- 3 | 1 4 | 5 | : 6 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/issue-57.md: -------------------------------------------------------------------------------- 1 | **asdfsadf** 2 | -------------- -------------- ----- ----------- 3 | **asdfasdf** asdfasd 180 safgafg 4 | asdfa 90 asgadsfg 5 | asdfsadf 40 asgasfg 6 | zxcvxczv 1 asgsafg 7 | zxcvxczv 1 zxcvzxv 8 | **xzcvxczv** zxcvzcxv 100 sdfgasg 9 | sagfsg 40 asfg 10 | asdgfasfg 70 adsfgbbvv 11 | asgsadg 30 adfgfdg 12 | Edsafdsag 1 asfgsafg 13 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/one_row_table.md: -------------------------------------------------------------------------------- 1 | 1 2 2 | --- --- 3 | 4 | : 5 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/simple_test.md: -------------------------------------------------------------------------------- 1 | \*\*\_Fruit\_\*\* \~\~Price\~\~ \_Number\_ \`Advantages\` 2 | ------------------- --------------- ------------------ -------------------------------- 3 | \*Bananas\~1\~\* \$1.34 12\~units\~ Benefits of eating bananas 4 | (\*\*Note the appropriately 5 | rendered block markdown\*\*): 6 | 7 | - \_built-in wrapper\_ 8 | - \~\~\*\*bright color\*\*\~\~ 9 | \*Oranges\~2\~\* \$2.10 5\^10\^\~units\~ Benefits of eating oranges: 10 | 11 | - \*\*cures\*\* scurvy 12 | - \`tasty\` 13 | 14 | : 15 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/testing_0_table_width.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_reference/testing_wrong_type.md: -------------------------------------------------------------------------------- 1 | +--------------+-----------+--------------+-------------------------+ 2 | | ***Fruit*** | ~~Price~~ | *Number* | `Advantages` | 3 | +:=============+===========+=============:+=========================+ 4 | | *Bananas~1~* | \$1.34 | 12~units~ | Benefits of eating | 5 | | | | | bananas (**Note the | 6 | | | | | appropriately rendered | 7 | | | | | block markdown**): | 8 | | | | | | 9 | | | | | - *built-in wrapper* | 10 | | | | | - ~~**bright | 11 | | | | | color**~~ | 12 | +--------------+-----------+--------------+-------------------------+ 13 | | *Oranges~2~* | \$2.10 | 5^10^~units~ | Benefits of eating | 14 | | | | | oranges: | 15 | | | | | | 16 | | | | | - **cures** scurvy | 17 | | | | | - `tasty` | 18 | +--------------+-----------+--------------+-------------------------+ 19 | 20 | : 0.1 21 | -------------------------------------------------------------------------------- /tests/files/md_codeblock_test.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | from pathlib import Path 3 | from typing import Tuple 4 | 5 | from panflute import convert_text 6 | from pytest import mark 7 | 8 | # use the function exactly used by the cli 9 | from pantable.codeblock_to_table import codeblock_to_table 10 | from pantable.util import parse_markdown_codeblock 11 | 12 | logger = getLogger('pantable') 13 | 14 | EXT = 'md' 15 | PWD = Path(__file__).parent 16 | DIRS = (PWD / 'md_codeblock', PWD / 'md_codeblock_reference') 17 | 18 | 19 | def read(path: Path, path_ref: Path) -> Tuple[str, str]: 20 | '''test parsing markdown codeblock 21 | ''' 22 | logger.info(f'Comparing {path} and {path_ref}...') 23 | with open(path, 'r') as f: 24 | text = f.read() 25 | with open(path_ref, 'r') as f: 26 | md_reference = f.read() 27 | 28 | try: 29 | kwargs = parse_markdown_codeblock(text) 30 | doc = codeblock_to_table(**kwargs) 31 | md_out = convert_text(doc, input_format='panflute', output_format='markdown') 32 | except TypeError: 33 | logger.error('Cannot parse input codeblock, leaving as is.') 34 | md_out = text 35 | 36 | return md_reference, md_out 37 | 38 | 39 | def read_io(name: str) -> Tuple[str, str]: 40 | paths = [dir_ / f'{name}.{EXT}' for dir_ in DIRS] 41 | return read(*paths) 42 | 43 | 44 | @mark.parametrize('name', (path.stem for path in DIRS[0].glob(f'*.{EXT}'))) 45 | def test_md_codeblock(name): 46 | res = read_io(name) 47 | assert res[0].strip() == res[1].strip() 48 | -------------------------------------------------------------------------------- /tests/files/md_reference/tables.md: -------------------------------------------------------------------------------- 1 | ``` table 2 | --- 3 | alignment: LRC 4 | caption: '*abcd*' 5 | markdown: true 6 | ... 7 | 1,2,3,4 8 | 1,2,3,4 9 | ``` 10 | -------------------------------------------------------------------------------- /tests/files/md_test.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | from pathlib import Path 3 | from typing import Tuple 4 | 5 | from panflute import convert_text 6 | from panflute.tools import pandoc_version 7 | from pytest import mark, xfail 8 | 9 | # use the function exactly used by the cli 10 | from pantable.table_to_codeblock import table_to_codeblock 11 | 12 | logger = getLogger('pantable') 13 | 14 | EXT = 'md' 15 | PWD = Path(__file__).parent 16 | DIRS = (PWD / 'md', PWD / 'md_reference') 17 | 18 | 19 | def read(path: Path, path_ref: Path) -> Tuple[str, str]: 20 | '''test parsing table to codeblock 21 | ''' 22 | logger.info(f'Comparing {path} and {path_ref}...') 23 | with open(path, 'r') as f: 24 | text = f.read() 25 | with open(path_ref, 'r') as f: 26 | md_reference = f.read() 27 | 28 | doc = convert_text(text, input_format='markdown') 29 | # input files should only have 1 single outter block 30 | assert len(doc) == 1 31 | table = doc[0] 32 | doc_out = table_to_codeblock(table, doc) 33 | md_out = convert_text(doc_out, input_format='panflute', output_format='markdown') 34 | 35 | return md_reference, md_out 36 | 37 | 38 | def read_io(name: str) -> Tuple[str, str]: 39 | paths = [dir_ / f'{name}.{EXT}' for dir_ in DIRS] 40 | return read(*paths) 41 | 42 | 43 | @mark.parametrize('name', (path.stem for path in DIRS[0].glob(f'*.{EXT}'))) 44 | def test_md(name: str): 45 | if pandoc_version.version < (2, 14) and name == 'tables': 46 | xfail("jgm/pandoc#7242 changed code-blocks output that is cosmetically different but semantically the same. Also see commit db7ce7d.") 47 | res = read_io(name) 48 | assert res[0].strip() == res[1].strip() 49 | -------------------------------------------------------------------------------- /tests/files/native/nordics.native: -------------------------------------------------------------------------------- 1 | [Table ("nordics",[],[("source","wikipedia")]) (Caption (Just [Str "Nordic",Space,Str "countries"]) 2 | [Para [Str "States",Space,Str "belonging",Space,Str "to",Space,Str "the",Space,Emph [Str "Nordics."]]]) 3 | [(AlignCenter,ColWidth 0.3) 4 | ,(AlignLeft,ColWidth 0.3) 5 | ,(AlignLeft,ColWidth 0.2) 6 | ,(AlignLeft,ColWidth 0.2)] 7 | (TableHead ("",["simple-head"],[]) 8 | [Row ("",[],[]) 9 | [Cell ("",[],[]) AlignCenter (RowSpan 1) (ColSpan 1) 10 | [Plain [Str "Name"]] 11 | ,Cell ("",[],[]) AlignCenter (RowSpan 1) (ColSpan 1) 12 | [Plain [Str "Capital"]] 13 | ,Cell ("",[],[]) AlignCenter (RowSpan 1) (ColSpan 1) 14 | [Plain [Str "Population",LineBreak,Str "(in",Space,Str "2018)"]] 15 | ,Cell ("",[],[]) AlignCenter (RowSpan 1) (ColSpan 1) 16 | [Plain [Str "Area",LineBreak,Str "(in",Space,Str "km",Superscript [Str "2"],Str ")"]]]]) 17 | [(TableBody ("",["souvereign-states"],[]) (RowHeadColumns 1) 18 | [] 19 | [Row ("",["country"],[]) 20 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 21 | [Plain [Str "Denmark"]] 22 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 23 | [Plain [Str "Copenhagen"]] 24 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 25 | [Plain [Str "5,809,502"]] 26 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 27 | [Plain [Str "43,094"]]] 28 | ,Row ("",["country"],[]) 29 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 30 | [Plain [Str "Finland"]] 31 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 32 | [Plain [Str "Helsinki"]] 33 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 34 | [Plain [Str "5,537,364"]] 35 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 36 | [Plain [Str "338,145"]]] 37 | ,Row ("",["country"],[]) 38 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 39 | [Plain [Str "Iceland"]] 40 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 41 | [Plain [Str "Reykjavik"]] 42 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 43 | [Plain [Str "343,518"]] 44 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 45 | [Plain [Str "103,000"]]] 46 | ,Row ("",["country"],[]) 47 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 48 | [Plain [Str "Norway"]] 49 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 50 | [Plain [Str "Oslo"]] 51 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 52 | [Plain [Str "5,372,191"]] 53 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 54 | [Plain [Str "323,802"]]] 55 | ,Row ("",["country"],[]) 56 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 57 | [Plain [Str "Sweden"]] 58 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 59 | [Plain [Str "Stockholm"]] 60 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 61 | [Plain [Str "10,313,447"]] 62 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 63 | [Plain [Str "450,295"]]]])] 64 | (TableFoot ("",[],[]) 65 | [Row ("summary",[],[]) 66 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 67 | [Plain [Str "Total"]] 68 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 69 | [] 70 | ,Cell ("total-population",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 71 | [Plain [Str "27,376,022"]] 72 | ,Cell ("total-area",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 73 | [Plain [Str "1,258,336"]]]])] 74 | -------------------------------------------------------------------------------- /tests/files/native/planets.native: -------------------------------------------------------------------------------- 1 | [Table ("",[],[]) (Caption Nothing 2 | [Para [Str "Data",Space,Str "about",Space,Str "the",Space,Str "planets",Space,Str "of",Space,Str "our",Space,Str "solar",Space,Str "system."]]) 3 | [(AlignCenter,ColWidthDefault) 4 | ,(AlignCenter,ColWidthDefault) 5 | ,(AlignDefault,ColWidthDefault) 6 | ,(AlignRight,ColWidthDefault) 7 | ,(AlignRight,ColWidthDefault) 8 | ,(AlignRight,ColWidthDefault) 9 | ,(AlignRight,ColWidthDefault) 10 | ,(AlignRight,ColWidthDefault) 11 | ,(AlignRight,ColWidthDefault) 12 | ,(AlignRight,ColWidthDefault) 13 | ,(AlignRight,ColWidthDefault) 14 | ,(AlignDefault,ColWidthDefault)] 15 | (TableHead ("",[],[]) 16 | [Row ("",[],[]) 17 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 2) 18 | [] 19 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 20 | [Plain [Str "Name"]] 21 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 22 | [Plain [Str "Mass",Space,Str "(10^24kg)"]] 23 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 24 | [Plain [Str "Diameter",Space,Str "(km)"]] 25 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 26 | [Plain [Str "Density",Space,Str "(kg/m^3)"]] 27 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 28 | [Plain [Str "Gravity",Space,Str "(m/s^2)"]] 29 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 30 | [Plain [Str "Length",Space,Str "of",Space,Str "day",Space,Str "(hours)"]] 31 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 32 | [Plain [Str "Distance",Space,Str "from",Space,Str "Sun",Space,Str "(10^6km)"]] 33 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 34 | [Plain [Str "Mean",Space,Str "temperature",Space,Str "(C)"]] 35 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 36 | [Plain [Str "Number",Space,Str "of",Space,Str "moons"]] 37 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 38 | [Plain [Str "Notes"]]]]) 39 | [(TableBody ("",[],[]) (RowHeadColumns 3) 40 | [] 41 | [Row ("",[],[]) 42 | [Cell ("",[],[]) AlignDefault (RowSpan 4) (ColSpan 2) 43 | [Plain [Str "Terrestrial",Space,Str "planets"]] 44 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 45 | [Plain [Str "Mercury"]] 46 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 47 | [Plain [Str "0.330"]] 48 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 49 | [Plain [Str "4,879"]] 50 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 51 | [Plain [Str "5427"]] 52 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 53 | [Plain [Str "3.7"]] 54 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 55 | [Plain [Str "4222.6"]] 56 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 57 | [Plain [Str "57.9"]] 58 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 59 | [Plain [Str "167"]] 60 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 61 | [Plain [Str "0"]] 62 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 63 | [Plain [Str "Closest",Space,Str "to",Space,Str "the",Space,Str "Sun"]]] 64 | ,Row ("",[],[]) 65 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 66 | [Plain [Str "Venus"]] 67 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 68 | [Plain [Str "4.87"]] 69 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 70 | [Plain [Str "12,104"]] 71 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 72 | [Plain [Str "5243"]] 73 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 74 | [Plain [Str "8.9"]] 75 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 76 | [Plain [Str "2802.0"]] 77 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 78 | [Plain [Str "108.2"]] 79 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 80 | [Plain [Str "464"]] 81 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 82 | [Plain [Str "0"]] 83 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 84 | []] 85 | ,Row ("",[],[]) 86 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 87 | [Plain [Str "Earth"]] 88 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 89 | [Plain [Str "5.97"]] 90 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 91 | [Plain [Str "12,756"]] 92 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 93 | [Plain [Str "5514"]] 94 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 95 | [Plain [Str "9.8"]] 96 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 97 | [Plain [Str "24.0"]] 98 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 99 | [Plain [Str "149.6"]] 100 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 101 | [Plain [Str "15"]] 102 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 103 | [Plain [Str "1"]] 104 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 105 | [Plain [Str "Our",Space,Str "world"]]] 106 | ,Row ("",[],[]) 107 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 108 | [Plain [Str "Mars"]] 109 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 110 | [Plain [Str "0.642"]] 111 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 112 | [Plain [Str "6,792"]] 113 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 114 | [Plain [Str "3933"]] 115 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 116 | [Plain [Str "3.7"]] 117 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 118 | [Plain [Str "24.7"]] 119 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 120 | [Plain [Str "227.9"]] 121 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 122 | [Plain [Str "-65"]] 123 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 124 | [Plain [Str "2"]] 125 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 126 | [Plain [Str "The",Space,Str "red",Space,Str "planet"]]] 127 | ,Row ("",[],[]) 128 | [Cell ("",[],[]) AlignDefault (RowSpan 4) (ColSpan 1) 129 | [Plain [Str "Jovian",Space,Str "planets"]] 130 | ,Cell ("",[],[]) AlignDefault (RowSpan 2) (ColSpan 1) 131 | [Plain [Str "Gas",Space,Str "giants"]] 132 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 133 | [Plain [Str "Jupiter"]] 134 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 135 | [Plain [Str "1898"]] 136 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 137 | [Plain [Str "142,984"]] 138 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 139 | [Plain [Str "1326"]] 140 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 141 | [Plain [Str "23.1"]] 142 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 143 | [Plain [Str "9.9"]] 144 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 145 | [Plain [Str "778.6"]] 146 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 147 | [Plain [Str "-110"]] 148 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 149 | [Plain [Str "67"]] 150 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 151 | [Plain [Str "The",Space,Str "largest",Space,Str "planet"]]] 152 | ,Row ("",[],[]) 153 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 154 | [Plain [Str "Saturn"]] 155 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 156 | [Plain [Str "568"]] 157 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 158 | [Plain [Str "120,536"]] 159 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 160 | [Plain [Str "687"]] 161 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 162 | [Plain [Str "9.0"]] 163 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 164 | [Plain [Str "10.7"]] 165 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 166 | [Plain [Str "1433.5"]] 167 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 168 | [Plain [Str "-140"]] 169 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 170 | [Plain [Str "62"]] 171 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 172 | []] 173 | ,Row ("",[],[]) 174 | [Cell ("",[],[]) AlignDefault (RowSpan 2) (ColSpan 1) 175 | [Plain [Str "Ice",Space,Str "giants"]] 176 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 177 | [Plain [Str "Uranus"]] 178 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 179 | [Plain [Str "86.8"]] 180 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 181 | [Plain [Str "51,118"]] 182 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 183 | [Plain [Str "1271"]] 184 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 185 | [Plain [Str "8.7"]] 186 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 187 | [Plain [Str "17.2"]] 188 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 189 | [Plain [Str "2872.5"]] 190 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 191 | [Plain [Str "-195"]] 192 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 193 | [Plain [Str "27"]] 194 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 195 | []] 196 | ,Row ("",[],[]) 197 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 198 | [Plain [Str "Neptune"]] 199 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 200 | [Plain [Str "102"]] 201 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 202 | [Plain [Str "49,528"]] 203 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 204 | [Plain [Str "1638"]] 205 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 206 | [Plain [Str "11.0"]] 207 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 208 | [Plain [Str "16.1"]] 209 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 210 | [Plain [Str "4495.1"]] 211 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 212 | [Plain [Str "-200"]] 213 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 214 | [Plain [Str "14"]] 215 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 216 | []] 217 | ,Row ("",[],[]) 218 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 2) 219 | [Plain [Str "Dwarf",Space,Str "planets"]] 220 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 221 | [Plain [Str "Pluto"]] 222 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 223 | [Plain [Str "0.0146"]] 224 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 225 | [Plain [Str "2,370"]] 226 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 227 | [Plain [Str "2095"]] 228 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 229 | [Plain [Str "0.7"]] 230 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 231 | [Plain [Str "153.3"]] 232 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 233 | [Plain [Str "5906.4"]] 234 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 235 | [Plain [Str "-225"]] 236 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 237 | [Plain [Str "5"]] 238 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 239 | [Plain [Str "Declassified",Space,Str "as",Space,Str "a",Space,Str "planet",Space,Str "in",Space,Str "2006."]]]])] 240 | (TableFoot ("",[],[]) 241 | [])] 242 | -------------------------------------------------------------------------------- /tests/files/native/students.native: -------------------------------------------------------------------------------- 1 | [Table ("students",[],[("source","mdn")]) (Caption Nothing 2 | [Para [Str "List",Space,Str "of",Space,Str "Students"]]) 3 | [(AlignLeft,ColWidth 0.5) 4 | ,(AlignLeft,ColWidth 0.5)] 5 | (TableHead ("",[],[]) 6 | [Row ("",[],[]) 7 | [Cell ("",[],[]) AlignCenter (RowSpan 1) (ColSpan 1) 8 | [Plain [Str "Student",Space,Str "ID"]] 9 | ,Cell ("",[],[]) AlignCenter (RowSpan 1) (ColSpan 1) 10 | [Plain [Str "Name"]]]]) 11 | [(TableBody ("",["souvereign-states"],[]) (RowHeadColumns 0) 12 | [Row ("",[],[]) 13 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 2) 14 | [Plain [Str "Computer",Space,Str "Science"]]]] 15 | [Row ("",[],[]) 16 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 17 | [Plain [Str "3741255"]] 18 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 19 | [Plain [Str "Jones,",Space,Str "Martha"]]] 20 | ,Row ("",[],[]) 21 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 22 | [Plain [Str "4077830"]] 23 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 24 | [Plain [Str "Pierce,",Space,Str "Benjamin"]]] 25 | ,Row ("",[],[]) 26 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 27 | [Plain [Str "5151701"]] 28 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 29 | [Plain [Str "Kirk,",Space,Str "James"]]]]) 30 | ,(TableBody ("",[],[]) (RowHeadColumns 0) 31 | [Row ("",[],[]) 32 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 2) 33 | [Plain [Str "Russian",Space,Str "Literature"]]]] 34 | [Row ("",[],[]) 35 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 36 | [Plain [Str "3971244"]] 37 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 38 | [Plain [Str "Nim,",Space,Str "Victor"]]]]) 39 | ,(TableBody ("",[],[]) (RowHeadColumns 0) 40 | [Row ("",[],[]) 41 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 2) 42 | [Plain [Str "Astrophysics"]]]] 43 | [Row ("",[],[]) 44 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 45 | [Plain [Str "4100332"]] 46 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 47 | [Plain [Str "Petrov,",Space,Str "Alexandra"]]] 48 | ,Row ("",[],[]) 49 | [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 50 | [Plain [Str "4100332"]] 51 | ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) 52 | [Plain [Str "Toyota,",Space,Str "Hiroko"]]]])] 53 | (TableFoot ("",[],[]) 54 | [])] 55 | -------------------------------------------------------------------------------- /tests/files/native_iden_test.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | from pathlib import Path 3 | from typing import Tuple 4 | 5 | from panflute import convert_text 6 | from pytest import mark 7 | 8 | from pantable.ast import PanCodeBlock, PanTable 9 | from pantable.util import parse_markdown_codeblock 10 | 11 | logger = getLogger('pantable') 12 | 13 | EXT = 'native' 14 | DIR = Path(__file__).parent / EXT 15 | 16 | 17 | def read(path: Path) -> Tuple[str, str, str, str, str]: 18 | '''test parsing native table into Pantable 19 | ''' 20 | logger.info(f'Testing case {path}...') 21 | with open(path, 'r') as f: 22 | native = f.read() 23 | doc = convert_text(native, input_format='native') 24 | # input files should only have 1 single outter block 25 | assert len(doc) == 1 26 | table = doc[0] 27 | pan_table = PanTable.from_panflute_ast(table) 28 | table_idem = pan_table.to_panflute_ast() 29 | # PanTableStr 30 | pan_table_markdown = pan_table.to_pantablemarkdown() 31 | pan_table_idem = pan_table_markdown.to_pantable() 32 | table_idem2 = pan_table_idem.to_panflute_ast() 33 | # PanCodeBlock 34 | pan_codeblock = pan_table_markdown.to_pancodeblock(fancy_table=True) 35 | pan_table_markdown_idem = pan_codeblock.to_pantablestr() 36 | pan_table_idem2 = pan_table_markdown_idem.to_pantable() 37 | table_idem3 = pan_table_idem2.to_panflute_ast() 38 | # CodeBlock 39 | pf_codeblock = pan_codeblock.to_panflute_ast() 40 | md_codeblock = convert_text(pf_codeblock, input_format='panflute', output_format='markdown') 41 | kwargs = parse_markdown_codeblock(md_codeblock) 42 | pan_codeblock_idem = PanCodeBlock.from_yaml_filter(**kwargs) 43 | pan_table_markdown_idem2 = pan_codeblock_idem.to_pantablestr() 44 | pan_table_idem3 = pan_table_markdown_idem2.to_pantable() 45 | table_idem4 = pan_table_idem3.to_panflute_ast() 46 | # check for idempotence 47 | native_orig = convert_text(table, input_format='panflute', output_format='native') 48 | native_idem = convert_text(table_idem, input_format='panflute', output_format='native') 49 | native_idem2 = convert_text(table_idem2, input_format='panflute', output_format='native') 50 | native_idem3 = convert_text(table_idem3, input_format='panflute', output_format='native') 51 | native_idem4 = convert_text(table_idem4, input_format='panflute', output_format='native') 52 | return native_orig, native_idem, native_idem2, native_idem3, native_idem4 53 | 54 | 55 | def read_io(name: str) -> Tuple[str, str, str, str, str]: 56 | path = DIR / f'{name}.{EXT}' 57 | return read(path) 58 | 59 | 60 | @mark.parametrize('name', (path.stem for path in DIR.glob(f'*.{EXT}'))) 61 | def test_native_iden(name: str): 62 | res = read_io(name) 63 | assert res[0] == res[1] 64 | assert res[0] == res[2] 65 | assert res[0] == res[3] 66 | assert res[0] == res[4] 67 | -------------------------------------------------------------------------------- /tests/files/native_reference/makefile: -------------------------------------------------------------------------------- 1 | SHELL = /usr/bin/env bash 2 | 3 | MDs = $(wildcard *.md) 4 | HTMLs = $(patsubst %.md,%.html,$(MDs)) 5 | PDFs = $(patsubst %.md,%.pdf,$(MDs)) 6 | 7 | all: $(HTMLs) $(PDFs) 8 | 9 | %.html: %.md 10 | pandoc -s -o $@ $< -F pantable 11 | 12 | %.pdf: %.md 13 | pandoc -s -o $@ $< -F pantable 14 | -------------------------------------------------------------------------------- /tests/files/native_reference/nordics.md: -------------------------------------------------------------------------------- 1 | ``` {#nordics .table source="wikipedia"} 2 | --- 3 | alignment: CLLL 4 | alignment-cells: CCCC 5 | caption: States belonging to the *Nordics.* 6 | fancy-table: true 7 | markdown: true 8 | ms: 9 | - 1 10 | - 0 11 | - 5 12 | - 1 13 | ns-head: 14 | - 1 15 | short-caption: Nordic countries 16 | width: 17 | - 3/10 18 | - 3/10 19 | - 1/5 20 | - 1/5 21 | ... 22 | {.simple-head} ===,Name,Capital,"Population\ 23 | (in 2018)","Area\ 24 | (in km^2^)" 25 | {.country},Denmark,Copenhagen,"5,809,502","43,094" 26 | {.country},Finland,Helsinki,"5,537,364","338,145" 27 | {.country},Iceland,Reykjavik,"343,518","103,000" 28 | {.country},Norway,Oslo,"5,372,191","323,802" 29 | {.souvereign-states} ___ {.country},Sweden,Stockholm,"10,313,447","450,295" 30 | === {#summary},Total,,"{#total-population} 31 | 27,376,022","{#total-area} 32 | 1,258,336" 33 | ``` 34 | -------------------------------------------------------------------------------- /tests/files/native_reference/planets.md: -------------------------------------------------------------------------------- 1 | ``` table 2 | --- 3 | alignment: CCDRRRRRRRR 4 | caption: Data about the planets of our solar system. 5 | fancy-table: true 6 | markdown: true 7 | ns-head: 8 | - 3 9 | ... 10 | ===,"(1, 2) 11 | ",,Name,Mass (10\^24kg),Diameter (km),Density (kg/m\^3),Gravity (m/s\^2),Length of day (hours),Distance from Sun (10\^6km),Mean temperature (C),Number of moons,Notes 12 | ,"(4, 2) 13 | Terrestrial planets",,Mercury,0.330,"4,879",5427,3.7,4222.6,57.9,167,0,Closest to the Sun 14 | ,,,Venus,4.87,"12,104",5243,8.9,2802.0,108.2,464,0, 15 | ,,,Earth,5.97,"12,756",5514,9.8,24.0,149.6,15,1,Our world 16 | ,,,Mars,0.642,"6,792",3933,3.7,24.7,227.9,-65,2,The red planet 17 | ,"(4, 1) 18 | Jovian planets","(2, 1) 19 | Gas giants",Jupiter,1898,"142,984",1326,23.1,9.9,778.6,-110,67,The largest planet 20 | ,,,Saturn,568,"120,536",687,9.0,10.7,1433.5,-140,62, 21 | ,,"(2, 1) 22 | Ice giants",Uranus,86.8,"51,118",1271,8.7,17.2,2872.5,-195,27, 23 | ,,,Neptune,102,"49,528",1638,11.0,16.1,4495.1,-200,14, 24 | ___,"(1, 2) 25 | Dwarf planets",,Pluto,0.0146,"2,370",2095,0.7,153.3,5906.4,-225,5,Declassified as a planet in 2006. 26 | ``` 27 | -------------------------------------------------------------------------------- /tests/files/native_reference/students.md: -------------------------------------------------------------------------------- 1 | ``` {#students .table source="mdn"} 2 | --- 3 | alignment: LL 4 | alignment-cells: CC 5 | caption: List of Students 6 | fancy-table: true 7 | markdown: true 8 | ms: 9 | - 1 10 | - 1 11 | - 3 12 | - 1 13 | - 1 14 | - 1 15 | - 2 16 | - 0 17 | width: 18 | - 1/2 19 | - 1/2 20 | ... 21 | ===,Student ID,Name 22 | {.souvereign-states} ---,"(1, 2) 23 | Computer Science", 24 | ,3741255,"Jones, Martha" 25 | ,4077830,"Pierce, Benjamin" 26 | {.souvereign-states} ___,5151701,"Kirk, James" 27 | ---,"(1, 2) 28 | Russian Literature", 29 | ___,3971244,"Nim, Victor" 30 | ---,"(1, 2) 31 | Astrophysics", 32 | ,4100332,"Petrov, Alexandra" 33 | ___,4100332,"Toyota, Hiroko" 34 | ``` 35 | -------------------------------------------------------------------------------- /tests/files/native_test.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | from pathlib import Path 3 | from typing import Tuple 4 | 5 | from panflute import convert_text 6 | from panflute.tools import pandoc_version 7 | from pytest import mark, xfail 8 | 9 | from pantable.ast import PanTable 10 | # use the function exactly used by the cli 11 | from pantable.table_to_codeblock import table_to_codeblock 12 | 13 | logger = getLogger('pantable') 14 | 15 | EXTs = ('native', 'md') 16 | PWD = Path(__file__).parent 17 | DIRS = (PWD / 'native', PWD / 'native_reference') 18 | 19 | 20 | def read_table_to_codeblock(path: Path, path_ref: Path) -> Tuple[str, str]: 21 | '''test parsing native to markdown codeblock with fancy-table 22 | ''' 23 | logger.info(f'Comparing {path} and {path_ref}...') 24 | with open(path, 'r') as f: 25 | text = f.read() 26 | with open(path_ref, 'r') as f: 27 | md_reference = f.read() 28 | 29 | doc = convert_text(text, input_format='native') 30 | # input files should only have 1 single outter block 31 | assert len(doc) == 1 32 | table = doc[0] 33 | doc_out = table_to_codeblock(table, doc, fancy_table=True) 34 | md_out = convert_text(doc_out, input_format='panflute', output_format='markdown') 35 | 36 | return md_reference, md_out 37 | 38 | 39 | def read_table_to_codeblock_io(name: str) -> Tuple[str, str]: 40 | paths = [dir_ / f'{name}.{ext}' for dir_, ext in zip(DIRS, EXTs)] 41 | return read_table_to_codeblock(*paths) 42 | 43 | 44 | @mark.parametrize('name', (path.stem for path in DIRS[0].glob(f'*.{EXTs[0]}'))) 45 | def test_table_to_codeblock(name: str): 46 | if pandoc_version.version < (2, 14) and name == 'planets': 47 | xfail("jgm/pandoc#7242 changed code-blocks output that is cosmetically different but semantically the same. Also see commit db7ce7d.") 48 | res = read_table_to_codeblock_io(name) 49 | assert res[0].strip() == res[1].strip() 50 | 51 | 52 | def read_table_to_codeblock_not_fancy(path: Path): 53 | '''test parsing native to markdown codeblock without fancy-table 54 | 55 | This is lossy so we only check it runs 56 | ''' 57 | logger.info(f'Loading {path}...') 58 | with open(path, 'r') as f: 59 | text = f.read() 60 | 61 | doc = convert_text(text, input_format='native') 62 | # input files should only have 1 single outter block 63 | assert len(doc) == 1 64 | table = doc[0] 65 | table_to_codeblock(table, doc, fancy_table=False) 66 | 67 | 68 | @mark.parametrize('name', (path.stem for path in DIRS[0].glob(f'*.{EXTs[0]}'))) 69 | def test_table_to_codeblock_not_fancy(name): 70 | path = DIRS[0] / f'{name}.{EXTs[0]}' 71 | read_table_to_codeblock_not_fancy(path) 72 | 73 | 74 | def read_table_to_codeblock_str(path: Path): 75 | '''test parsing native to markdown codeblock without markdown 76 | 77 | This is lossy so we only check it runs 78 | ''' 79 | logger.info(f'Loading {path}...') 80 | with open(path, 'r') as f: 81 | text = f.read() 82 | 83 | doc = convert_text(text, input_format='native') 84 | # input files should only have 1 single outter block 85 | assert len(doc) == 1 86 | table = doc[0] 87 | pan_table = PanTable.from_panflute_ast(table) 88 | pan_table_str = pan_table.to_pantablestr() 89 | pan_table_str.to_pancodeblock() 90 | pan_table_str.to_pantable() 91 | 92 | 93 | @mark.parametrize('name', (path.stem for path in DIRS[0].glob(f'*.{EXTs[0]}'))) 94 | def test_table_to_codeblock_str(name): 95 | path = DIRS[0] / f'{name}.{EXTs[0]}' 96 | read_table_to_codeblock_str(path) 97 | -------------------------------------------------------------------------------- /tests/util_test.py: -------------------------------------------------------------------------------- 1 | from pytest import mark 2 | 3 | from pantable.util import convert_texts, convert_texts_fast, eq_panflute_elems 4 | 5 | # construct some texts cases 6 | texts_1 = [ 7 | 'some **markdown** here', 8 | 'and ~~some~~ other?' 9 | ] 10 | 11 | texts_2 = [ 12 | 'some *very* intersting markdown [example]{#so_fancy}', 13 | '''# Comical 14 | 15 | Text 16 | 17 | # Totally comical 18 | 19 | Text''' 20 | ] 21 | 22 | textss = [texts_1, texts_2, texts_1 + texts_2] 23 | 24 | # reference answers 25 | elemss = [convert_texts(texts) for texts in textss] 26 | 27 | 28 | @mark.parametrize('elems,texts', zip(elemss, textss)) 29 | def test_convert_texts_markdown_to_panflute(elems, texts): 30 | assert eq_panflute_elems(elems, convert_texts_fast(texts)) 31 | 32 | 33 | @mark.parametrize('elems,texts', zip(elemss, textss)) 34 | def test_convert_texts_panflute_to_markdown(elems, texts): 35 | assert texts == convert_texts_fast(elems, input_format='panflute', output_format='markdown') 36 | --------------------------------------------------------------------------------