├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.md ├── cython_template ├── __check_build │ ├── README.md │ ├── __init__.py │ ├── _check_build.pyx │ └── setup.py ├── __init__.py ├── example_code.pyx ├── setup.py └── tests │ ├── __init__.py │ └── test_example_code.py ├── setup.py └── tools └── cythonize.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | *.c 9 | *.cxx 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *,cover 48 | .hypothesis/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | 63 | # IPython Notebook 64 | .ipynb_checkpoints 65 | 66 | # Temporary Files 67 | *~ 68 | \#* 69 | cythonize.dat -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | # sudo false implies containerized builds 4 | sudo: false 5 | 6 | python: 7 | - 2.7 8 | - 3.4 9 | - 3.5 10 | 11 | env: 12 | global: 13 | # Directory where tests are run from 14 | - TEST_DIR=/tmp/cython_template 15 | - CONDA_DEPS="pip nose numpy cython" 16 | 17 | before_install: 18 | - export MINICONDA=$HOME/miniconda 19 | - export PATH="$MINICONDA/bin:$PATH" 20 | - hash -r 21 | # Install conda only if necessary 22 | - command -v conda >/dev/null || { wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; 23 | bash miniconda.sh -b -f -p $MINICONDA; } 24 | - conda config --set always_yes yes 25 | - conda update conda 26 | - conda info -a 27 | - conda install python=$TRAVIS_PYTHON_VERSION $CONDA_DEPS 28 | 29 | install: 30 | - python setup.py install 31 | 32 | script: 33 | - mkdir -p $TEST_DIR 34 | - cd $TEST_DIR && nosetests -v cython_template 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Jake Vanderplas 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include *.py 3 | recursive-include cython_template *.py *.pyx *.pxd 4 | recursive-include tools *.py 5 | include LICENSE 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Package Template for a Project Using Cython 2 | 3 | [![build status](http://img.shields.io/travis/jakevdp/cython_template/master.svg?style=flat)](https://travis-ci.org/jakevdp/cython_template) 4 | [![license](http://img.shields.io/badge/license-BSD-blue.svg?style=flat)](https://github.com/jakevdp/cython_template/blob/master/LICENSE) 5 | 6 | This is a package template for a Python project using Cython. There are many 7 | different ways to build cython extensions within a project, as seen in the 8 | [Cython documentation](http://docs.cython.org/src/quickstart/build.html), but 9 | the best way to integrate Cython into a distributed package is not always clear. 10 | 11 | This template borrows utility scripts used in the [scipy](http://scipy.org) 12 | and [scikit-learn](http://scikit-learn.org) projects, which give the template 13 | a few nice features: 14 | 15 | - Cython is required when the package is built, but not when the package is 16 | installed from a distributed source. That means that cython-generated C 17 | code is not included in the git repository, but *is* automatically included 18 | in any distribution. Further, it means that Cython is a requirement for 19 | package developers, but not for package users. 20 | - The ``__check_build`` submodule includes scripts which detect whether the 21 | package has been properly built, and raise useful errors if this is not the 22 | case. This is especially important for when users try to import the package 23 | from the source directory without first building/compiling the cython 24 | sources. 25 | 26 | Note that any "*.pyx" files added to the repository will have to be explicitly 27 | added to the ``setup.py`` file within the same directory: in most cases you 28 | should be able to simply copy the example [``setup.py``](https://github.com/jakevdp/cython_template/blob/master/cython_template/setup.py) within this repository. 29 | 30 | This repository has a very permissive BSD license: please feel free to 31 | use the contents to help set up your own cython-driven packages! -------------------------------------------------------------------------------- /cython_template/__check_build/README.md: -------------------------------------------------------------------------------- 1 | ``__check_build`` 2 | ================= 3 | The purpose of this submodule is to give the user a readable error when trying 4 | to import the package from within the source tree. 5 | 6 | -------------------------------------------------------------------------------- /cython_template/__check_build/__init__.py: -------------------------------------------------------------------------------- 1 | """ Module to give helpful messages to the user that did not 2 | compile package properly, 3 | 4 | This code was adapted from scikit-learn's check_build utility. 5 | """ 6 | import os 7 | 8 | PACKAGE_NAME = 'cython_template' 9 | 10 | INPLACE_MSG = """ 11 | It appears that you are importing {package} from within the source tree. 12 | Please either use an inplace install or try from another location. 13 | """.format(package=PACKAGE_NAME) 14 | 15 | STANDARD_MSG = """ 16 | If you have used an installer, please check that it is suited for your 17 | Python version, your operating system and your platform. 18 | """ 19 | 20 | ERROR_TEMPLATE = """{error} 21 | ___________________________________________________________________________ 22 | Contents of {local_dir}: 23 | {contents} 24 | ___________________________________________________________________________ 25 | It seems that the {package} has not been built correctly. 26 | 27 | If you have installed {package} from source, please do not forget 28 | to build the package before using it: run `python setup.py install` 29 | in the source directory. 30 | {msg}""" 31 | 32 | 33 | def raise_build_error(e): 34 | # Raise a comprehensible error and list the contents of the 35 | # directory to help debugging on the mailing list. 36 | local_dir = os.path.split(__file__)[0] 37 | msg = STANDARD_MSG 38 | if local_dir == "megaman/__check_build": 39 | # Picking up the local install: this will work only if the 40 | # install is an 'inplace build' 41 | msg = INPLACE_MSG 42 | dir_content = list() 43 | for i, filename in enumerate(os.listdir(local_dir)): 44 | if ((i + 1) % 3): 45 | dir_content.append(filename.ljust(26)) 46 | else: 47 | dir_content.append(filename + '\n') 48 | contents = ''.join(dir_content).strip() 49 | raise ImportError(ERROR_TEMPLATE.format(error=e, 50 | local_dir=local_dir, 51 | contents=contents, 52 | package=PACKAGE_NAME, 53 | msg=msg)) 54 | 55 | try: 56 | from ._check_build import check_build 57 | except ImportError as e: 58 | raise_build_error(e) 59 | -------------------------------------------------------------------------------- /cython_template/__check_build/_check_build.pyx: -------------------------------------------------------------------------------- 1 | # Adapted from scikit-learn __check_build script (BSD-licensed) 2 | 3 | def check_build(): 4 | return 5 | -------------------------------------------------------------------------------- /cython_template/__check_build/setup.py: -------------------------------------------------------------------------------- 1 | # Adapted from scikit-learn __check_build script (BSD-licensed) 2 | 3 | import numpy 4 | 5 | 6 | def configuration(parent_package='', top_path=None): 7 | from numpy.distutils.misc_util import Configuration 8 | config = Configuration('__check_build', parent_package, top_path) 9 | config.add_extension('_check_build', 10 | sources=['_check_build.c']) 11 | return config 12 | 13 | 14 | if __name__ == '__main__': 15 | from numpy.distutils.core import setup 16 | setup(**configuration(top_path='').todict()) 17 | -------------------------------------------------------------------------------- /cython_template/__init__.py: -------------------------------------------------------------------------------- 1 | """Template for packages using Cython""" 2 | 3 | __version__ = "1.0" 4 | 5 | # This import will check whether the cython sources have been built, 6 | # and if not will raise a useful error. 7 | from . import __check_build 8 | 9 | from .example_code import example_function 10 | -------------------------------------------------------------------------------- /cython_template/example_code.pyx: -------------------------------------------------------------------------------- 1 | def example_function(int N): 2 | """Example cython function""" 3 | cdef int count = 0 4 | for i in range(N): 5 | count += i 6 | return count 7 | -------------------------------------------------------------------------------- /cython_template/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | PACKAGE_NAME = 'cython_template' 4 | 5 | 6 | def configuration(parent_package='', top_path=None): 7 | from numpy.distutils.misc_util import Configuration 8 | 9 | config = Configuration(PACKAGE_NAME, parent_package, top_path) 10 | 11 | config.add_subpackage('__check_build') 12 | config.add_subpackage('tests') 13 | config.add_extension('example_code', 14 | sources=['example_code.c']) 15 | 16 | return config 17 | 18 | 19 | if __name__ == '__main__': 20 | from numpy.distutils.core import setup 21 | setup(**configuration(top_path='').todict()) 22 | -------------------------------------------------------------------------------- /cython_template/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakevdp/cython_template/920a64e70f35cb88991acb23a2cc051b45b10c87/cython_template/tests/__init__.py -------------------------------------------------------------------------------- /cython_template/tests/test_example_code.py: -------------------------------------------------------------------------------- 1 | from cython_template.example_code import example_function 2 | 3 | 4 | def test_example_function(): 5 | for N in range(10): 6 | assert example_function(N) == 0.5 * N * (N - 1) 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Setup script for cython template 2 | # Adapted from scikit-learn setup.py script (BSD-licensed) 3 | 4 | import io 5 | import os 6 | import re 7 | import sys 8 | import subprocess 9 | 10 | 11 | PACKAGE_NAME = 'cython_template' 12 | DESCRIPTION = "cython_template: Template for Packages using Cython" 13 | LONG_DESCRIPTION = DESCRIPTION 14 | AUTHOR = "Jake VanderPlas" 15 | AUTHOR_EMAIL = "jakevdp@uw.edu" 16 | URL = 'https://github.com/jakevdp/cython_template' 17 | DOWNLOAD_URL = 'https://github.com/jakevdp/cython_template' 18 | LICENSE = 'BSD 2-clause' 19 | 20 | 21 | def version(package, encoding='utf-8'): 22 | """Obtain the packge version from a python file e.g. pkg/__init__.py 23 | 24 | See . 25 | """ 26 | path = os.path.join(os.path.dirname(__file__), package, '__init__.py') 27 | with io.open(path, encoding=encoding) as fp: 28 | version_info = fp.read() 29 | version_match = re.search(r"""^__version__ = ['"]([^'"]*)['"]""", 30 | version_info, re.M) 31 | if not version_match: 32 | raise RuntimeError("Unable to find version string.") 33 | return version_match.group(1) 34 | 35 | 36 | def generate_cython(package): 37 | """Cythonize all sources in the package""" 38 | cwd = os.path.abspath(os.path.dirname(__file__)) 39 | print("Cythonizing sources") 40 | p = subprocess.call([sys.executable, 41 | os.path.join(cwd, 'tools', 'cythonize.py'), 42 | package], 43 | cwd=cwd) 44 | if p != 0: 45 | raise RuntimeError("Running cythonize failed!") 46 | 47 | 48 | def configuration(parent_package='',top_path=None): 49 | from numpy.distutils.misc_util import Configuration 50 | config = Configuration(None, parent_package, top_path) 51 | config.set_options(ignore_setup_xxx_py=True, 52 | assume_default_configuration=True, 53 | delegate_options_to_subpackages=True, 54 | quiet=True) 55 | 56 | config.add_subpackage(PACKAGE_NAME) 57 | 58 | return config 59 | 60 | 61 | def setup_package(): 62 | from numpy.distutils.core import setup 63 | 64 | old_path = os.getcwd() 65 | local_path = os.path.dirname(os.path.abspath(sys.argv[0])) 66 | src_path = local_path 67 | 68 | os.chdir(local_path) 69 | sys.path.insert(0, local_path) 70 | 71 | # Run build 72 | old_path = os.getcwd() 73 | os.chdir(src_path) 74 | sys.path.insert(0, src_path) 75 | 76 | cwd = os.path.abspath(os.path.dirname(__file__)) 77 | if not os.path.exists(os.path.join(cwd, 'PKG-INFO')): 78 | # Generate Cython sources, unless building from source release 79 | generate_cython(PACKAGE_NAME) 80 | 81 | try: 82 | setup(name=PACKAGE_NAME, 83 | author=AUTHOR, 84 | author_email=AUTHOR_EMAIL, 85 | url=URL, 86 | download_url=DOWNLOAD_URL, 87 | description=DESCRIPTION, 88 | long_description = LONG_DESCRIPTION, 89 | version=version(PACKAGE_NAME), 90 | license=LICENSE, 91 | configuration=configuration, 92 | classifiers=[ 93 | 'Development Status :: 4 - Beta', 94 | 'Environment :: Console', 95 | 'Intended Audience :: Science/Research', 96 | 'License :: OSI Approved :: BSD License', 97 | 'Natural Language :: English', 98 | 'Programming Language :: Python :: 2.7', 99 | 'Programming Language :: Python :: 3.4', 100 | 'Programming Language :: Python :: 3.5']) 101 | finally: 102 | del sys.path[0] 103 | os.chdir(old_path) 104 | 105 | return 106 | 107 | 108 | if __name__ == '__main__': 109 | setup_package() 110 | -------------------------------------------------------------------------------- /tools/cythonize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ cythonize 3 | 4 | Cythonize pyx files into C files as needed. 5 | 6 | Usage: cythonize [root_dir] 7 | 8 | Checks pyx files to see if they have been changed relative to their 9 | corresponding C files. If they have, then runs cython on these files to 10 | recreate the C files. 11 | 12 | The script detects changes in the pyx/pxd files using checksums 13 | [or hashes] stored in a database file 14 | 15 | Simple script to invoke Cython on all .pyx 16 | files; while waiting for a proper build system. Uses file hashes to 17 | figure out if rebuild is needed. 18 | 19 | It is called by ./setup.py sdist so that sdist package can be installed without 20 | cython 21 | 22 | Originally written by Dag Sverre Seljebotn, and adapted from statsmodel 0.6.1 23 | (Modified BSD 3-clause) 24 | 25 | We copied it for scikit-learn. 26 | 27 | Note: this script does not check any of the dependent C libraries; it only 28 | operates on the Cython .pyx files or their corresponding Cython header (.pxd) 29 | files. 30 | """ 31 | # Author: Arthur Mensch 32 | # Author: Raghav R V 33 | # 34 | # License: BSD 3 clause 35 | # see http://github.com/scikit-learn/scikit-learn 36 | 37 | 38 | from __future__ import division, print_function, absolute_import 39 | 40 | import os 41 | import re 42 | import sys 43 | import hashlib 44 | import subprocess 45 | 46 | HASH_FILE = 'cythonize.dat' 47 | 48 | 49 | # WindowsError is not defined on unix systems 50 | try: 51 | WindowsError 52 | except NameError: 53 | WindowsError = None 54 | 55 | 56 | def cythonize(cython_file, gen_file): 57 | try: 58 | from Cython.Compiler.Version import version as cython_version 59 | from distutils.version import LooseVersion 60 | if LooseVersion(cython_version) < LooseVersion('0.21'): 61 | raise Exception('Building scikit-learn requires Cython >= 0.21') 62 | 63 | except ImportError: 64 | pass 65 | 66 | flags = ['--fast-fail'] 67 | if gen_file.endswith('.cpp'): 68 | flags += ['--cplus'] 69 | 70 | try: 71 | try: 72 | rc = subprocess.call(['cython'] + 73 | flags + ["-o", gen_file, cython_file]) 74 | if rc != 0: 75 | raise Exception('Cythonizing %s failed' % cython_file) 76 | except OSError: 77 | # There are ways of installing Cython that don't result in a cython 78 | # executable on the path, see scipy issue gh-2397. 79 | rc = subprocess.call([sys.executable, '-c', 80 | 'import sys; from Cython.Compiler.Main ' 81 | 'import setuptools_main as main;' 82 | ' sys.exit(main())'] + flags + 83 | ["-o", gen_file, cython_file]) 84 | if rc != 0: 85 | raise Exception('Cythonizing %s failed' % cython_file) 86 | except OSError: 87 | raise OSError('Cython needs to be installed') 88 | 89 | 90 | def load_hashes(filename): 91 | """Load the hashes dict from the hashfile""" 92 | # { filename : (sha1 of header if available or 'NA', 93 | # sha1 of input, 94 | # sha1 of output) } 95 | 96 | hashes = {} 97 | try: 98 | with open(filename, 'r') as cython_hash_file: 99 | for hash_record in cython_hash_file: 100 | (filename, header_hash, 101 | cython_hash, gen_file_hash) = hash_record.split() 102 | hashes[filename] = (header_hash, cython_hash, gen_file_hash) 103 | except (KeyError, ValueError, AttributeError, IOError): 104 | hashes = {} 105 | return hashes 106 | 107 | 108 | def save_hashes(hashes, filename): 109 | """Save the hashes dict to the hashfile""" 110 | with open(filename, 'w') as cython_hash_file: 111 | for key, value in hashes.items(): 112 | cython_hash_file.write("%s %s %s %s\n" 113 | % (key, value[0], value[1], value[2])) 114 | 115 | 116 | def sha1_of_file(filename): 117 | h = hashlib.sha1() 118 | with open(filename, "rb") as f: 119 | h.update(f.read()) 120 | return h.hexdigest() 121 | 122 | 123 | def clean_path(path): 124 | """Clean the path""" 125 | path = path.replace(os.sep, '/') 126 | if path.startswith('./'): 127 | path = path[2:] 128 | return path 129 | 130 | 131 | def get_hash_tuple(header_path, cython_path, gen_file_path): 132 | """Get the hashes from the given files""" 133 | 134 | header_hash = (sha1_of_file(header_path) 135 | if os.path.exists(header_path) else 'NA') 136 | from_hash = sha1_of_file(cython_path) 137 | to_hash = (sha1_of_file(gen_file_path) 138 | if os.path.exists(gen_file_path) else 'NA') 139 | 140 | return header_hash, from_hash, to_hash 141 | 142 | 143 | def cythonize_if_unchanged(path, cython_file, gen_file, hashes): 144 | full_cython_path = os.path.join(path, cython_file) 145 | full_header_path = full_cython_path.replace('.pyx', '.pxd') 146 | full_gen_file_path = os.path.join(path, gen_file) 147 | 148 | current_hash = get_hash_tuple(full_header_path, full_cython_path, 149 | full_gen_file_path) 150 | 151 | if current_hash == hashes.get(clean_path(full_cython_path)): 152 | print('%s has not changed' % full_cython_path) 153 | return 154 | 155 | print('Processing %s' % full_cython_path) 156 | cythonize(full_cython_path, full_gen_file_path) 157 | 158 | # changed target file, recompute hash 159 | current_hash = get_hash_tuple(full_header_path, full_cython_path, 160 | full_gen_file_path) 161 | 162 | # Update the hashes dict with the new hash 163 | hashes[clean_path(full_cython_path)] = current_hash 164 | 165 | 166 | def check_and_cythonize(root_dir): 167 | print(root_dir) 168 | hashes = load_hashes(HASH_FILE) 169 | 170 | for cur_dir, dirs, files in os.walk(root_dir): 171 | for filename in files: 172 | if filename.endswith('.pyx'): 173 | gen_file_ext = '.c' 174 | # Cython files with libcpp imports should be compiled to cpp 175 | with open(os.path.join(cur_dir, filename), 'rb') as f: 176 | data = f.read() 177 | m = re.search(b"libcpp", data, re.I | re.M) 178 | if m: 179 | gen_file_ext = ".cpp" 180 | cython_file = filename 181 | gen_file = filename.replace('.pyx', gen_file_ext) 182 | cythonize_if_unchanged(cur_dir, cython_file, gen_file, hashes) 183 | 184 | # Save hashes once per module. This prevents cythonizing prev. 185 | # files again when debugging broken code in a single file 186 | save_hashes(hashes, HASH_FILE) 187 | 188 | 189 | def main(root_dir): 190 | check_and_cythonize(root_dir) 191 | 192 | 193 | if __name__ == '__main__': 194 | try: 195 | root_dir_arg = sys.argv[1] 196 | except IndexError: 197 | raise ValueError("Usage: python cythonize.py ") 198 | main(root_dir_arg) 199 | --------------------------------------------------------------------------------