├── .coveragerc ├── .github └── workflows │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── cppimport ├── __init__.py ├── __main__.py ├── build_module.py ├── checksum.py ├── filepaths.py ├── find.py ├── import_hook.py ├── importer.py └── templating.py ├── environment.yml ├── pyproject.toml ├── release ├── setup.cfg ├── setup.py └── tests ├── apackage ├── __init__.py ├── inner │ ├── __init__.py │ └── mymodule.cpp ├── mymodule.cpp └── rel_import_tester.py ├── conftest.py ├── cpp14module.cpp ├── extra_sources.cpp ├── extra_sources1.cpp ├── free_module.cpp ├── hook_test.cpp ├── mymodule.cpp ├── raw_extension.c ├── test_cppimport.py ├── thing.h └── thing2.h /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source=cppimport 3 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - synchronize 8 | push: 9 | branches: 10 | - main 11 | tags: 12 | - '*' 13 | jobs: 14 | test: 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | os: ["ubuntu-latest", "macos-latest"] 19 | python-version: [ "3.8", "3.9", "3.10"] 20 | name: Test (${{ matrix.python-version }}, ${{ matrix.os }}) 21 | runs-on: ${{ matrix.os }} 22 | defaults: 23 | run: 24 | shell: bash -l {0} 25 | steps: 26 | - uses: actions/checkout@v2 27 | - uses: conda-incubator/setup-miniconda@v3 28 | with: 29 | mamba-version: "*" 30 | channels: conda-forge 31 | activate-environment: cppimport 32 | environment-file: environment.yml 33 | python-version: ${{ matrix.python-version }} 34 | - name: Install cppimport 35 | run: | 36 | pip install --no-deps -e . 37 | - name: Lint with flake8 38 | run: | 39 | flake8 . 40 | - name: Check formatting with black 41 | run: | 42 | black --check . 43 | - name: Check import ordering with isort 44 | run: | 45 | isort --check . 46 | - name: Test 47 | if: ${{ matrix.os == 'macos-latest' }} 48 | run: | 49 | CFLAGS='-stdlib=libc++' pytest --cov=./ --cov-report=xml 50 | - name: Test 51 | if: ${{ matrix.os != 'macos-latest' }} 52 | run: | 53 | pytest --cov=./ --cov-report=xml 54 | build-and-publish: 55 | # based on https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ 56 | # also see: https://github.com/marketplace/actions/pypi-publish#advanced-release-management 57 | needs: test 58 | name: Build and publish Python distributions to PyPI and TestPyPI 59 | runs-on: ubuntu-latest 60 | steps: 61 | - uses: actions/checkout@v3 62 | - uses: conda-incubator/setup-miniconda@v2 63 | with: 64 | mamba-version: "*" 65 | channels: conda-forge 66 | activate-environment: cppimport 67 | environment-file: environment.yml 68 | python-version: ${{ matrix.python-version }} 69 | - name: Get history and tags for SCM versioning to work 70 | run: | 71 | git fetch --prune --unshallow 72 | git fetch --depth=1 origin +refs/tags/*:refs/tags/* 73 | - name: Build sdist 74 | run: >- 75 | python setup.py sdist 76 | - name: Publish distribution 📦 to PyPI 77 | if: startsWith(github.ref, 'refs/tags') 78 | uses: pypa/gh-action-pypi-publish@master 79 | with: 80 | password: ${{ secrets.PYPI_API_TOKEN }} 81 | verbose: true 82 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | *.so 4 | *.cppimporthash 5 | .rendered.* 6 | __pycache__ 7 | build 8 | cppimport.egg-info 9 | dist 10 | .cache 11 | .tox 12 | .mypy_cache/ 13 | .coverage 14 | htmlcov 15 | **/.DS_Store 16 | .eggs 17 | cppimport/_version.py 18 | *.lock -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 20.8b1 4 | hooks: 5 | - id: black 6 | language_version: python3 7 | - repo: https://gitlab.com/pycqa/flake8 8 | rev: 3.8.4 9 | hooks: 10 | - id: flake8 11 | - repo: https://github.com/pycqa/isort 12 | rev: 5.7.0 13 | hooks: 14 | - id: isort 15 | name: isort (python) 16 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, feel free to add an issue or pull request! There really aren't any rules, but if you're mean, I'll be sad. I'm happy to collaborate on pull requests if you would like. There's no need to submit a perfect, finished product. 4 | 5 | To install in development mode and run the tests: 6 | ``` 7 | git clone git@github.com:tbenthompson/cppimport.git 8 | cd cppimport 9 | conda env create 10 | conda activate cppimport 11 | pre-commit install 12 | pip install --no-use-pep517 --disable-pip-version-check -e . 13 | pytests 14 | ``` 15 | 16 | # Architecture 17 | 18 | ## Entrypoints: 19 | 20 | The main entrypoint for cppimport is the `cppimport.import_hook` module, which interfaces with the Python importing system to allow things like `import mycppfilename`. For a C++ file to be a valid import target, it needs to have the word `cppimport` in its first line. Without this first line constraint, it is possible for the importing system to cause imports in other Python packages to fail. Before adding the first-line constraint, the cppimport import_hook had the unfortunate consequence of breaking some scipy modules that had adjacent C and C++ files in the directory tree. 21 | 22 | There is an alternative, and more explicit interface provided by the `imp`, `imp_from_filepath` and `build` functions here. 23 | * `imp` does exactly what the import hook does except via a function so that instead of `import foomodule` we would do `foomodule = imp('foomodule')`. 24 | * `imp_from_filepath` is even more explicit, allowing the user to pass a C++ filepath rather than a modulename. For example, `foomodule = imp('../cppcodedir/foodmodule.cpp')`. This is rarely necessary but can be handy for debugging. 25 | * `build` is similar to `imp` except that the library is only built and not actually loaded as a Python module. 26 | 27 | `imp`, `imp_from_filepath` and `build` are in the `__init__.py` to separate external facing API from the guts of the package that live in internal submodules. 28 | 29 | ## What happens when we import a C++ module. 30 | 31 | 1. First the `cppimport.find.find_module_cpppath` function is used to find a C++ file that matches the desired module name. 32 | 2. Next, we determine if there's already an existing compiled extension that we can use. If there is, the `cppimport.importer.is_build_needed` function is used to determine if the extension is up to date with the current code. If the extension is up to date, we attempt to load it. If the extension is loaded successfully, we return the module and we're done! However, if for whichever reason, we can't load an existing extension, we need to build the extension, a process directed by `cppimport.importer.template_and_build`. 33 | 3. The first step of building is to run the C++ file through the Mako templating system with the `cppimport.templating.run_templating` function. The main purpose of this is to allow users to embed configuration information into their C++ file. Without some sort of similar mechanism, there would be no way of passing information to build system because the `import modulename` statement can't carry information. The templating serves a secondary benefit in that simple code generation can be performed if needed. However, most users probably stick to a simple header or footer similar to the one demonstrated in the README. 34 | 4. Next, we use setuptools to build the C++ extension using `cppimport.build_module.build_module`. This function calls setuptools with the appropriate arguments to build the extension in place next to the C++ file in the directory tree. 35 | 5. Next, we call `cppimport.checksum.checksum_save` to add a hash of the appended contents of all relevant source and header files. This checksum is appended to the end of the `.so` or `.dylib` file. This seems legal according to specifications and, in practice, causes no problems. 36 | 6. Finally, the compiled and loaded extension module is returned to the user. 37 | 38 | ## Useful links 39 | 40 | * PEP 302 that made this possible: https://www.python.org/dev/peps/pep-0302/ 41 | * The gory details of Python importing: https://docs.python.org/3/reference/import.html 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2021 T. Ben Thompson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include VERSION 3 | recursive-include tests *.py 4 | recursive-include tests *.h 5 | recursive-include tests *.cpp 6 | recursive-include tests *.c 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cppimport - Import C++ directly from Python! 2 | 3 |

4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 |

13 | 14 | ## Contributing and architecture 15 | 16 | See [CONTRIBUTING.md](CONTRIBUTING.md) for details on the internals of `cppimport` and how to get involved in development. 17 | 18 | ## Installation 19 | 20 | Install with `pip install cppimport`. 21 | 22 | ## A quick example 23 | 24 | Save the C++ code below as `somecode.cpp`. 25 | ```c++ 26 | // cppimport 27 | #include 28 | 29 | namespace py = pybind11; 30 | 31 | int square(int x) { 32 | return x * x; 33 | } 34 | 35 | PYBIND11_MODULE(somecode, m) { 36 | m.def("square", &square); 37 | } 38 | /* 39 | <% 40 | setup_pybind11(cfg) 41 | %> 42 | */ 43 | ``` 44 | 45 | Then open a Python interpreter and import the C++ extension: 46 | ```python 47 | >>> import cppimport.import_hook 48 | >>> import somecode #This will pause for a moment to compile the module 49 | >>> somecode.square(9) 50 | 81 51 | ``` 52 | 53 | Hurray, you've called some C++ code from Python using a combination of `cppimport` and [`pybind11`](https://github.com/pybind/pybind11). 54 | 55 | I'm a big fan of the workflow that this enables, where you can edit both C++ files and Python and recompilation happens transparently! It's also handy for quickly whipping together an optimized version of a slow Python function. 56 | 57 | ## An explanation 58 | 59 | Okay, now that I've hopefully convinced you on how exciting this is, let's get into the details of how to do this yourself. First, the comment at top is essential to opt in to cppimport. Don't forget this! (See below for an explanation of why this is necessary.) 60 | ```c++ 61 | // cppimport 62 | ``` 63 | 64 | The bulk of the file is a generic, simple [pybind11](https://github.com/pybind/pybind11) extension. We include the `pybind11` headers, then define a simple function that squares `x`, then export that function as part of a Python extension called `somecode`. 65 | 66 | Finally at the end of the file, there's a section I'll call the "configuration block": 67 | ``` 68 | <% 69 | setup_pybind11(cfg) 70 | %> 71 | ``` 72 | This region surrounded by `<%` and `%>` is a [Mako](https://www.makotemplates.org/) code block. The region is evaluated as Python code during the build process and provides configuration info like compiler and linker flags to the cppimport build system. 73 | 74 | Note that because of the Mako pre-processing, the comments around the configuration block may be omitted. Putting the configuration block at the end of the file, while optional, ensures that line numbers remain correct in compilation error messages. 75 | 76 | ## Building for production 77 | In production deployments you usually don't want to include a c/c++ compiler, all the sources and compile at runtime. Therefore, a simple cli utility for pre-compiling all source files is provided. This utility may, for example, be used in CI/CD pipelines. 78 | 79 | Usage is as simple as 80 | 81 | ```commandline 82 | python -m cppimport build 83 | ``` 84 | 85 | This will build all `*.c` and `*.cpp` files in the current directory (and it's subdirectories) if they are eligible to be imported (i.e. contain the `// cppimport` comment in the first line). 86 | 87 | Alternatively, you may specifiy one or more root directories or source files to be built: 88 | 89 | ```commandline 90 | python -m cppimport build ./my/directory/ ./my/single/file.cpp 91 | ``` 92 | _Note: When specifying a path to a file, the header check (`// cppimport`) is skipped for that file._ 93 | 94 | ### Fine-tuning for production 95 | To further improve startup performance for production builds, you can opt-in to skip the checksum and compiled binary existence checks during importing by either setting the environment variable `CPPIMPORT_RELEASE_MODE` to `true` or setting the configuration from within Python: 96 | ```python 97 | cppimport.settings['release_mode'] = True 98 | ``` 99 | **Warning:** Make sure to have all binaries pre-compiled when in release mode, as importing any missing ones will cause exceptions. 100 | 101 | ## Frequently asked questions 102 | 103 | ### What's actually going on? 104 | 105 | Sometimes Python just isn't fast enough. Or you have existing code in a C or C++ library. So, you write a Python *extension module*, a library of compiled code. I recommend [pybind11](https://github.com/pybind/pybind11) for C++ to Python bindings or [cffi](https://cffi.readthedocs.io/en/latest/) for C to Python bindings. I've done this a lot over the years. But, I discovered that my productivity is slower when my development process goes from *Edit -> Test* in just Python to *Edit -> Compile -> Test* in Python plus C++. So, `cppimport` combines the process of compiling and importing an extension in Python so that you can just run `import foobar` and not have to worry about multiple steps. Internally, `cppimport` looks for a file `foobar.cpp`. Assuming one is found, it's run through the Mako templating system to gather compiler options, then it's compiled and loaded as an extension module. 106 | 107 | ### Does cppimport recompile every time a module is imported? 108 | No! Compilation should only happen the first time the module is imported. The C++ source is compared with a checksum on each import to determine if any relevant file has changed. Additional dependencies (e.g. header files!) can be tracked by adding to the Mako header: 109 | ```python 110 | cfg['dependencies'] = ['file1.h', 'file2.h'] 111 | ``` 112 | The checksum is computed by simply appending the contents of the extension C++ file together with the files in `cfg['sources']` and `cfg['dependencies']`. 113 | 114 | ### How can I set compiler or linker args? 115 | 116 | Standard distutils configuration options are valid: 117 | 118 | ```python 119 | cfg['extra_link_args'] = ['...'] 120 | cfg['extra_compile_args'] = ['...'] 121 | cfg['libraries'] = ['...'] 122 | cfg['include_dirs'] = ['...'] 123 | ``` 124 | 125 | For example, to use C++11, add: 126 | ```python 127 | cfg['extra_compile_args'] = ['-std=c++11'] 128 | ``` 129 | 130 | ### How can I split my extension across multiple source files? 131 | 132 | In the configuration block: 133 | ```python 134 | cfg['sources'] = ['extra_source1.cpp', 'extra_source2.cpp'] 135 | ``` 136 | 137 | ### cppimport isn't doing what I want, can I get more verbose output? 138 | `cppimport` uses the standard Python logging tools. Please add logging handlers to either the root logger or the `"cppimport"` logger. For example, to output all debug level log messages: 139 | 140 | ```python 141 | root_logger = logging.getLogger() 142 | root_logger.setLevel(logging.DEBUG) 143 | 144 | handler = logging.StreamHandler(sys.stdout) 145 | handler.setLevel(logging.DEBUG) 146 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 147 | handler.setFormatter(formatter) 148 | root_logger.addHandler(handler) 149 | ``` 150 | 151 | ### How can I force a rebuild even when the checksum matches? 152 | 153 | Set: 154 | ```python 155 | cppimport.settings['force_rebuild'] = True 156 | ``` 157 | 158 | And if this is a common occurence, I would love to hear your use case and why the combination of the checksum, `cfg['dependencies']` and `cfg['sources']` is insufficient! 159 | 160 | Note that `force_rebuild` does not work when importing the module concurrently. 161 | 162 | ### Can I import my model concurrently? 163 | 164 | It's (mostly) safe to use `cppimport` to import a module concurrently using multiple threads, processes or even machines! 165 | There's an exception if your filesystem does not support file locking - see the next section. 166 | 167 | Before building a module, `cppimport` obtains a lockfile preventing other processors from building it at the same time - this prevents clashes that can lead to failure. 168 | Other processes will wait maximum 10 mins until the first process has built the module and load it. If your module does not build within 10 mins then it will timeout. 169 | You can increase the timeout time in the settings: 170 | 171 | ```python 172 | cppimport.settings['lock_timeout'] = 10*60 # 10 mins 173 | ``` 174 | 175 | You should not use `force_rebuild` when importing concurrently. 176 | 177 | ### Acquiring the lock hangs or times out unexpectedly - what's going on? 178 | Certain platforms (e.g. those running 179 | a Data Virtualization Service, DVS) do not support file locking. If you're on Linux with access to `flock`, you can test whether 180 | locking is supported (credit to [this page](https://help.univention.com/t/howto-verify-the-mounted-filesystem-supports-file-locking/10149)): 181 | 182 | ```bash 183 | touch testfile 184 | flock ./testfile true && echo ok || echo nok 185 | ``` 186 | 187 | If locking is not supported, you can disable the file lock in 188 | the cppimport global settings: 189 | 190 | ```python 191 | cppimport.settings['use_filelock'] = False 192 | ``` 193 | 194 | This setting must be changed before you import any 195 | code. By setting `use_filelock=False`, you become responsible 196 | for ensuring that only a single process 197 | (re)builds the package at a time. For example: if you're 198 | using [mpi4py](https://mpi4py.readthedocs.io/en/stable/) 199 | to run independent, communicating processes, here's how 200 | to protect the build: 201 | 202 | ```python 203 | from mpi4py import MPI 204 | import cppimport, cppimport.import_hook 205 | cppimport.settings["use_filelock"] = False 206 | 207 | pid = MPI.COMM_WORLD.Get_rank() 208 | 209 | if pid == 0: 210 | import somecode # Process 0 compiles extension if needed 211 | MPI.COMM_WORLD.Barrier() # Remaining processes wait 212 | import somecode # All processes use compiled extension 213 | ``` 214 | 215 | ### How can I get information about filepaths in the configuration block? 216 | The module name is available as the `fullname` variable and the C++ module file is available as `filepath`. 217 | For example, 218 | ``` 219 | <% 220 | module_dir = os.path.dirname(filepath) 221 | %> 222 | ``` 223 | 224 | ### How can I make compilation faster? 225 | 226 | In single file extensions, this is a fundamental issue with C++. Heavily templated code is often quite slow to compile. 227 | 228 | If your extension has multiple source files using the `cfg['sources']` capability, then you might be hoping for some kind of incremental compilation. For the uninitiated, incremental compilation involves only recompiling those source files that have changed. Unfortunately this isn't possible because cppimport is built on top of the setuptools and distutils and these standard library components do not support incremental compilation. 229 | 230 | I recommend following the suggestions on [this SO answer](http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils). That is: 231 | 232 | 1. Use `ccache` to reduce the cost of rebuilds 233 | 2. Enable parallel compilation. This can be done with `cfg['parallel'] = True` in the C++ file's configuration header. 234 | 235 | As a further thought, if your extension has many source files and you're hoping to do incremental compiles, that probably indicates that you've outgrown `cppimport` and should consider using a more complete build system like CMake. 236 | 237 | ### Why does the import hook need "cppimport" on the first line of the .cpp file? 238 | Modifying the Python import system is a global modification and thus affects all imports from any other package. As a result, when I first implemented `cppimport`, other packages (e.g. `scipy`) suddenly started breaking because import statements internal to those packages were importing C or C++ files instead of the modules they were intended to import. To avoid this failure mode, the import hook uses an "opt in" system where C and C++ files can specify they are meant to be used with cppimport by having a comment on the first line that includes the text "cppimport". 239 | 240 | As an alternative to the import hook, you can use `imp` or `imp_from_filepath`. The `cppimport.imp` and `cppimport.imp_from_filepath` performs exactly the same operation as the import hook but in a slightly more explicit way: 241 | ``` 242 | foobar = cppimport.imp("foobar") 243 | foobar = cppimport.imp_from_filepath("src/foobar.cpp") 244 | ``` 245 | By default, these explicit function do not require the "cppimport" keyword on the first line of the C++ source file. 246 | 247 | ### Windows? 248 | The CI system does not run on Windows. A PR would be welcome adding further Windows support. I've used `cppimport` with MinGW-w64 and Python 3.6 and had good success. I've also had reports that `cppimport` works on Windows with Python 3.6 and Visual C++ 2015 Build Tools. The main challenge is making sure that distutils is aware of your available compilers. Try out the suggestion [here](https://stackoverflow.com/questions/3297254/how-to-use-mingws-gcc-compiler-when-installing-python-package-using-pip). 249 | 250 | ## cppimport uses the MIT License 251 | -------------------------------------------------------------------------------- /cppimport/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | See CONTRIBUTING.md for a description of the project structure and the internal logic. 3 | """ 4 | 5 | import ctypes 6 | import logging 7 | import os 8 | 9 | from cppimport.find import _check_first_line_contains_cppimport 10 | 11 | try: 12 | from ._version import version as __version__ 13 | from ._version import version_tuple 14 | except ImportError: 15 | __version__ = "unknown version" 16 | version_tuple = (0, 0, "unknown version") 17 | 18 | settings = dict( 19 | force_rebuild=False, # `force_rebuild` with multiple processes is not supported 20 | file_exts=[".cpp", ".c"], 21 | rtld_flags=ctypes.RTLD_LOCAL, 22 | use_filelock=True, # Filelock if multiple processes try to build simultaneously 23 | lock_suffix=".lock", 24 | lock_timeout=10 * 60, 25 | remove_strict_prototypes=True, 26 | release_mode=os.getenv("CPPIMPORT_RELEASE_MODE", "0").lower() 27 | in ("true", "yes", "1"), 28 | ) 29 | _logger = logging.getLogger("cppimport") 30 | 31 | 32 | def imp(fullname, opt_in=False): 33 | """ 34 | `imp` is the explicit alternative to using cppimport.import_hook. 35 | 36 | Parameters 37 | ---------- 38 | fullname : the name of the module to import. 39 | opt_in : should we require C++ files to opt in via adding "cppimport" to 40 | the first line of the file? This is on by default for the 41 | import hook, but is off by default for this function since the 42 | intent to import a C++ module is clearly specified. 43 | 44 | Returns 45 | ------- 46 | module : the compiled and loaded Python extension module 47 | """ 48 | from cppimport.find import find_module_cpppath 49 | 50 | # Search through sys.path to find a file that matches the module 51 | filepath = find_module_cpppath(fullname, opt_in) 52 | return imp_from_filepath(filepath, fullname) 53 | 54 | 55 | def imp_from_filepath(filepath, fullname=None): 56 | """ 57 | `imp_from_filepath` serves the same purpose as `imp` except allows 58 | specifying the exact filepath of the C++ file. 59 | 60 | Parameters 61 | ---------- 62 | filepath : the filepath to the C++ file to build and import. 63 | fullname : the name of the module to import. This can be different from the 64 | module name inferred from the filepath if desired. 65 | 66 | Returns 67 | ------- 68 | module : the compiled and loaded Python extension module 69 | """ 70 | from cppimport.importer import ( 71 | build_safely, 72 | is_build_needed, 73 | load_module, 74 | setup_module_data, 75 | try_load, 76 | ) 77 | 78 | filepath = os.path.abspath(filepath) 79 | if fullname is None: 80 | fullname = os.path.splitext(os.path.basename(filepath))[0] 81 | module_data = setup_module_data(fullname, filepath) 82 | # The call to try_load is necessary here because there are times when the 83 | # only evidence a rebuild is needed comes from attempting to load an 84 | # existing extension module. For example, if the extension was built on a 85 | # different architecture or with different Python headers and will produce 86 | # an error when loaded, then the load will fail. In that situation, we will 87 | # need to rebuild. 88 | if is_build_needed(module_data) or not try_load(module_data): 89 | build_safely(filepath, module_data) 90 | load_module(module_data) 91 | return module_data["module"] 92 | 93 | 94 | def build(fullname): 95 | """ 96 | `build` builds a extension module like `imp` but does not import the 97 | extension. 98 | 99 | Parameters 100 | ---------- 101 | fullname : the name of the module to import. 102 | 103 | Returns 104 | ------- 105 | ext_path : the path to the compiled extension. 106 | """ 107 | from cppimport.find import find_module_cpppath 108 | 109 | # Search through sys.path to find a file that matches the module 110 | filepath = find_module_cpppath(fullname) 111 | return build_filepath(filepath, fullname=fullname) 112 | 113 | 114 | def build_filepath(filepath, fullname=None): 115 | """ 116 | `build_filepath` builds a extension module like `build` but allows 117 | to directly specify a file path. 118 | 119 | Parameters 120 | ---------- 121 | filepath : the filepath to the C++ file to build. 122 | fullname : the name of the module to build. 123 | 124 | Returns 125 | ------- 126 | ext_path : the path to the compiled extension. 127 | """ 128 | from cppimport.importer import ( 129 | build_safely, 130 | is_build_needed, 131 | load_module, 132 | setup_module_data, 133 | ) 134 | 135 | filepath = os.path.abspath(filepath) 136 | if fullname is None: 137 | fullname = os.path.splitext(os.path.basename(filepath))[0] 138 | module_data = setup_module_data(fullname, filepath) 139 | if is_build_needed(module_data): 140 | build_safely(filepath, module_data) 141 | load_module(module_data) 142 | # Return the path to the built module 143 | return module_data["ext_path"] 144 | 145 | 146 | def build_all(root_directory): 147 | """ 148 | `build_all` builds a extension module like `build` for each eligible (that is, 149 | containing the "cppimport" header) source file within the given `root_directory`. 150 | 151 | Parameters 152 | ---------- 153 | root_directory : the root directory to search for cpp source files in. 154 | """ 155 | for directory, _, files in os.walk(root_directory): 156 | for file in files: 157 | if ( 158 | not file.startswith(".") 159 | and os.path.splitext(file)[1] in settings["file_exts"] 160 | ): 161 | full_path = os.path.join(directory, file) 162 | if _check_first_line_contains_cppimport(full_path): 163 | _logger.info(f"Building: {full_path}") 164 | build_filepath(full_path) 165 | 166 | 167 | ######## BACKWARDS COMPATIBILITY ######### 168 | # Below here, we pay penance for mistakes. 169 | # TODO: Add DeprecationWarning 170 | 171 | """ 172 | For backwards compatibility, support this alias for the imp function 173 | """ 174 | cppimport = imp 175 | 176 | 177 | def force_rebuild(to=True): 178 | settings["force_rebuild"] = to 179 | 180 | 181 | def turn_off_strict_prototypes(): 182 | pass # turned off by default. 183 | 184 | 185 | def set_rtld_flags(flags): 186 | settings["rtld_flags"] = flags 187 | -------------------------------------------------------------------------------- /cppimport/__main__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import os 4 | import sys 5 | 6 | from cppimport import build_all, build_filepath, settings 7 | 8 | 9 | def _run_from_commandline(raw_args): 10 | parser = argparse.ArgumentParser("cppimport") 11 | 12 | parser.add_argument( 13 | "--verbose", "-v", action="store_true", help="Increase log verbosity." 14 | ) 15 | parser.add_argument( 16 | "--quiet", "-q", action="store_true", help="Only print critical log messages." 17 | ) 18 | 19 | subparsers = parser.add_subparsers(dest="action") 20 | subparsers.required = True 21 | 22 | build_parser = subparsers.add_parser( 23 | "build", 24 | help="Build one or more cpp source files.", 25 | ) 26 | build_parser.add_argument( 27 | "root", 28 | help="The file or directory to build. If a directory is given, " 29 | "cppimport walks it recursively to build all eligible source " 30 | "files.", 31 | nargs="*", 32 | ) 33 | build_parser.add_argument( 34 | "--force", "-f", action="store_true", help="Force rebuild." 35 | ) 36 | 37 | args = parser.parse_args(raw_args[1:]) 38 | 39 | if args.quiet: 40 | logging.basicConfig(level=logging.CRITICAL) 41 | elif args.verbose: 42 | logging.basicConfig(level=logging.DEBUG) 43 | else: 44 | logging.basicConfig(level=logging.INFO) 45 | 46 | if args.action == "build": 47 | if args.force: 48 | settings["force_rebuild"] = True 49 | 50 | for path in args.root or ["."]: 51 | path = os.path.abspath(os.path.expandvars(path)) 52 | if os.path.isfile(path): 53 | build_filepath(path) 54 | elif os.path.isdir(path): 55 | build_all(path or os.getcwd()) 56 | else: 57 | raise FileNotFoundError( 58 | f'The given root path "{path}" could not be found.' 59 | ) 60 | else: 61 | parser.print_usage() 62 | 63 | 64 | if __name__ == "__main__": 65 | _run_from_commandline(sys.argv) 66 | -------------------------------------------------------------------------------- /cppimport/build_module.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import io 3 | import logging 4 | import os 5 | import shutil 6 | import tempfile 7 | 8 | import setuptools 9 | import setuptools.command.build_ext 10 | 11 | import distutils 12 | import distutils.sysconfig 13 | 14 | import cppimport 15 | from cppimport.filepaths import make_absolute 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def build_module(module_data): 21 | _handle_strict_prototypes() 22 | 23 | build_path = tempfile.mkdtemp() 24 | 25 | full_module_name = module_data["fullname"] 26 | filepath = module_data["filepath"] 27 | cfg = module_data["cfg"] 28 | 29 | module_data["abs_include_dirs"] = [ 30 | make_absolute(module_data["filedirname"], d) 31 | for d in cfg.get("include_dirs", []) 32 | ] + [os.path.dirname(filepath)] 33 | module_data["abs_library_dirs"] = [ 34 | make_absolute(module_data["filedirname"], d) 35 | for d in cfg.get("library_dirs", []) 36 | ] 37 | module_data["dependency_dirs"] = module_data["abs_include_dirs"] + [ 38 | module_data["filedirname"] 39 | ] 40 | module_data["extra_source_filepaths"] = [ 41 | make_absolute(module_data["filedirname"], s) for s in cfg.get("sources", []) 42 | ] 43 | 44 | ext = ImportCppExt( 45 | os.path.dirname(filepath), 46 | full_module_name, 47 | language="c++", 48 | sources=( 49 | module_data["extra_source_filepaths"] 50 | + [module_data["rendered_src_filepath"]] 51 | ), 52 | include_dirs=module_data["abs_include_dirs"], 53 | extra_compile_args=cfg.get("extra_compile_args", []), 54 | extra_link_args=cfg.get("extra_link_args", []), 55 | library_dirs=module_data["abs_library_dirs"], 56 | libraries=cfg.get("libraries", []), 57 | ) 58 | 59 | args = [ 60 | "build_ext", 61 | "--inplace", 62 | "--build-temp=" + build_path, 63 | "--build-lib=" + build_path, 64 | "-v", 65 | ] 66 | 67 | setuptools_args = dict( 68 | name=full_module_name, 69 | ext_modules=[ext], 70 | script_args=args, 71 | cmdclass={"build_ext": BuildImportCppExt}, 72 | ) 73 | 74 | # Monkey patch in the parallel compiler if requested. 75 | # TODO: this will still cause problems if there is multithreaded code 76 | # interacting with distutils. Ideally, we'd just subclass CCompiler 77 | # instead. 78 | if cfg.get("parallel"): 79 | old_compile = distutils.ccompiler.CCompiler.compile 80 | distutils.ccompiler.CCompiler.compile = _parallel_compile 81 | 82 | f = io.StringIO() 83 | with contextlib.redirect_stdout(f): 84 | with contextlib.redirect_stderr(f): 85 | setuptools.setup(**setuptools_args) 86 | logger.debug(f"Setuptools/compiler output: {f.getvalue()}") 87 | 88 | # Remove the parallel compiler to not corrupt the outside environment. 89 | if cfg.get("parallel"): 90 | distutils.ccompiler.CCompiler.compile = old_compile 91 | 92 | shutil.rmtree(build_path) 93 | 94 | 95 | def _handle_strict_prototypes(): 96 | if not cppimport.settings["remove_strict_prototypes"]: 97 | return 98 | 99 | cfg_vars = distutils.sysconfig.get_config_vars() 100 | for key, value in cfg_vars.items(): 101 | if value is str: 102 | cfg_vars[key] = value.replace("-Wstrict-prototypes", "") 103 | 104 | 105 | class ImportCppExt(setuptools.Extension): 106 | """ 107 | Subclass setuptools.Extension to add self.libdest specifying where the shared 108 | library should be placed after being compiled with BuildImportCppExt. 109 | """ 110 | 111 | def __init__(self, libdest, *args, **kwargs): 112 | self.libdest = libdest 113 | setuptools.Extension.__init__(self, *args, **kwargs) 114 | 115 | 116 | class BuildImportCppExt(setuptools.command.build_ext.build_ext): 117 | """ 118 | Subclass setuptools build_ext to put the compiled shared library in the 119 | appropriate place in the source tree from the ImportCppExt.libdest value. 120 | """ 121 | 122 | def copy_extensions_to_source(self): 123 | for ext in self.extensions: 124 | fullname = self.get_ext_fullname(ext.name) 125 | filename = self.get_ext_filename(fullname) 126 | src_filename = os.path.join(self.build_lib, filename) 127 | dest_filename = os.path.join(ext.libdest, os.path.basename(filename)) 128 | 129 | distutils.file_util.copy_file( 130 | src_filename, dest_filename, verbose=self.verbose, dry_run=self.dry_run 131 | ) 132 | 133 | 134 | # Patch for parallel compilation with distutils 135 | # From: http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils # noqa: E501 136 | def _parallel_compile( 137 | self, 138 | sources, 139 | output_dir=None, 140 | macros=None, 141 | include_dirs=None, 142 | debug=0, 143 | extra_preargs=None, 144 | extra_postargs=None, 145 | depends=None, 146 | ): 147 | # these lines are copied directly from distutils.ccompiler.CCompiler 148 | macros, objects, extra_postargs, pp_opts, build = self._setup_compile( 149 | output_dir, macros, include_dirs, sources, depends, extra_postargs 150 | ) 151 | cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) 152 | 153 | # Determine the number of compilation threads. Unless there are special 154 | # circumstances, this is the number of cores on the machine 155 | N = 1 156 | try: 157 | import multiprocessing 158 | import multiprocessing.pool 159 | 160 | N = multiprocessing.cpu_count() 161 | except (ImportError, NotImplementedError): 162 | pass 163 | 164 | def _single_compile(obj): 165 | try: 166 | src, ext = build[obj] 167 | except KeyError: 168 | return 169 | self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) 170 | 171 | # imap is evaluated on demand, converting to list() forces execution 172 | list(multiprocessing.pool.ThreadPool(N).imap(_single_compile, objects)) 173 | return objects 174 | -------------------------------------------------------------------------------- /cppimport/checksum.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import json 3 | import logging 4 | import struct 5 | 6 | from cppimport.filepaths import make_absolute 7 | 8 | _TAG = b"cppimport" 9 | _FMT = struct.Struct("q" + str(len(_TAG)) + "s") 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | def is_checksum_valid(module_data): 15 | """ 16 | Load the saved checksum from the extension file check if it matches the 17 | checksum computed from current source files. 18 | """ 19 | deps, old_checksum = _load_checksum_trailer(module_data) 20 | if old_checksum is None: 21 | return False # Already logged error in load_checksum_trailer. 22 | try: 23 | return old_checksum == _calc_cur_checksum(deps, module_data) 24 | except OSError as e: 25 | logger.info( 26 | "Checksummed file not found while checking cppimport checksum " 27 | "(%s); rebuilding." % e 28 | ) 29 | return False 30 | 31 | 32 | def _load_checksum_trailer(module_data): 33 | try: 34 | with open(module_data["ext_path"], "rb") as f: 35 | f.seek(-_FMT.size, 2) 36 | json_len, tag = _FMT.unpack(f.read(_FMT.size)) 37 | if tag != _TAG: 38 | logger.info( 39 | "The extension is missing the trailer tag and thus is missing" 40 | " its checksum; rebuilding." 41 | ) 42 | return None, None 43 | f.seek(-(_FMT.size + json_len), 2) 44 | json_s = f.read(json_len) 45 | except FileNotFoundError: 46 | logger.info("Failed to find compiled extension; rebuilding.") 47 | return None, None 48 | except OSError: 49 | logger.info("Checksum trailer invalid. Rebuilding.") 50 | return None, None 51 | 52 | try: 53 | deps, old_checksum = json.loads(json_s) 54 | except ValueError: 55 | logger.info( 56 | "Failed to load checksum trailer info from already existing " 57 | "compiled extension; rebuilding." 58 | ) 59 | return None, None 60 | return deps, old_checksum 61 | 62 | 63 | def checksum_save(module_data): 64 | """ 65 | Calculate the module checksum and then write it to the end of the shared 66 | object. 67 | """ 68 | dep_filepaths = ( 69 | [ 70 | make_absolute(module_data["filedirname"], d) 71 | for d in module_data["cfg"].get("dependencies", []) 72 | ] 73 | + module_data["extra_source_filepaths"] 74 | + [module_data["filepath"]] 75 | ) 76 | cur_checksum = _calc_cur_checksum(dep_filepaths, module_data) 77 | _save_checksum_trailer(module_data, dep_filepaths, cur_checksum) 78 | 79 | 80 | def _save_checksum_trailer(module_data, dep_filepaths, cur_checksum): 81 | # We can just append the checksum to the shared object; this is effectively 82 | # legal (see e.g. https://stackoverflow.com/questions/10106447). 83 | dump = json.dumps([dep_filepaths, cur_checksum]).encode("ascii") 84 | dump += _FMT.pack(len(dump), _TAG) 85 | with open(module_data["ext_path"], "ab", buffering=0) as file: 86 | file.write(dump) 87 | 88 | 89 | def _calc_cur_checksum(file_lst, module_data): 90 | text = b"" 91 | for filepath in file_lst: 92 | with open(filepath, "rb") as f: 93 | text += f.read() 94 | return hashlib.md5(text).hexdigest() 95 | -------------------------------------------------------------------------------- /cppimport/filepaths.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def make_absolute(this_dir, s): 5 | if os.path.isabs(s): 6 | return s 7 | else: 8 | return os.path.join(this_dir, s) 9 | -------------------------------------------------------------------------------- /cppimport/find.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | 5 | import cppimport 6 | from cppimport.filepaths import make_absolute 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def find_module_cpppath(modulename, opt_in=False): 12 | filepath = _find_module_cpppath(modulename, opt_in) 13 | if filepath is None: 14 | raise ImportError( 15 | "Couldn't find a file matching the module name: " 16 | + str(modulename) 17 | + " (opt_in = " 18 | + str(opt_in) 19 | + ")" 20 | ) 21 | return filepath 22 | 23 | 24 | def _find_module_cpppath(modulename, opt_in=False): 25 | modulepath_without_ext = modulename.replace(".", os.sep) 26 | moduledir = os.path.dirname(modulepath_without_ext + ".throwaway") 27 | matching_dirs = _find_matching_path_dirs(moduledir) 28 | abs_matching_dirs = _make_dirs_absolute(matching_dirs) 29 | 30 | for ext in cppimport.settings["file_exts"]: 31 | modulefilename = os.path.basename(modulepath_without_ext + ext) 32 | outfilename = _find_file_in_folders(modulefilename, abs_matching_dirs, opt_in) 33 | if outfilename is not None: 34 | return outfilename 35 | 36 | return None 37 | 38 | 39 | def _make_dirs_absolute(dirs): 40 | out = [] 41 | for d in dirs: 42 | if d == "": 43 | d = os.getcwd() 44 | out.append(make_absolute(os.getcwd(), d)) 45 | return out 46 | 47 | 48 | def _find_matching_path_dirs(moduledir): 49 | if moduledir == "": 50 | return sys.path 51 | 52 | ds = [] 53 | for dir in sys.path: 54 | test_path = os.path.join(dir, moduledir) 55 | if os.path.exists(test_path) and os.path.isdir(test_path): 56 | ds.append(test_path) 57 | return ds 58 | 59 | 60 | def _find_file_in_folders(filename, paths, opt_in): 61 | for d in paths: 62 | if not os.path.exists(d): 63 | continue 64 | 65 | if os.path.isfile(d): 66 | continue 67 | 68 | for f in os.listdir(d): 69 | if f != filename: 70 | continue 71 | filepath = os.path.join(d, f) 72 | if opt_in and not _check_first_line_contains_cppimport(filepath): 73 | logger.debug( 74 | "Found file but the first line doesn't " 75 | "contain cppimport so it will be skipped: " + filepath 76 | ) 77 | continue 78 | return filepath 79 | return None 80 | 81 | 82 | def _check_first_line_contains_cppimport(filepath): 83 | with open(filepath, "rb") as f: 84 | return b"cppimport" in f.readline() 85 | -------------------------------------------------------------------------------- /cppimport/import_hook.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | import traceback 4 | 5 | import cppimport 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class Hook(object): 11 | def __init__(self): 12 | self._running = False 13 | 14 | def find_spec(self, fullname, path, target=None): 15 | # Prevent re-entry by the underlying importer 16 | if self._running: 17 | return 18 | 19 | try: 20 | self._running = True 21 | cppimport.imp(fullname, opt_in=True) 22 | except ImportError: 23 | # ImportError should be quashed because that simply means cppimport 24 | # didn't find anything, and probably shouldn't have found anything! 25 | logger.debug(traceback.format_exc()) 26 | finally: 27 | self._running = False 28 | 29 | 30 | # Add the hook to the list of import handlers for Python. 31 | hook_obj = Hook() 32 | sys.meta_path.insert(0, hook_obj) 33 | -------------------------------------------------------------------------------- /cppimport/importer.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import logging 3 | import os 4 | import sys 5 | import sysconfig 6 | from contextlib import suppress 7 | from time import sleep, time 8 | 9 | import filelock 10 | 11 | import cppimport 12 | from cppimport.build_module import build_module 13 | from cppimport.checksum import checksum_save, is_checksum_valid 14 | from cppimport.templating import run_templating 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class add_to_sys_path: 20 | """A Context Manager to temporary add a path to sys.path""" 21 | 22 | def __init__(self, path): 23 | self.path = path 24 | 25 | def __enter__(self): 26 | sys.path.insert(0, self.path) 27 | 28 | def __exit__(self, exc_type, exc_value, traceback): 29 | try: 30 | sys.path.remove(self.path) 31 | except ValueError: 32 | pass 33 | 34 | 35 | def build_safely(filepath, module_data): 36 | """Protect against race conditions when multiple processes executing 37 | `template_and_build`""" 38 | binary_path = module_data["ext_path"] 39 | lock_path = binary_path + cppimport.settings["lock_suffix"] 40 | 41 | def build_completed(): 42 | return os.path.exists(binary_path) and is_checksum_valid(module_data) 43 | 44 | t = time() 45 | 46 | if cppimport.settings["use_filelock"]: 47 | # Race to obtain the lock and build. Other processes can wait 48 | while not build_completed() and time() - t < cppimport.settings["lock_timeout"]: 49 | try: 50 | with filelock.FileLock(lock_path, timeout=1): 51 | if build_completed(): 52 | break 53 | template_and_build(filepath, module_data) 54 | except filelock.Timeout: 55 | logging.debug(f"Could not obtain lock (pid {os.getpid()})") 56 | if cppimport.settings["force_rebuild"]: 57 | raise ValueError( 58 | "force_build must be False to build concurrently." 59 | "This process failed to claim a filelock indicating that" 60 | " a concurrent build is in progress" 61 | ) 62 | sleep(1) 63 | 64 | if os.path.exists(lock_path): 65 | with suppress(OSError): 66 | os.remove(lock_path) 67 | 68 | if not build_completed(): 69 | raise Exception( 70 | f"Could not compile binary as lock already taken and timed out." 71 | f" Try increasing the timeout setting if " 72 | f"the build time is longer (pid {os.getpid()})." 73 | ) 74 | else: 75 | template_and_build(filepath, module_data) 76 | 77 | 78 | def template_and_build(filepath, module_data): 79 | logger.debug(f"Compiling {filepath}.") 80 | run_templating(module_data) 81 | build_module(module_data) 82 | checksum_save(module_data) 83 | 84 | 85 | def setup_module_data(fullname, filepath): 86 | module_data = dict() 87 | module_data["fullname"] = fullname 88 | module_data["filepath"] = filepath 89 | module_data["filedirname"] = os.path.dirname(module_data["filepath"]) 90 | module_data["filebasename"] = os.path.basename(module_data["filepath"]) 91 | module_data["ext_name"] = get_module_name(fullname) + get_extension_suffix() 92 | module_data["ext_path"] = os.path.join( 93 | os.path.dirname(filepath), module_data["ext_name"] 94 | ) 95 | return module_data 96 | 97 | 98 | def get_module_name(full_module_name): 99 | return full_module_name.split(".")[-1] 100 | 101 | 102 | def get_extension_suffix(): 103 | ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") 104 | if ext_suffix is None: 105 | ext_suffix = sysconfig.get_config_var("SO") 106 | return ext_suffix 107 | 108 | 109 | def _actually_load_module(module_data): 110 | with add_to_sys_path(module_data["filedirname"]): 111 | module_data["module"] = importlib.import_module(module_data["fullname"]) 112 | 113 | 114 | def load_module(module_data): 115 | if hasattr(sys, "getdlopenflags"): 116 | # It can be useful to set rtld_flags to RTLD_GLOBAL. This allows 117 | # extensions that are loaded later to share the symbols from this 118 | # extension. This is primarily useful in a project where several 119 | # interdependent extensions are loaded but it's undesirable to combine 120 | # the multiple extensions into a single extension. 121 | old_flags = sys.getdlopenflags() 122 | new_flags = old_flags | cppimport.settings["rtld_flags"] 123 | sys.setdlopenflags(new_flags) 124 | _actually_load_module(module_data) 125 | sys.setdlopenflags(old_flags) 126 | else: 127 | _actually_load_module(module_data) 128 | 129 | 130 | def is_build_needed(module_data): 131 | if cppimport.settings["force_rebuild"]: 132 | return True 133 | if cppimport.settings["release_mode"]: 134 | logger.debug( 135 | f"Release mode is enabled. Thus, file {module_data['filepath']} is " 136 | f"not being compiled." 137 | ) 138 | return False 139 | if not is_checksum_valid(module_data): 140 | return True 141 | logger.debug(f"Matching checksum for {module_data['filepath']} --> not compiling") 142 | return False 143 | 144 | 145 | def try_load(module_data): 146 | """Try loading the module to test if it's not corrupt and for the correct 147 | architecture""" 148 | try: 149 | load_module(module_data) 150 | return True 151 | except ImportError as e: 152 | logger.info( 153 | f"ImportError during import with matching checksum: {e}. Trying to rebuild." 154 | ) 155 | with suppress(OSError): 156 | os.remove(module_data["fullname"]) 157 | return False 158 | -------------------------------------------------------------------------------- /cppimport/templating.py: -------------------------------------------------------------------------------- 1 | import io 2 | import logging 3 | import os 4 | 5 | import mako.exceptions 6 | import mako.lookup 7 | import mako.runtime 8 | import mako.template 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def run_templating(module_data): 14 | module_data["cfg"] = BuildArgs( 15 | sources=[], 16 | include_dirs=[], 17 | extra_compile_args=[], 18 | libraries=[], 19 | library_dirs=[], 20 | extra_link_args=[], 21 | dependencies=[], 22 | parallel=False, 23 | ) 24 | module_data["setup_pybind11"] = setup_pybind11 25 | buf = io.StringIO() 26 | ctx = mako.runtime.Context(buf, **module_data) 27 | 28 | filepath = module_data["filepath"] 29 | try: 30 | template_dirs = [os.path.dirname(filepath)] 31 | lookup = mako.lookup.TemplateLookup(directories=template_dirs) 32 | tmpl = lookup.get_template(module_data["filebasename"]) 33 | tmpl.render_context(ctx) 34 | except: # noqa: E722 35 | logger.exception(mako.exceptions.text_error_template().render()) 36 | raise 37 | 38 | rendered_src_filepath = get_rendered_source_filepath(filepath) 39 | 40 | with open(rendered_src_filepath, "w", newline="") as f: 41 | f.write(buf.getvalue()) 42 | 43 | module_data["rendered_src_filepath"] = rendered_src_filepath 44 | 45 | 46 | class BuildArgs(dict): 47 | """ 48 | This exists for backwards compatibility with old configuration key names. 49 | TODO: Add deprecation warnings to allow removing this sometime in the future. 50 | """ 51 | 52 | _key_mapping = { 53 | "compiler_args": "extra_compile_args", 54 | "linker_args": "extra_link_args", 55 | } 56 | 57 | def __getitem__(self, key): 58 | return super(BuildArgs, self).__getitem__(self._key_mapping.get(key, key)) 59 | 60 | def __setitem__(self, key, value): 61 | super(BuildArgs, self).__setitem__(self._key_mapping.get(key, key), value) 62 | 63 | 64 | def setup_pybind11(cfg): 65 | import pybind11 66 | 67 | cfg["include_dirs"] += [pybind11.get_include(), pybind11.get_include(True)] 68 | # Prefix with c++11 arg instead of suffix so that if a user specifies c++14 69 | # (or later!) then it won't be overridden. 70 | cfg["compiler_args"] = ["-std=c++11", "-fvisibility=hidden"] + cfg["compiler_args"] 71 | 72 | 73 | def get_rendered_source_filepath(filepath): 74 | dirname = os.path.dirname(filepath) 75 | filename = os.path.basename(filepath) 76 | return os.path.join(dirname, ".rendered." + filename) 77 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: cppimport 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - pybind11 6 | - mako 7 | - black 8 | - regex>=2021 9 | - flake8 10 | - isort 11 | - pytest 12 | - pytest-cov 13 | - pre-commit 14 | - filelock 15 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.isort] 2 | profile = "black" 3 | # distutils is leaving stdlib. setuptools provides a compatibility import, but needs to be imported first 4 | known_thirdparty = ["distutils"] 5 | known_setuptools = ["setuptools"] 6 | sections = ["FUTURE", "STDLIB", "SETUPTOOLS", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"] 7 | 8 | [tool.pytest.ini_options] 9 | addopts = "-s --tb=short" 10 | 11 | [build-system] 12 | requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] 13 | build-backend = "setuptools.build_meta" 14 | 15 | [tool.setuptools_scm] 16 | version_scheme = "post-release" 17 | write_to = "cppimport/_version.py" 18 | -------------------------------------------------------------------------------- /release: -------------------------------------------------------------------------------- 1 | GIT: 2 | git commit -m "yy.mm.dd" 3 | git tag yy.mm.dd 4 | git push --atomic origin main yy.mm.dd 5 | wait for github action to complete 6 | create release on github 7 | 8 | SANITY TEST: 9 | open new terminal 10 | mamba create -n testenv python=3 pip 11 | conda activate testenv 12 | pip install --force-reinstall --no-cache cppimport 13 | cd tests 14 | python -c 'import cppimport; assert(cppimport.imp("mymodule").add(1,2) == 3);' 15 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203, E266 4 | per-file-ignores= 5 | cppimport/__init__.py:F401 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | description = open("README.md").read() 4 | 5 | setup( 6 | use_scm_version={"version_scheme": "post-release"}, 7 | setup_requires=["setuptools_scm"], 8 | packages=["cppimport"], 9 | install_requires=["mako", "pybind11", "filelock"], 10 | zip_safe=False, 11 | name="cppimport", 12 | description="Import C++ files directly from Python!", 13 | long_description=description, 14 | long_description_content_type="text/markdown", 15 | url="https://github.com/tbenthompson/cppimport", 16 | author="T. Ben Thompson", 17 | author_email="t.ben.thompson@gmail.com", 18 | license="MIT", 19 | platforms=["any"], 20 | classifiers=[ 21 | "Development Status :: 5 - Production/Stable", 22 | "Intended Audience :: Developers", 23 | "Operating System :: OS Independent", 24 | "Operating System :: POSIX", 25 | "Topic :: Software Development", 26 | "License :: OSI Approved :: MIT License", 27 | "Programming Language :: Python :: 3", 28 | "Programming Language :: C++", 29 | ], 30 | ) 31 | -------------------------------------------------------------------------------- /tests/apackage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbenthompson/cppimport/0c4ad1c94cae6e560f8094bbec8a4e9737724fdd/tests/apackage/__init__.py -------------------------------------------------------------------------------- /tests/apackage/inner/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbenthompson/cppimport/0c4ad1c94cae6e560f8094bbec8a4e9737724fdd/tests/apackage/inner/__init__.py -------------------------------------------------------------------------------- /tests/apackage/inner/mymodule.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | <% 3 | import pybind11 4 | cfg['compiler_args'] = ['-std=c++11'] 5 | cfg['include_dirs'] = [pybind11.get_include(), pybind11.get_include(True)] 6 | %> 7 | */ 8 | #include 9 | 10 | namespace py = pybind11; 11 | 12 | int add(int i, int j) { 13 | return i + j; 14 | } 15 | 16 | PYBIND11_PLUGIN(mymodule) { 17 | pybind11::module m("mymodule", "auto-compiled c++ extension"); 18 | m.def("add", &add); 19 | return m.ptr(); 20 | } 21 | -------------------------------------------------------------------------------- /tests/apackage/mymodule.cpp: -------------------------------------------------------------------------------- 1 | /* cppimport 2 | <% 3 | import pybind11 4 | cfg['compiler_args'] = ['-std=c++11'] 5 | cfg['include_dirs'] = [pybind11.get_include(), pybind11.get_include(True)] 6 | %> 7 | */ 8 | #include 9 | 10 | namespace py = pybind11; 11 | 12 | int add(int i, int j) { 13 | return i + j; 14 | } 15 | 16 | PYBIND11_PLUGIN(mymodule) { 17 | pybind11::module m("mymodule", "auto-compiled c++ extension"); 18 | m.def("add", &add); 19 | return m.ptr(); 20 | } 21 | -------------------------------------------------------------------------------- /tests/apackage/rel_import_tester.py: -------------------------------------------------------------------------------- 1 | from . import mymodule 2 | 3 | 4 | def f(): 5 | return mymodule.add(1, 2) 6 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | def pytest_addoption(parser): 2 | parser.addoption( 3 | "--multiprocessing", 4 | action="store_true", 5 | dest="multiprocessing", 6 | default=False, 7 | help="enable multiprocessing tests with filelock", 8 | ) 9 | -------------------------------------------------------------------------------- /tests/cpp14module.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | <% 3 | import pybind11 4 | cfg['compiler_args'] = ['-std=c++14'] 5 | cfg['include_dirs'] = [pybind11.get_include(), pybind11.get_include(True)] 6 | %> 7 | */ 8 | #include 9 | 10 | namespace py = pybind11; 11 | 12 | // Use auto instead of int to check C++14 13 | auto add(int i, int j) { 14 | return i + j; 15 | } 16 | 17 | PYBIND11_MODULE(cpp14module, m) { 18 | m.def("add", &add); 19 | } 20 | -------------------------------------------------------------------------------- /tests/extra_sources.cpp: -------------------------------------------------------------------------------- 1 | <% 2 | setup_pybind11(cfg) 3 | cfg['sources'] = ['extra_sources1.cpp'] 4 | cfg['parallel'] = True 5 | %> 6 | #include 7 | 8 | int square(int x); 9 | 10 | int square_sum(int x, int y) { 11 | return square(x) + square(y); 12 | } 13 | 14 | PYBIND11_MODULE(extra_sources, m) { 15 | m.def("square_sum", &square_sum); 16 | } 17 | -------------------------------------------------------------------------------- /tests/extra_sources1.cpp: -------------------------------------------------------------------------------- 1 | int square(int x) { 2 | return x * x; 3 | } 4 | -------------------------------------------------------------------------------- /tests/free_module.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { 4 | std::cout << "HI!" << std::endl; 5 | } 6 | -------------------------------------------------------------------------------- /tests/hook_test.cpp: -------------------------------------------------------------------------------- 1 | /*cppimport*/ 2 | <% 3 | setup_pybind11(cfg) 4 | %> 5 | #include 6 | 7 | PYBIND11_MODULE(hook_test, m) { 8 | m.def("sub", [] (int i, int j) { return i - j; } ); 9 | } 10 | -------------------------------------------------------------------------------- /tests/mymodule.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | <% 3 | setup_pybind11(cfg) 4 | cfg['dependencies'] = ['thing.h'] 5 | %> 6 | */ 7 | #include 8 | #include "thing.h" 9 | #include "thing2.h" 10 | 11 | namespace py = pybind11; 12 | 13 | int add(int i, int j) { 14 | return i + j; 15 | } 16 | 17 | PYBIND11_MODULE(mymodule, m) { 18 | m.def("add", &add); 19 | #ifdef THING_DEFINED 20 | #pragma message "stuff" 21 | py::class_(m, "Thing") 22 | .def(py::init<>()) 23 | .def("cheer", &Thing::cheer); 24 | #endif 25 | } 26 | -------------------------------------------------------------------------------- /tests/raw_extension.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #if PY_MAJOR_VERSION >= 3 4 | #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) 5 | #define MOD_DEF(ob, name, doc, methods) \ 6 | static struct PyModuleDef moduledef = { \ 7 | PyModuleDef_HEAD_INIT, name, doc, -1, methods, }; \ 8 | ob = PyModule_Create(&moduledef); 9 | #define MOD_SUCCESS_VAL(val) val 10 | #else 11 | #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) 12 | #define MOD_DEF(ob, name, doc, methods) \ 13 | ob = Py_InitModule3(name, methods, doc); 14 | #define MOD_SUCCESS_VAL(val) 15 | #endif 16 | 17 | static PyObject* add(PyObject* self, PyObject* args) { 18 | int a, b; 19 | int class = 1; 20 | if (!PyArg_ParseTuple(args, "ii", &a, &b)) { 21 | return NULL; 22 | } 23 | return Py_BuildValue("i", a + b); 24 | } 25 | 26 | static PyMethodDef methods[] = { 27 | {"add", add, METH_VARARGS, ""}, 28 | {NULL} 29 | }; 30 | 31 | MOD_INIT(raw_extension) { 32 | PyObject* m; 33 | MOD_DEF(m, "raw_extension", "", methods) 34 | return MOD_SUCCESS_VAL(m); 35 | } 36 | -------------------------------------------------------------------------------- /tests/test_cppimport.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import copy 3 | import logging 4 | import os 5 | import shutil 6 | import subprocess 7 | import sys 8 | from multiprocessing import Process 9 | from tempfile import TemporaryDirectory 10 | 11 | import pytest 12 | 13 | import cppimport 14 | import cppimport.build_module 15 | import cppimport.templating 16 | from cppimport.find import find_module_cpppath 17 | 18 | cppimport.settings["use_filelock"] = False # Filelock only enabled for multiprocessing 19 | multiprocessing_enable = pytest.mark.skipif("not config.getoption('multiprocessing')") 20 | 21 | root_logger = logging.getLogger() 22 | root_logger.setLevel(logging.DEBUG) 23 | 24 | handler = logging.StreamHandler(sys.stdout) 25 | handler.setLevel(logging.DEBUG) 26 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 27 | handler.setFormatter(formatter) 28 | root_logger.addHandler(handler) 29 | 30 | 31 | @contextlib.contextmanager 32 | def appended(filename, text): 33 | with open(filename, "r") as f: 34 | orig = f.read() 35 | with open(filename, "a") as f: 36 | f.write(text) 37 | try: 38 | yield 39 | finally: 40 | with open(filename, "w") as f: 41 | f.write(orig) 42 | 43 | 44 | def subprocess_check(test_code, returncode=0): 45 | p = subprocess.run( 46 | [sys.executable, "-c", test_code], 47 | cwd=os.path.dirname(__file__), 48 | stdout=subprocess.PIPE, 49 | stderr=subprocess.PIPE, 50 | ) 51 | if len(p.stdout) > 0: 52 | print(p.stdout.decode("utf-8")) 53 | if len(p.stderr) > 0: 54 | print(p.stderr.decode("utf-8")) 55 | assert p.returncode == returncode 56 | 57 | 58 | @contextlib.contextmanager 59 | def tmp_dir(files=None): 60 | """Create a temporary directory and copy `files` into it. `files` can also 61 | include directories.""" 62 | files = files if files else [] 63 | 64 | with TemporaryDirectory() as tmp_path: 65 | for f in files: 66 | if os.path.isdir(f): 67 | shutil.copytree(f, os.path.join(tmp_path, os.path.basename(f))) 68 | else: 69 | shutil.copyfile(f, os.path.join(tmp_path, os.path.basename(f))) 70 | yield tmp_path 71 | 72 | 73 | def test_find_module_cpppath(): 74 | mymodule_loc = find_module_cpppath("mymodule") 75 | mymodule_dir = os.path.dirname(mymodule_loc) 76 | assert os.path.basename(mymodule_loc) == "mymodule.cpp" 77 | 78 | apackage = find_module_cpppath("apackage.mymodule") 79 | apackage_correct = os.path.join(mymodule_dir, "apackage", "mymodule.cpp") 80 | assert apackage == apackage_correct 81 | 82 | inner = find_module_cpppath("apackage.inner.mymodule") 83 | inner_correct = os.path.join(mymodule_dir, "apackage", "inner", "mymodule.cpp") 84 | assert inner == inner_correct 85 | 86 | 87 | def test_get_rendered_source_filepath(): 88 | rendered_path = cppimport.templating.get_rendered_source_filepath("abc.cpp") 89 | assert rendered_path == ".rendered.abc.cpp" 90 | 91 | 92 | def module_tester(mod, cheer=False): 93 | assert mod.add(1, 2) == 3 94 | if cheer: 95 | mod.Thing().cheer() 96 | 97 | 98 | def test_mymodule(): 99 | mymodule = cppimport.imp("mymodule") 100 | module_tester(mymodule) 101 | 102 | 103 | def test_mymodule_build(): 104 | cppimport.build("mymodule") 105 | import mymodule 106 | 107 | module_tester(mymodule) 108 | 109 | 110 | def test_mymodule_from_filepath(): 111 | mymodule = cppimport.imp_from_filepath("tests/mymodule.cpp") 112 | module_tester(mymodule) 113 | 114 | 115 | def test_package_mymodule(): 116 | mymodule = cppimport.imp("apackage.mymodule") 117 | module_tester(mymodule) 118 | 119 | 120 | def test_inner_package_mymodule(): 121 | mymodule = cppimport.imp("apackage.inner.mymodule") 122 | module_tester(mymodule) 123 | 124 | 125 | def test_with_file_in_syspath(): 126 | orig_sys_path = copy.copy(sys.path) 127 | sys.path.append(os.path.join(os.path.dirname(__file__), "mymodule.cpp")) 128 | cppimport.imp("mymodule") 129 | sys.path = orig_sys_path 130 | 131 | 132 | def test_rebuild_after_failed_compile(): 133 | cppimport.imp("mymodule") 134 | test_code = """ 135 | import cppimport; 136 | cppimport.settings["use_filelock"] = False; 137 | mymodule = cppimport.imp("mymodule"); 138 | assert(mymodule.add(1,2) == 3) 139 | """ 140 | with appended("tests/mymodule.cpp", ";asdf;"): 141 | subprocess_check(test_code, 1) 142 | subprocess_check(test_code, 0) 143 | 144 | 145 | add_to_thing = """ 146 | #include 147 | struct Thing { 148 | void cheer() { 149 | std::cout << "WAHHOOOO" << std::endl; 150 | } 151 | }; 152 | #define THING_DEFINED 153 | """ 154 | 155 | 156 | def test_no_rebuild_if_no_deps_change(): 157 | cppimport.imp("mymodule") 158 | test_code = """ 159 | import cppimport; 160 | cppimport.settings["use_filelock"] = False; 161 | mymodule = cppimport.imp("mymodule"); 162 | assert(not hasattr(mymodule, 'Thing')) 163 | """ 164 | with appended("tests/thing2.h", add_to_thing): 165 | subprocess_check(test_code) 166 | 167 | 168 | def test_rebuild_header_after_change(): 169 | cppimport.imp("mymodule") 170 | test_code = """ 171 | import cppimport; 172 | cppimport.settings["use_filelock"] = False; 173 | mymodule = cppimport.imp("mymodule"); 174 | mymodule.Thing().cheer() 175 | """ 176 | with appended("tests/thing.h", add_to_thing): 177 | subprocess_check(test_code) 178 | assert open("tests/thing.h", "r").read() == "" 179 | 180 | 181 | def test_raw_extensions(): 182 | raw_extension = cppimport.imp("raw_extension") 183 | assert raw_extension.add(1, 2) == 3 184 | 185 | 186 | def test_extra_sources_and_parallel(): 187 | cppimport.settings["force_rebuild"] = True 188 | mod = cppimport.imp("extra_sources") 189 | cppimport.settings["force_rebuild"] = False 190 | assert mod.square_sum(3, 4) == 25 191 | 192 | 193 | def test_import_hook(): 194 | import cppimport.import_hook 195 | 196 | # Force rebuild to make sure we're not just reloading the already compiled 197 | # module from disk 198 | cppimport.force_rebuild(True) 199 | import hook_test 200 | 201 | cppimport.force_rebuild(False) 202 | assert hook_test.sub(3, 1) == 2 203 | 204 | 205 | def test_submodule_import_hook(): 206 | import cppimport.import_hook 207 | 208 | # Force rebuild to make sure we're not just reloading the already compiled 209 | # module from disk 210 | cppimport.force_rebuild(True) 211 | import apackage.mymodule 212 | 213 | cppimport.force_rebuild(False) 214 | assert apackage.mymodule.add(3, 1) == 4 215 | 216 | 217 | def test_relative_import(): 218 | import cppimport.import_hook 219 | 220 | cppimport.force_rebuild(True) 221 | from apackage.rel_import_tester import f 222 | 223 | cppimport.force_rebuild(False) 224 | print(f()) 225 | assert f() == 3 226 | 227 | 228 | @multiprocessing_enable 229 | def test_multiple_processes(): 230 | """ 231 | Only runs if the flag --multiprocessing is passed to 232 | pytest. This function requires file locking enabled. 233 | """ 234 | with tmp_dir(["tests/hook_test.cpp"]) as tmp_path: 235 | test_code = f""" 236 | import os; 237 | os.chdir('{tmp_path}'); 238 | import cppimport.import_hook; 239 | import hook_test; 240 | """ 241 | processes = [ 242 | Process(target=subprocess_check, args=(test_code,)) for i in range(100) 243 | ] 244 | 245 | for p in processes: 246 | p.start() 247 | 248 | for p in processes: 249 | p.join() 250 | 251 | assert all(p.exitcode == 0 for p in processes) 252 | -------------------------------------------------------------------------------- /tests/thing.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tbenthompson/cppimport/0c4ad1c94cae6e560f8094bbec8a4e9737724fdd/tests/thing.h -------------------------------------------------------------------------------- /tests/thing2.h: -------------------------------------------------------------------------------- 1 | 2 | --------------------------------------------------------------------------------