├── .codecov.yml ├── .github ├── dependabot.yml └── workflows │ ├── main.yml │ ├── publish.yml │ └── testtests.yml ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── LICENSE.md ├── MANIFEST.in ├── Makefile ├── README.md ├── ci ├── install-julia.sh ├── install_pycall.py └── test-upload │ └── tox.ini ├── conftest.py ├── docs ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── api.rst │ ├── conf.py │ ├── development.rst │ ├── how_it_works.rst │ ├── index.rst │ ├── installation.rst │ ├── limitations.rst │ ├── pytest.rst │ ├── sysimage.rst │ ├── testing.rst │ ├── troubleshooting.rst │ └── usage.rst ├── setup.cfg ├── setup.py ├── src └── julia │ ├── __init__.py │ ├── api.py │ ├── compile.jl │ ├── core.py │ ├── find_libpython.py │ ├── install-packagecompiler.jl │ ├── install.jl │ ├── ipy │ ├── __init__.py │ ├── monkeypatch_completer.py │ ├── monkeypatch_interactiveshell.py │ └── revise.py │ ├── julia_py.py │ ├── juliainfo.jl │ ├── juliainfo.py │ ├── libjulia.py │ ├── magic.py │ ├── options.py │ ├── patch.jl │ ├── precompile.jl │ ├── pseudo_python_cli.py │ ├── pyjulia_helper.jl │ ├── pytestplugin.py │ ├── python_jl.py │ ├── release.py │ ├── runtests.py │ ├── sysimage.py │ ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── test_compatible_exe.py │ ├── test_core.py │ ├── test_find_libpython.py │ ├── test_install.py │ ├── test_juliainfo.py │ ├── test_juliaoptions.py │ ├── test_libjulia.py │ ├── test_magic.py │ ├── test_options.py │ ├── test_plugin.py │ ├── test_pseudo_python_cli.py │ ├── test_python_jl.py │ ├── test_runtests.py │ ├── test_sysimage.py │ ├── test_tools.py │ ├── test_utils.py │ └── utils.py │ ├── tools.py │ ├── utils.py │ └── with_rebuilt.py └── tox.ini /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: off 4 | patch: off 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 2 | 3 | version: 2 4 | updates: 5 | - package-ecosystem: "github-actions" 6 | directory: "/" 7 | schedule: 8 | interval: "daily" 9 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Main workflow 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags: '*' 8 | pull_request: 9 | workflow_dispatch: 10 | 11 | jobs: 12 | test: 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | matrix: 16 | os: 17 | - ubuntu-latest 18 | - macos-latest 19 | - windows-latest 20 | architecture: [x64, x86] 21 | python-version: 22 | - '3.9' 23 | - '3.12' 24 | julia-version: 25 | - '1.6' 26 | - '1.9' 27 | - '1' 28 | exclude: 29 | - os: ubuntu-latest 30 | architecture: x86 31 | - os: macos-latest 32 | architecture: x86 33 | - os: macos-latest 34 | julia-version: '1.6' 35 | - os: windows-latest 36 | julia-version: '1.6' 37 | - os: macos-latest 38 | julia-version: 'nightly' 39 | - os: windows-latest 40 | julia-version: 'nightly' 41 | include: 42 | # Python 3.8 for testing `test_compiled_modules_no`: 43 | - os: ubuntu-latest 44 | architecture: x64 45 | python-version: '3.8' 46 | julia-version: '1' 47 | - os: macos-latest 48 | architecture: x64 49 | python-version: '3.8' 50 | julia-version: '1' 51 | - os: windows-latest 52 | architecture: x64 53 | python-version: '3.8' 54 | julia-version: '1' 55 | fail-fast: false 56 | name: Test 57 | py${{ matrix.python-version }} 58 | jl${{ matrix.julia-version }} 59 | ${{ matrix.os }} ${{ matrix.architecture }} 60 | steps: 61 | - uses: actions/checkout@v4 62 | - name: Setup python 63 | uses: actions/setup-python@v5 64 | with: 65 | python-version: ${{ matrix.python-version }} 66 | architecture: ${{ matrix.architecture }} 67 | - name: Setup julia 68 | uses: julia-actions/setup-julia@v1 69 | with: 70 | version: ${{ matrix.julia-version }} 71 | arch: ${{ matrix.architecture }} 72 | - run: python src/julia/find_libpython.py --list-all --verbose 73 | - name: Install tox 74 | run: | 75 | python -m pip install --upgrade pip 76 | python -m pip install --upgrade tox 77 | - name: Install PyCall 78 | run: python ci/install_pycall.py 79 | - name: Run test 80 | run: python -m tox -- --verbose --cov=julia 81 | id: tox-tests 82 | continue-on-error: ${{ matrix.julia-version == 'nightly' || (matrix.os == 'windows-latest' && matrix.architecture == 'x86') }} 83 | env: 84 | CI: 'true' # run tests marked by @only_in_ci 85 | TOXENV: py 86 | PYJULIA_TEST_REBUILD: 'yes' 87 | - run: cat .tox/py/log/pytest.log 88 | if: always() 89 | - name: Upload coverage to Codecov 90 | uses: codecov/codecov-action@v3 91 | if: steps.tox-tests.outcome == 'success' 92 | with: 93 | file: ./coverage.xml 94 | name: codecov-umbrella 95 | - name: Report allowed failures 96 | if: steps.tox-tests.outcome != 'success' 97 | run: echo "Allowed failure for this configuration." 98 | 99 | check: 100 | runs-on: ubuntu-latest 101 | strategy: 102 | matrix: 103 | toxenv: ['style', 'doc'] 104 | fail-fast: false 105 | name: Check ${{ matrix.toxenv }} 106 | steps: 107 | - uses: actions/checkout@v4 108 | - name: Setup python 109 | uses: actions/setup-python@v5 110 | with: 111 | python-version: '3.10' 112 | - run: python -m pip install --upgrade tox 113 | - run: python -m tox -e ${{ matrix.toxenv }} 114 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI 2 | 3 | on: 4 | push: 5 | branches: 6 | - release/test-uploaded 7 | - release/test 8 | - release/main 9 | workflow_dispatch: 10 | 11 | jobs: 12 | publish-to-testpypi: 13 | runs-on: ubuntu-20.04 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Set up Python 3.9 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: 3.9 20 | - name: Install setuptools and wheel 21 | run: | 22 | python -m pip install --user setuptools wheel 23 | - name: Build a binary wheel and a source tarball 24 | run: | 25 | python setup.py bdist_wheel 26 | python setup.py sdist 27 | # TODO: switch to pep517 28 | - name: Publish distribution 📦 to Test PyPI 29 | if: >- 30 | (github.ref == 'refs/heads/release/test') || 31 | (github.ref == 'refs/heads/release/main') 32 | uses: pypa/gh-action-pypi-publish@37f50c210e3d2f9450da2cd423303d6a14a6e29f # v1.5.1 33 | with: 34 | password: ${{ secrets.test_pypi_password }} 35 | repository_url: https://test.pypi.org/legacy/ 36 | skip_existing: true 37 | 38 | test-uploaded: 39 | needs: publish-to-testpypi 40 | runs-on: ubuntu-20.04 41 | steps: 42 | - uses: actions/checkout@v4 43 | - name: Set up Python 3.9 44 | uses: actions/setup-python@v5 45 | with: 46 | python-version: 3.9 47 | - name: Set up Julia 48 | uses: julia-actions/setup-julia@v1 49 | with: 50 | version: '1.6' 51 | - name: Install tox 52 | run: python -m pip install --user tox 53 | - run: cd ci/test-upload && python -m tox 54 | 55 | publish-to-pypi: 56 | needs: test-uploaded 57 | runs-on: ubuntu-20.04 58 | strategy: 59 | matrix: # using `matrix` to define a constant 60 | package: ['julia==0.6.2'] 61 | steps: 62 | - name: Set up Python 3.9 63 | uses: actions/setup-python@v5 64 | with: 65 | python-version: 3.9 66 | - name: Install setuptools and wheel 67 | run: | 68 | python -m pip install --user setuptools wheel 69 | - name: Download from TestPyPI 70 | run: | 71 | pip download --dest dist --no-deps --index-url https://test.pypi.org/simple/ ${{ matrix.package }} 72 | pip download --dest dist --no-deps --index-url https://test.pypi.org/simple/ ${{ matrix.package }} --no-binary :all: 73 | - run: ls -lh dist 74 | - name: Publish distribution 📦 to PyPI 75 | if: github.ref == 'refs/heads/release/main' 76 | uses: pypa/gh-action-pypi-publish@37f50c210e3d2f9450da2cd423303d6a14a6e29f # v1.5.1 77 | with: 78 | password: ${{ secrets.pypi_password }} 79 | -------------------------------------------------------------------------------- /.github/workflows/testtests.yml: -------------------------------------------------------------------------------- 1 | name: Test tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags: '*' 8 | pull_request: 9 | workflow_dispatch: 10 | 11 | jobs: 12 | test__using_default_setup: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Setup python 17 | uses: actions/setup-python@v5 18 | - run: python -m pip install --upgrade tox 19 | - run: python -m tox -- --no-julia -k test__using_default_setup 20 | env: 21 | PYJULIA_TEST_RUNTIME: dummy 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .pytest_cache 28 | .tox 29 | coverage.xml 30 | nosetests.xml 31 | 32 | # Translations 33 | *.mo 34 | 35 | # Mr Developer 36 | .mr.developer.cfg 37 | .project 38 | .pydevproject 39 | 40 | .ropeproject 41 | # PyCharm 42 | .idea/* 43 | 44 | # created by distutils during build process 45 | MANIFEST 46 | 47 | # Mac Os 48 | .DS_Store 49 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_first_party = julia 3 | default_section = THIRDPARTY 4 | 5 | # Black-compatible setting. See: https://github.com/ambv/black 6 | multi_line_output = 3 7 | include_trailing_comma = True 8 | force_grid_wrap = 0 9 | use_parentheses = True 10 | line_length = 88 11 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/mirrors-isort 3 | rev: v4.3.17 4 | hooks: 5 | - id: isort 6 | - repo: https://github.com/ambv/black 7 | rev: 19.3b0 8 | hooks: 9 | - id: black 10 | # See: 11 | # * https://black.readthedocs.io/en/stable/version_control_integration.html 12 | # * https://github.com/ambv/black/blob/master/.pre-commit-hooks.yaml 13 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright © 2013 by Steven G. Johnson, Fernando Perez, Jeff Bezanson, Stefan Karpinski, Keno Fischer, Jake Bolewski, Takafumi Arakaki, and other contributors. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.md 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Yes python packaging sucks. 2 | # I hope that a makefile 3 | # will help everyone to update the pakage. 4 | # don't foret to bump the version number in setup.py 5 | 6 | .PHONY: dist 7 | 8 | clean: 9 | rm -rf build 10 | rm -rf dist 11 | 12 | dist: clean 13 | # source distribution 14 | python3 setup.py sdist 15 | # 'compiled' distribution 16 | # you mignt need to `pip3 install wheel` 17 | python3 setup.py bdist_wheel 18 | 19 | upload: dist 20 | # upload to Python package index` 21 | twine upload dist/* 22 | 23 | download-testpypi: dist/.download-testpypi 24 | dist/.download-testpypi: 25 | rm -rf dist 26 | pip download --dest dist --no-deps --index-url https://test.pypi.org/simple/ julia 27 | pip download --dest dist --no-deps --index-url https://test.pypi.org/simple/ julia --no-binary :all: 28 | touch $@ 29 | 30 | release: dist/.download-testpypi 31 | twine upload dist/julia-*-any.whl dist/julia-*.tar.gz 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PyJulia 2 | ======= 3 | 4 | > [!CAUTION] 5 | > Ongoing development of the Python/Julia interface has transitioned to [PythonCall.jl/juliacall](https://github.com/JuliaPy/PythonCall.jl), please consider using that package instead. 6 | 7 | [![Stable documentation](https://img.shields.io/badge/docs-stable-blue.svg)](https://pyjulia.readthedocs.io/en/stable/) 8 | [![Latest documentation](https://img.shields.io/badge/docs-latest-blue.svg)](https://pyjulia.readthedocs.io/en/latest/) 9 | [![Main workflow](https://github.com/JuliaPy/pyjulia/workflows/Main%20workflow/badge.svg)](https://github.com/JuliaPy/pyjulia/actions?query=workflow%3A%22Main+workflow%22) 10 | [![DOI](https://zenodo.org/badge/14576985.svg)](https://zenodo.org/badge/latestdoi/14576985) 11 | 12 | Experimenting with developing a better interface to [Julia language](https://julialang.org/) that works with [Python](https://www.python.org/) 3 and Julia v1.4+. 13 | 14 | Quick usage 15 | ----------- 16 | 17 | ```console 18 | $ python3 -m pip install julia # install PyJulia 19 | ... # you may need `--user` after `install` 20 | 21 | $ python3 22 | >>> import julia 23 | >>> julia.install() # install PyCall.jl etc. 24 | >>> from julia import Base # short demo 25 | >>> Base.sind(90) 26 | 1.0 27 | ``` 28 | 29 | See more in the [documentation](https://pyjulia.readthedocs.io). 30 | -------------------------------------------------------------------------------- /ci/install-julia.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # install julia vX.Y.Z: ./install-julia.sh X.Y.Z 3 | # install julia nightly: ./install-julia.sh nightly 4 | 5 | # LICENSE 6 | # 7 | # Copyright © 2013 by Steven G. Johnson, Fernando Perez, Jeff 8 | # Bezanson, Stefan Karpinski, Keno Fischer, Jake Bolewski, Takafumi 9 | # Arakaki, and other contributors. 10 | # 11 | # Permission is hereby granted, free of charge, to any person obtaining 12 | # a copy of this software and associated documentation files (the 13 | # "Software"), to deal in the Software without restriction, including 14 | # without limitation the rights to use, copy, modify, merge, publish, 15 | # distribute, sublicense, and/or sell copies of the Software, and to 16 | # permit persons to whom the Software is furnished to do so, subject to 17 | # the following conditions: 18 | # 19 | # The above copyright notice and this permission notice shall be 20 | # included in all copies or substantial portions of the Software. 21 | # 22 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 23 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 24 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 25 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 26 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 27 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 28 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 29 | 30 | # stop on error 31 | set -e 32 | VERSION="$1" 33 | 34 | case "$VERSION" in 35 | nightly) 36 | BASEURL="https://julialangnightlies-s3.julialang.org/bin" 37 | JULIANAME="julia-latest" 38 | ;; 39 | [0-9]*.[0-9]*.[0-9]*) 40 | BASEURL="https://julialang-s3.julialang.org/bin" 41 | SHORTVERSION="$(echo "$VERSION" | grep -Eo '^[0-9]+\.[0-9]+')" 42 | JULIANAME="$SHORTVERSION/julia-$VERSION" 43 | ;; 44 | [0-9]*.[0-9]) 45 | BASEURL="https://julialang-s3.julialang.org/bin" 46 | SHORTVERSION="$(echo "$VERSION" | grep -Eo '^[0-9]+\.[0-9]+')" 47 | JULIANAME="$SHORTVERSION/julia-$VERSION-latest" 48 | ;; 49 | *) 50 | echo "Unrecognized VERSION=$VERSION, exiting" 51 | exit 1 52 | ;; 53 | esac 54 | 55 | case $(uname) in 56 | Linux) 57 | case $(uname -m) in 58 | x86_64) 59 | ARCH="x64" 60 | case "$JULIANAME" in 61 | julia-latest) 62 | SUFFIX="linux64" 63 | ;; 64 | *) 65 | SUFFIX="linux-x86_64" 66 | ;; 67 | esac 68 | ;; 69 | i386 | i486 | i586 | i686) 70 | ARCH="x86" 71 | case "$JULIANAME" in 72 | julia-latest) 73 | SUFFIX="linux32" 74 | ;; 75 | *) 76 | SUFFIX="linux-i686" 77 | ;; 78 | esac 79 | ;; 80 | *) 81 | echo "Do not have Julia binaries for this architecture, exiting" 82 | exit 1 83 | ;; 84 | esac 85 | echo "$BASEURL/linux/$ARCH/$JULIANAME-$SUFFIX.tar.gz" 86 | curl -L "$BASEURL/linux/$ARCH/$JULIANAME-$SUFFIX.tar.gz" | tar -xz 87 | sudo ln -s $PWD/julia-*/bin/julia /usr/local/bin/julia 88 | ;; 89 | Darwin) 90 | if [ -e /usr/local/bin/julia ]; then 91 | echo "/usr/local/bin/julia already exists, exiting" 92 | exit 1 93 | elif [ -e julia.dmg ]; then 94 | echo "julia.dmg already exists, exiting" 95 | exit 1 96 | elif [ -e ~/julia ]; then 97 | echo "~/julia already exists, exiting" 98 | exit 1 99 | fi 100 | curl -Lo julia.dmg "$BASEURL/mac/x64/$JULIANAME-mac64.dmg" 101 | hdiutil mount -mountpoint /Volumes/Julia julia.dmg 102 | cp -Ra /Volumes/Julia/*.app/Contents/Resources/julia ~ 103 | ln -s ~/julia/bin/julia /usr/local/bin/julia 104 | # TODO: clean up after self? 105 | ;; 106 | *) 107 | echo "Do not have Julia binaries for this platform, exiting" 108 | exit 1 109 | ;; 110 | esac 111 | -------------------------------------------------------------------------------- /ci/install_pycall.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.insert( 5 | 0, os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.pardir, "src") 6 | ) 7 | 8 | import julia # isort:skip 9 | 10 | julia.install(color=True) 11 | -------------------------------------------------------------------------------- /ci/test-upload/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py 3 | skipsdist = True 4 | 5 | [testenv] 6 | deps = 7 | shell-retry == 0.0.8 8 | 9 | # These are the packages listed in extras_require in setup.py. 10 | # Not using `julia[test]` to avoid installing the test 11 | # dependencies from `test.pypi.org`: 12 | numpy 13 | ipython 14 | pytest >= 4.4 15 | mock 16 | 17 | commands = 18 | shell-retry --backoff=2 --interval-max=20 --retry-count=30 --verbose -- \ 19 | pip install --index-url https://test.pypi.org/simple/ julia==0.6.2 20 | 21 | python -c "from julia import install; install()" 22 | python -m julia.runtests -- \ 23 | --log-file {envlogdir}/pytest.log \ 24 | {posargs} 25 | 26 | [pytest] 27 | log_file_level = DEBUG 28 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | if sys.version_info[0] < 3: 4 | collect_ignore_glob = [ 5 | "**/monkeypatch_completer.py", 6 | "**/monkeypatch_interactiveshell.py", 7 | ] 8 | # Theses files are ignored as import fails at collection phase. 9 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | API 3 | ===== 4 | 5 | Utility functions 6 | ================= 7 | 8 | .. autofunction:: julia.install 9 | 10 | 11 | Low-level API 12 | ============= 13 | 14 | .. autoclass:: julia.api.Julia 15 | :members: 16 | :special-members: __init__ 17 | 18 | .. autoclass:: julia.api.LibJulia 19 | :members: 20 | 21 | .. autoclass:: julia.api.JuliaInfo 22 | :members: 23 | 24 | .. autoclass:: julia.api.JuliaError 25 | :members: 26 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | # fmt: off 22 | 23 | project = 'PyJulia' 24 | copyright = '2019, The Julia and IPython development teams' 25 | author = 'The Julia and IPython development teams' 26 | 27 | # The short X.Y version 28 | version = '0.6.2' 29 | # The full version, including alpha/beta/rc tags 30 | release = '0.6.2' 31 | 32 | 33 | # -- General configuration --------------------------------------------------- 34 | 35 | # If your documentation needs a minimal Sphinx version, state it here. 36 | # 37 | # needs_sphinx = '1.0' 38 | 39 | # Add any Sphinx extension module names here, as strings. They can be 40 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 41 | # ones. 42 | extensions = [ 43 | 'sphinx.ext.napoleon', 44 | 'sphinx.ext.autodoc', 45 | 'sphinx.ext.intersphinx', 46 | ] 47 | 48 | # Add any paths that contain templates here, relative to this directory. 49 | templates_path = ['_templates'] 50 | 51 | # The suffix(es) of source filenames. 52 | # You can specify multiple suffix as a list of string: 53 | # 54 | source_suffix = { 55 | '.rst': 'restructuredtext', 56 | } 57 | 58 | # The master toctree document. 59 | master_doc = 'index' 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | # 64 | # This is also used if you do content translation via gettext catalogs. 65 | # Usually you set "language" from the command line for these cases. 66 | language = None 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | # This pattern also affects html_static_path and html_extra_path. 71 | exclude_patterns = [] 72 | 73 | # The reST default role (used for this markup: `text`) to use for all 74 | # documents. 75 | default_role = 'any' 76 | 77 | # The name of the Pygments (syntax highlighting) style to use. 78 | pygments_style = None 79 | 80 | 81 | # -- Options for HTML output ------------------------------------------------- 82 | 83 | # The theme to use for HTML and HTML Help pages. See the documentation for 84 | # a list of builtin themes. 85 | # 86 | html_theme = 'alabaster' 87 | 88 | # Theme options are theme-specific and customize the look and feel of a theme 89 | # further. For a list of options available for each theme, see the 90 | # documentation. 91 | # 92 | # https://alabaster.readthedocs.io/en/latest/customization.html 93 | html_theme_options = { 94 | 'github_banner': True, 95 | 'github_user': 'JuliaPy', 96 | 'github_repo': 'pyjulia', 97 | 'fixed_sidebar': True, 98 | } 99 | 100 | # Add any paths that contain custom static files (such as style sheets) here, 101 | # relative to this directory. They are copied after the builtin static files, 102 | # so a file named "default.css" will overwrite the builtin "default.css". 103 | html_static_path = ['_static'] 104 | 105 | # Custom sidebar templates, must be a dictionary that maps document names 106 | # to template names. 107 | # 108 | # The default sidebars (for documents that don't match any pattern) are 109 | # defined by theme itself. Builtin themes are using these templates by 110 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 111 | # 'searchbox.html']``. 112 | # 113 | # https://alabaster.readthedocs.io/en/latest/installation.html 114 | html_sidebars = { 115 | '**': [ 116 | 'about.html', 117 | 'navigation.html', 118 | 'relations.html', 119 | 'searchbox.html', 120 | 'donate.html', 121 | ] 122 | } 123 | 124 | 125 | # -- Options for HTMLHelp output --------------------------------------------- 126 | 127 | # Output file base name for HTML help builder. 128 | htmlhelp_basename = 'PyJuliadoc' 129 | 130 | 131 | # -- Options for LaTeX output ------------------------------------------------ 132 | 133 | latex_elements = { 134 | # The paper size ('letterpaper' or 'a4paper'). 135 | # 136 | # 'papersize': 'letterpaper', 137 | 138 | # The font size ('10pt', '11pt' or '12pt'). 139 | # 140 | # 'pointsize': '10pt', 141 | 142 | # Additional stuff for the LaTeX preamble. 143 | # 144 | # 'preamble': '', 145 | 146 | # Latex figure (float) alignment 147 | # 148 | # 'figure_align': 'htbp', 149 | } 150 | 151 | # Grouping the document tree into LaTeX files. List of tuples 152 | # (source start file, target name, title, 153 | # author, documentclass [howto, manual, or own class]). 154 | latex_documents = [ 155 | (master_doc, 'PyJulia.tex', 'PyJulia Documentation', 156 | 'The Julia and IPython development teams', 'manual'), 157 | ] 158 | 159 | 160 | # -- Options for manual page output ------------------------------------------ 161 | 162 | # One entry per manual page. List of tuples 163 | # (source start file, name, description, authors, manual section). 164 | man_pages = [ 165 | (master_doc, 'pyjulia', 'PyJulia Documentation', 166 | [author], 1) 167 | ] 168 | 169 | 170 | # -- Options for Texinfo output ---------------------------------------------- 171 | 172 | # Grouping the document tree into Texinfo files. List of tuples 173 | # (source start file, target name, title, author, 174 | # dir menu entry, description, category) 175 | texinfo_documents = [ 176 | (master_doc, 'PyJulia', 'PyJulia Documentation', 177 | author, 'PyJulia', 'One line description of project.', 178 | 'Miscellaneous'), 179 | ] 180 | 181 | 182 | # -- Options for Epub output ------------------------------------------------- 183 | 184 | # Bibliographic Dublin Core info. 185 | epub_title = project 186 | 187 | # The unique identifier of the text. This can be a ISBN number 188 | # or the project homepage. 189 | # 190 | # epub_identifier = '' 191 | 192 | # A unique identification for the text. 193 | # 194 | # epub_uid = '' 195 | 196 | # A list of files that should not be packed into the epub file. 197 | epub_exclude_files = ['search.html'] 198 | 199 | 200 | # -- Extension configuration ------------------------------------------------- 201 | 202 | # -- Options for intersphinx extension --------------------------------------- 203 | 204 | # Example configuration for intersphinx: refer to the Python standard library. 205 | intersphinx_mapping = {'https://docs.python.org/': None} 206 | -------------------------------------------------------------------------------- /docs/source/development.rst: -------------------------------------------------------------------------------- 1 | Development 2 | =========== 3 | 4 | Release 5 | ------- 6 | 7 | Step 1: Release 8 | ^^^^^^^^^^^^^^^ 9 | 10 | Bump the version number and push the change to ``release/main`` branch 11 | in https://github.com/JuliaPy/pyjulia. This triggers a CI that: 12 | 13 | 1. releases the package on ``test.pypi.org``, 14 | 2. installs the released package, 15 | 3. runs the test with the installed package and then 16 | 4. re-releases the package on ``pypi.org``. 17 | 18 | 19 | Step 2: Tag 20 | ^^^^^^^^^^^ 21 | 22 | Create a Git tag with the form ``vX.Y.Z``, merge ``release/main`` to 23 | ``master`` branch, and then push the tag and ``master`` branch. 24 | 25 | 26 | Special branches 27 | ---------------- 28 | 29 | ``release/main`` 30 | Push to this branch triggers the deploy to ``test.pypi.org``, test 31 | the uploaded package, and then re-upload it to ``pypi.org``. 32 | 33 | ``release/test`` 34 | Push to this branch triggers the deploy to ``test.pypi.org`` and 35 | test the uploaded package. 36 | -------------------------------------------------------------------------------- /docs/source/how_it_works.rst: -------------------------------------------------------------------------------- 1 | How it works 2 | ------------ 3 | 4 | PyJulia loads the ``libjulia`` library and executes the statements 5 | therein. To convert the variables, the ``PyCall`` package is used. 6 | Python references to Julia objects are reference counted by Python, and 7 | retained in the ``PyCall.pycall_gc`` mapping on the Julia side (the 8 | mapping is removed when reference count drops to zero, so that the Julia 9 | object may be freed). 10 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to PyJulia’s documentation! 2 | =================================== 3 | 4 | |github-action| 5 | 6 | Experimenting with developing a better interface to 7 | `Julia language `_ that works with 8 | `Python `_ 3 and Julia v1.0+. 9 | 10 | PyJulia is tested against Python versions 3.5+ 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | :caption: Contents: 15 | 16 | installation 17 | usage 18 | troubleshooting 19 | api 20 | sysimage 21 | pytest 22 | how_it_works 23 | limitations 24 | testing 25 | development 26 | 27 | Indices and tables 28 | ================== 29 | 30 | * :ref:`genindex` 31 | * :ref:`modindex` 32 | * :ref:`search` 33 | 34 | .. |github-action| 35 | image:: https://github.com/JuliaPy/pyjulia/workflows/Main%20workflow/badge.svg 36 | :target: https://github.com/JuliaPy/pyjulia/actions?query=workflow%3A%22Main+workflow%22 37 | :alt: Main workflow 38 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Installation 3 | ============== 4 | 5 | .. admonition:: tl;dr 6 | 7 | 1. :ref:`Install Julia `. 8 | 9 | 2. :ref:`Install PyJulia ` by 10 | 11 | .. code-block:: console 12 | 13 | $ python3 -m pip install --user julia 14 | 15 | Remove ``--user`` if you are using a virtual environment. 16 | 17 | 3. :ref:`Install Julia dependencies of PyJulia ` 18 | by 19 | 20 | .. code-block:: console 21 | 22 | $ python3 23 | >>> import julia 24 | >>> julia.install() 25 | 26 | See below for more detailed explanations. 27 | 28 | **Note:** If you are using Python installed with Ubuntu or ``conda``, 29 | PyJulia may not work with the default setting. For workarounds, see 30 | :doc:`Troubleshooting `. Same caution applies to any 31 | Debian-based and possibly other GNU/Linux distributions. 32 | 33 | 34 | .. _install-julia: 35 | 36 | Step 1: Install Julia 37 | ===================== 38 | 39 | Get the Julia installer from https://julialang.org/downloads/. See 40 | also the `Platform Specific Instructions 41 | `_. 42 | 43 | Your python installation must be able to call command line program 44 | ``julia``. If your installer does not add the Julia binary directory to 45 | your ``PATH``, you will have to add it. *An alias will not work.* 46 | 47 | Alternatively, you can pass the file path of the Julia executable to 48 | PyJulia functions. See `julia.install` and `Julia`. 49 | 50 | 51 | .. _install-pyjulia: 52 | 53 | Step 2: Install PyJulia 54 | ======================= 55 | 56 | **Note:** If you are not familiar with ``pip`` and have some troubles 57 | with the following installation steps, we recommend going through the 58 | `Tutorial in Python Packaging User Guide 59 | `_ or 60 | `pip's User Guide `_. 61 | 62 | To get released versions you can use: 63 | 64 | .. code-block:: console 65 | 66 | $ python3 -m pip install --user julia 67 | $ python2 -m pip install --user julia # If you need Python 2 68 | 69 | where ``--user`` should be omitted if you are using virtual environment 70 | (``virtualenv``, ``venv``, ``conda``, etc.). 71 | 72 | If you are interested in using the development version, you can install 73 | PyJulia directly from GitHub: 74 | 75 | .. code-block:: console 76 | 77 | $ python3 -m pip install --user 'https://github.com/JuliaPy/pyjulia/archive/master.zip#egg=julia' 78 | 79 | You may clone it directly to (say) your home directory. 80 | 81 | .. code-block:: console 82 | 83 | $ git clone https://github.com/JuliaPy/pyjulia 84 | 85 | then inside the ``pyjulia`` directory you need to run the python setup 86 | file 87 | 88 | .. code-block:: console 89 | 90 | $ cd pyjulia 91 | $ python3 -m pip install --user . 92 | $ python3 -m pip install --user -e . # If you want "development install" 93 | 94 | The ``-e`` flag makes a development install, meaning that any change to 95 | PyJulia source tree will take effect at next python interpreter restart 96 | without having to reissue an install command. 97 | 98 | See :doc:`Testing ` for how to run tests. 99 | 100 | 101 | .. _install-julia-packages: 102 | 103 | Step 3: Install Julia packages required by PyJulia 104 | ================================================== 105 | 106 | Launch a Python REPL and run the following code 107 | 108 | >>> import julia 109 | >>> julia.install() 110 | 111 | This installs Julia packages required by PyJulia. See also 112 | `julia.install`. 113 | 114 | Alternatively, you can use Julia's builtin package manager. 115 | 116 | .. code-block:: jlcon 117 | 118 | julia> using Pkg 119 | julia> Pkg.add("PyCall") 120 | 121 | Note that PyCall must be built with Python executable that is used to 122 | import PyJulia. See https://github.com/JuliaPy/PyCall.jl for more 123 | information about configuring PyCall. 124 | -------------------------------------------------------------------------------- /docs/source/limitations.rst: -------------------------------------------------------------------------------- 1 | Limitations 2 | ----------- 3 | 4 | Mismatch in valid set of identifiers 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | Not all valid Julia identifiers are valid Python identifiers. Unicode 8 | identifiers are invalid in Python 2.7 and so PyJulia cannot call or 9 | access Julia methods/variables with names that are not ASCII only. 10 | Although Python 3 allows Unicode identifiers, they are more aggressively 11 | normalized than Julia. For example, ``ϵ`` (GREEK LUNATE EPSILON SYMBOL) 12 | and ``ε`` (GREEK SMALL LETTER EPSILON) are identical in Python 3 but 13 | different in Julia. Additionally, it is a common idiom in Julia to 14 | append a ``!`` character to methods which mutate their arguments. These 15 | method names are invalid Python identifers. PyJulia renames these 16 | methods by subsituting ``!`` with ``_b``. For example, the Julia method 17 | ``sum!`` can be called in PyJulia using ``sum_b(...)``. 18 | 19 | Pre-compilation mechanism in Julia 1.0 20 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 21 | 22 | There was a major overhaul in the module loading system between Julia 23 | 0.6 and 1.0. As a result, the “hack” supporting the PyJulia to load 24 | PyCall stopped working. For the implementation detail of the hack, see: 25 | https://github.com/JuliaPy/pyjulia/tree/v0.3.0/src/julia/fake-julia 26 | 27 | For the update on this problem, see: 28 | https://github.com/JuliaLang/julia/issues/28518 29 | 30 | Ctrl-C does not work / terminates the whole Python process 31 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 32 | 33 | Currently, initializing PyJulia (e.g., by ``from julia import Main``) 34 | disables ``KeyboardInterrupt`` handling in the Python process. If you 35 | are using normal ``python`` interpreter, it means that canceling the 36 | input by Ctrl-C does not work and repeatedly providing Ctrl-C terminates 37 | the whole Python process with the error message 38 | ``WARNING: Force throwing a SIGINT``. Using IPython 7.0 or above is 39 | recommended to avoid such accidental shutdown. 40 | 41 | It also means that there is no safe way to cancel long-running 42 | computations or I/O at the moment. Sending SIGINT with Ctrl-C will 43 | terminate the whole Python process. 44 | 45 | For the update on this problem, see: 46 | https://github.com/JuliaPy/pyjulia/issues/211 47 | 48 | No threading support 49 | ~~~~~~~~~~~~~~~~~~~~ 50 | 51 | PyJulia cannot be used in different threads since libjulia is not 52 | thread safe. However, you can `use multiple threads within Julia 53 | `_. 54 | For example, start IPython by ``JULIA_NUM_THREADS=4 ipython`` and then 55 | run: 56 | 57 | .. code:: julia 58 | 59 | In [1]: %load_ext julia.magic 60 | Initializing Julia runtime. This may take some time... 61 | 62 | In [2]: %%julia 63 | ...: a = zeros(10) 64 | ...: Threads.@threads for i = 1:10 65 | ...: a[i] = Threads.threadid() 66 | ...: end 67 | ...: a 68 | Out[3]: array([1., 1., 1., 2., 2., 2., 3., 3., 4., 4.]) 69 | 70 | PyJulia does not release GIL 71 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 72 | 73 | PyJulia does not release the Global Interpreter Lock (GIL) while calling 74 | Julia functions since PyCall expects the GIL to be acquired always. It 75 | means that Python code and Julia code cannot run in parallel. 76 | -------------------------------------------------------------------------------- /docs/source/pytest.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | pytest plugin 3 | =============== 4 | 5 | .. program:: pytest 6 | 7 | PyJulia automatically installs a `pytest plugin 8 | `_. It takes care of 9 | tricky aspects of PyJulia initialization: 10 | 11 | * It loads ``libjulia`` as early as possible to avoid incompatibility 12 | of shared libraries such as ``libstdc++`` (assuming that the ones 13 | bundled with ``julia`` are newer than the ones otherwise loaded). 14 | 15 | * It provides a way to succinctly mark certain tests require Julia 16 | runtime (see `Fixture`_ and `Marker`_). 17 | 18 | * The tests requiring Julia can be skipped with :option:`--no-julia`. 19 | 20 | * It enables debug-level logging. This is highly recommended 21 | especially in CI setting as miss-configuration of PyJulia may result 22 | in segmentation fault in which Python cannot provide useful 23 | traceback. 24 | 25 | To activate PyJulia's pytest plugin [#]_ add ``-p julia.pytestplugin`` 26 | to the command line option. There are several ways to do this by 27 | default in your project. One option is to include this using 28 | ``addopts`` setup of ``pytest.ini`` or ``tox.ini`` file. See `How to 29 | change command line options defaults 30 | `_: 31 | 32 | .. code-block:: ini 33 | 34 | [pytest] 35 | addopts = 36 | -p julia.pytestplugin 37 | 38 | .. [#] This plugin is not activated by default (as in normal 39 | ``pytest-*`` plugin packages) to avoid accidentally breaking user's 40 | ``pytest`` setup when PyJulia is included as a non-test dependency. 41 | 42 | 43 | Options 44 | ======= 45 | 46 | Following options can be passed to :program:`pytest` 47 | 48 | .. option:: --no-julia 49 | 50 | Skip tests that require julia. 51 | 52 | .. option:: --julia 53 | 54 | Undo ``--no-julia``; i.e., run tests that require julia. 55 | 56 | .. option:: --julia-runtime 57 | 58 | Julia executable to be used. Defaults to environment variable 59 | `PYJULIA_TEST_RUNTIME`. 60 | 61 | .. option:: --julia- 62 | 63 | Some ```` that can be passed to ``julia`` executable 64 | (e.g., ``--compiled-modules=no``) can be passed to ``pytest`` 65 | plugin by ``--julia-`` (e.g., 66 | ``--julia-compiled-modules=no``). See ``pytest -p 67 | julia.pytestplugin --help`` for the actual list of options. 68 | 69 | 70 | Fixture 71 | ======= 72 | 73 | PyJulia's pytest plugin includes a `pytest fixture 74 | `_ ``julia`` which is 75 | set to an instance of :class:`.Julia` that is appropriately 76 | initialized. Example usage:: 77 | 78 | def test_eval(julia): 79 | assert julia.eval("1 + 1") == 2 80 | 81 | This fixture also "marks" that this test requires a Julia runtime. 82 | Thus, the tests using ``julia`` fixture are not run when 83 | :option:`--no-julia` is passed. 84 | 85 | 86 | Marker 87 | ====== 88 | 89 | PyJulia's pytest plugin also includes a `pytest marker 90 | `_ ``julia`` 91 | which can be used to mark that the test requires PyJulia setup. It is 92 | similar to ``julia`` fixture but it does not instantiate the actual 93 | :class:`.Julia` object. 94 | 95 | Example usage:: 96 | 97 | import pytest 98 | 99 | @pytest.mark.julia 100 | def test_import(): 101 | from julia import MyModule 102 | -------------------------------------------------------------------------------- /docs/source/sysimage.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Custom Julia system image 3 | =========================== 4 | 5 | .. versionadded:: 0.4 6 | 7 | If you use standard ``julia`` program, the basic functionalities and 8 | standard libraries of Julia are loaded from so called *system image* 9 | file which contains the machine code compiled from the Julia code. 10 | The Julia runtime can be configured to use a customized system image 11 | which may contain non-standard packages. This is a very effective way 12 | to reduce startup time of complex Julia packages such as PyCall. 13 | Furthermore, it can be used to workaround the problem in statically 14 | linked Python executable if you have the problem described in 15 | :ref:`statically-linked`. 16 | 17 | How to use a custom system image 18 | ================================ 19 | 20 | To compile a custom system image for PyJulia, run 21 | 22 | .. code-block:: console 23 | 24 | $ python3 -m julia.sysimage sys.so 25 | 26 | where ``sys.dll`` and ``sys.dylib`` may be used instead of ``sys.so`` 27 | in Windows and macOS, respectively. 28 | 29 | The command line interface `julia.sysimage` will: 30 | 31 | * Install packages required for compiling the system image in an 32 | isolated Julia environment. 33 | * Install PyCall to be compiled into the system image in an isolated 34 | Julia environment. 35 | * Create the system image at the given path (``./sys.so`` in the above 36 | example). 37 | 38 | To use this system image with PyJulia, you need to specify its path 39 | using ``sysimage`` keyword argument of the `Julia` constructor. For 40 | example, if you run `python3` REPL at the directory where you ran the 41 | above `julia.sysimage` command, you can do 42 | 43 | >>> from julia import Julia 44 | >>> jl = Julia(sysimage="sys.so") 45 | 46 | to initialize PyJulia. To check that this Julia runtime is using the 47 | correct system image, look at the output of ``Base.julia_cmd()`` 48 | 49 | >>> from julia import Base 50 | >>> Base.julia_cmd() 51 | 52 | 53 | 54 | Limitations 55 | =========== 56 | 57 | * ``PyCall`` and its dependencies cannot be updated after the system 58 | image is created. A new system image has to be created to update 59 | those packages. 60 | 61 | * The system image generated by `julia.sysimage` uses a different set 62 | of precompilation cache paths for each pair of ``julia-py`` 63 | executable and the system image file. Precompiled cache files 64 | generated by ``julia`` or a different ``julia-py`` executable cannot 65 | be reused by PyJulia when using the system image generated by 66 | `julia.sysimage`. 67 | 68 | * The absolute path of ``julia-py`` is embedded in the system image. 69 | This system image is not usable if ``julia-py`` is removed. 70 | 71 | 72 | Command line interfaces 73 | ======================= 74 | 75 | ``python3 -m julia.sysimage`` 76 | ----------------------------- 77 | 78 | .. automodule:: julia.sysimage 79 | :no-members: 80 | 81 | ``julia-py`` 82 | ------------ 83 | 84 | .. automodule:: julia.julia_py 85 | :no-members: 86 | -------------------------------------------------------------------------------- /docs/source/testing.rst: -------------------------------------------------------------------------------- 1 | Testing 2 | ------- 3 | 4 | PyJulia can be tested by simply running |tox|_. 5 | 6 | .. code-block:: console 7 | 8 | $ tox 9 | 10 | The full syntax for invoking |tox|_ is 11 | 12 | .. |tox| replace:: ``tox`` 13 | .. _tox: https://tox.readthedocs.io 14 | 15 | .. code-block:: console 16 | 17 | $ [PYJULIA_TEST_REBUILD=yes] \ 18 | [PYJULIA_TEST_RUNTIME=] \ 19 | tox [options] [-- pytest options] 20 | 21 | .. envvar:: PYJULIA_TEST_REBUILD 22 | 23 | *Be careful using this environment variable!* When it is set to 24 | ``yes``, your ``PyCall.jl`` installation will be rebuilt using the 25 | Python interpreter used for testing. The test suite tries to build 26 | back to the original configuration but the precompilation would be 27 | in the stale state after the test. Note also that it does not work 28 | if you unconditionally set ``PYTHON`` environment variable in your 29 | Julia startup file. 30 | 31 | .. envvar:: PYJULIA_TEST_RUNTIME 32 | 33 | ``julia`` executable to be used for testing. 34 | See also `pytest --julia-runtime`. 35 | 36 | ``[-- pytest options]`` 37 | Positional arguments after ``--`` are passed to |pytest|_. 38 | 39 | .. |pytest| replace:: ``pytest`` 40 | .. _pytest: https://pytest.org 41 | 42 | 43 | For example, 44 | 45 | .. code-block:: console 46 | 47 | $ PYJULIA_TEST_REBUILD=yes \ 48 | PYJULIA_TEST_RUNTIME=~/julia/julia \ 49 | tox -e py37 -- -s 50 | 51 | means to execute tests with 52 | 53 | * PyJulia in shared-cache mode 54 | * ``julia`` executable at ``~/julia/julia`` 55 | * Python 3.7 56 | * |pytest|_'s capturing mode turned off 57 | -------------------------------------------------------------------------------- /docs/source/troubleshooting.rst: -------------------------------------------------------------------------------- 1 | Troubleshooting 2 | --------------- 3 | 4 | .. _statically-linked: 5 | 6 | Your Python interpreter is statically linked to libpython 7 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 8 | 9 | If you use Python installed with Debian-based Linux distribution such as 10 | Ubuntu or install Python by ``conda``, you might have noticed that 11 | PyJulia cannot be initialized properly out-of-the-box. This is because 12 | those Python executables are statically linked to libpython. (See 13 | :doc:`limitations` for why that's a problem.) 14 | 15 | If you are unsure if your ``python`` has this problem, you can quickly 16 | check it by: 17 | 18 | .. code-block:: console 19 | 20 | $ ldd /usr/bin/python 21 | linux-vdso.so.1 (0x00007ffd73f7c000) 22 | libpthread.so.0 => /usr/lib/libpthread.so.0 (0x00007f10ef84e000) 23 | libc.so.6 => /usr/lib/libc.so.6 (0x00007f10ef68a000) 24 | libpython3.7m.so.1.0 => /usr/lib/libpython3.7m.so.1.0 (0x00007f10ef116000) 25 | /lib64/ld-linux-x86-64.so.2 => /usr/lib64/ld-linux-x86-64.so.2 (0x00007f10efaa4000) 26 | libdl.so.2 => /usr/lib/libdl.so.2 (0x00007f10ef111000) 27 | libutil.so.1 => /usr/lib/libutil.so.1 (0x00007f10ef10c000) 28 | libm.so.6 => /usr/lib/libm.so.6 (0x00007f10eef87000) 29 | 30 | in Linux where ``/usr/bin/python`` should be replaced with the path to 31 | your ``python`` command (use ``which python`` to find it out). In macOS, 32 | use ``otool -L`` instead of ``ldd``. If it does not print the path to 33 | libpython like ``/usr/lib/libpython3.7m.so.1.0`` in above example, you 34 | need to use one of the workaround below. 35 | 36 | Turn off compilation cache 37 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 38 | .. versionadded:: 0.3 39 | 40 | The easiest workaround is to pass ``compiled_modules=False`` to the 41 | ``Julia`` constructor. 42 | 43 | .. code-block:: pycon 44 | 45 | >>> from julia.api import Julia 46 | >>> jl = Julia(compiled_modules=False) 47 | 48 | This is equivalent to ``julia``\ ’s command line option 49 | ``--compiled-modules=no`` and disables the precompilation cache 50 | mechanism in Julia. Note that this option slows down loading and using 51 | Julia packages especially for complex and large ones. 52 | 53 | See also API documentation of `Julia`. 54 | 55 | Create a custom system image 56 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 57 | .. versionadded:: 0.4 58 | 59 | A very powerful way to avoid this the issue due to precompilation 60 | cache is to create a custom system image. This also has an additional 61 | benefit that initializing PyJulia becomes instant. See 62 | :doc:`sysimage` for how to create and use a custom system image. 63 | 64 | ``python-jl``: an easy workaround 65 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 66 | .. versionadded:: 0.2 67 | 68 | Another easy workaround is to use the ``python-jl`` command bundled in 69 | PyJulia. This can be used instead of normal ``python`` command for basic 70 | use-cases such as: 71 | 72 | .. code-block:: console 73 | 74 | $ python-jl your_script.py 75 | $ python-jl -c 'from julia.Base import banner; banner()' 76 | $ python-jl -m IPython 77 | 78 | See ``python-jl --help`` for more information. 79 | 80 | How ``python-jl`` works 81 | ''''''''''''''''''''''' 82 | 83 | Note that ``python-jl`` works by launching Python interpreter inside 84 | Julia. Importantly, it means that PyJulia has to be installed in the 85 | Python environment with which PyCall is configured. That is to say, 86 | following commands must work for ``python-jl`` to be usable: 87 | 88 | .. code-block:: jlcon 89 | 90 | julia> using PyCall 91 | 92 | julia> pyimport("julia") 93 | PyObject 94 | 95 | In fact, you can simply use PyJulia inside the Julia REPL, if you are 96 | comfortable with working in it: 97 | 98 | .. code-block:: jlcon 99 | 100 | julia> using PyCall 101 | 102 | julia> py""" 103 | from julia import Julia 104 | Julia(init_julia=False) 105 | # Then use your Python module: 106 | from your_module_using_pyjulia import function 107 | function() 108 | """ 109 | 110 | Ultimate fix: build your own Python 111 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 112 | 113 | Alternatively, you can use `pyenv `_ 114 | to build Python with ``--enable-shared`` option (see `their Wiki page 115 | `_). 116 | Of course, manually building from Python source distribution with the 117 | same configuration also works. 118 | 119 | .. code-block:: console 120 | 121 | $ PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.6.6 122 | Downloading Python-3.6.6.tar.xz... 123 | -> https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tar.xz 124 | Installing Python-3.6.6... 125 | Installed Python-3.6.6 to /home/USER/.pyenv/versions/3.6.6 126 | 127 | $ ldd ~/.pyenv/versions/3.6.6/bin/python3.6 | grep libpython 128 | libpython3.6m.so.1.0 => /home/USER/.pyenv/versions/3.6.6/lib/libpython3.6m.so.1.0 (0x00007fca44c8b000) 129 | 130 | For more discussion, see: https://github.com/JuliaPy/pyjulia/issues/185 131 | 132 | Segmentation fault in IPython 133 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 134 | 135 | You may experience segmentation fault when using PyJulia in old versions 136 | of IPython. You can avoid this issue by updating IPython to 7.0 or 137 | above. Alternatively, you can use IPython via Jupyter (e.g., 138 | ``jupyter console``) to workaround the problem. 139 | 140 | Error due to ``libstdc++`` version 141 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 142 | 143 | When you use PyJulia with another Python extension, you may see an error 144 | like :literal:`version `GLIBCXX_3.4.22' not found` (Linux) or 145 | ``The procedure entry point ... could not be located in the dynamic link library libstdc++6.dll`` 146 | (Windows). In this case, you might have observed that initializing 147 | PyJulia first fixes the problem. This is because Julia (or likely its 148 | dependencies like LLVM) requires a recent version of ``libstdc++``. 149 | 150 | Possible fixes: 151 | 152 | - Initialize PyJulia (e.g., by ``from julia import Main``) as early as 153 | possible. Note that just importing PyJulia (``import julia``) does 154 | not work. 155 | - Load ``libstdc++.so.6`` first by setting environment variable 156 | ``LD_PRELOAD`` (Linux) to 157 | ``/PATH/TO/JULIA/DIR/lib/julia/libstdc++.so.6`` where 158 | ``/PATH/TO/JULIA/DIR/lib`` is the directory which has 159 | ``libjulia.so``. macOS and Windows likely to have similar mechanisms 160 | (untested). 161 | - Similarly, set environment variable ``LD_LIBRARY_PATH`` (Linux) to 162 | ``/PATH/TO/JULIA/DIR/lib/julia`` directory. Using 163 | ``DYLD_LIBRARY_PATH`` on macOS and ``PATH`` on Windows may work 164 | (untested). 165 | 166 | See: https://github.com/JuliaPy/pyjulia/issues/180, 167 | https://github.com/JuliaPy/pyjulia/issues/223 168 | -------------------------------------------------------------------------------- /docs/source/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ----- 3 | 4 | PyJulia provides a high-level interface which assumes a “normal” setup 5 | (e.g., ``julia`` program is in your ``PATH``) and a low-level interface 6 | which can be used in a customized setup. 7 | 8 | High-level interface 9 | ~~~~~~~~~~~~~~~~~~~~ 10 | 11 | To call a Julia function in a Julia module, import the Julia module (say 12 | ``Base``) with: 13 | 14 | .. code-block:: pycon 15 | 16 | >>> from julia import Base 17 | 18 | and then call Julia functions in ``Base`` from python, e.g., 19 | 20 | .. code-block:: pycon 21 | 22 | >>> Base.sind(90) 23 | 24 | Other variants of Python import syntax also work: 25 | 26 | .. code-block:: pycon 27 | 28 | >>> import julia.Base 29 | >>> from julia.Base import Enums # import a submodule 30 | >>> from julia.Base import sin, sind # import functions from a module 31 | 32 | The global namespace of Julia’s interpreter can be accessed via a 33 | special module ``julia.Main``: 34 | 35 | .. code-block:: pycon 36 | 37 | >>> from julia import Main 38 | 39 | You can set names in this module to send Python values to Julia: 40 | 41 | .. code-block:: pycon 42 | 43 | >>> Main.xs = [1, 2, 3] 44 | 45 | which allows it to be accessed directly from Julia code, e.g., it can be 46 | evaluated at Julia side using Julia syntax: 47 | 48 | .. code-block:: pycon 49 | 50 | >>> Main.eval("sin.(xs)") 51 | 52 | Low-level interface 53 | ~~~~~~~~~~~~~~~~~~~ 54 | 55 | If you need a custom setup for PyJulia, it must be done *before* 56 | importing any Julia modules. For example, to use the Julia executable 57 | named ``custom_julia``, run: 58 | 59 | .. code-block:: pycon 60 | 61 | >>> from julia import Julia 62 | >>> jl = Julia(runtime="custom_julia") 63 | 64 | You can then use, e.g., 65 | 66 | .. code-block:: pycon 67 | 68 | >>> from julia import Base 69 | 70 | See also the API documentation for `Julia`. 71 | 72 | IPython magic 73 | ~~~~~~~~~~~~~ 74 | 75 | In IPython (and therefore in Jupyter), you can directly execute Julia 76 | code using ``%julia`` magic: 77 | 78 | .. code-block:: python 79 | 80 | In [1]: %load_ext julia.magic 81 | Initializing Julia runtime. This may take some time... 82 | 83 | In [2]: %julia [1 2; 3 4] .+ 1 84 | Out[2]: 85 | array([[2, 3], 86 | [4, 5]], dtype=int64) 87 | 88 | You can call Python code from inside of ``%julia`` blocks via ``$var`` 89 | for accessing single variables or ``py"..."`` for more complex 90 | expressions: 91 | 92 | .. code-block:: julia 93 | 94 | In [3]: arr = [1, 2, 3] 95 | 96 | In [4]: %julia $arr .+ 1 97 | Out[4]: 98 | array([2, 3, 4], dtype=int64) 99 | 100 | In [5]: %julia sum(py"[x**2 for x in arr]") 101 | Out[5]: 14 102 | 103 | Inside of strings and quote blocks, ``$var`` and ``py"..."`` don’t call 104 | Python and instead retain their usual Julia behavior. To call Python 105 | code in these cases, you can “escape” one extra time: 106 | 107 | .. code-block:: julia 108 | 109 | In [6]: foo = "Python" 110 | %julia foo = "Julia" 111 | %julia ("this is $foo", "this is $($foo)") 112 | Out[6]: ('this is Julia', 'this is Python') 113 | 114 | Expressions in macro arguments also always retain the Julia behavior: 115 | 116 | .. code-block:: julia 117 | 118 | In [7]: %julia @eval $foo 119 | Out[7]: 'Julia' 120 | 121 | Results are automatically converted between equivalent Python/Julia 122 | types (should they exist). You can turn this off by appending ``o`` to 123 | the Python string: 124 | 125 | .. code-block:: python 126 | 127 | In [8]: %julia typeof(py"1"), typeof(py"1"o) 128 | Out[8]: (, ) 129 | 130 | Code inside ``%julia`` blocks obeys the Python scope: 131 | 132 | .. code-block:: python 133 | 134 | In [9]: x = "global" 135 | ...: def f(): 136 | ...: x = "local" 137 | ...: ret = %julia py"x" 138 | ...: return ret 139 | ...: f() 140 | Out[9]: 'local' 141 | 142 | IPython configuration 143 | ^^^^^^^^^^^^^^^^^^^^^ 144 | 145 | PyJulia-IPython integration can be configured via IPython’s 146 | configuration system. For the non-default behaviors, add the following 147 | lines in, e.g., ``~/.ipython/profile_default/ipython_config.py`` (see 148 | `Introduction to IPython 149 | configuration `_). 150 | 151 | To disable code completion in ``%julia`` and ``%%julia`` magics, use 152 | 153 | .. code-block:: python 154 | 155 | c.JuliaMagics.completion = False # default: True 156 | 157 | To disable code highlighting in ``%%julia`` magic for terminal 158 | (non-Jupyter) IPython, use 159 | 160 | .. code-block:: python 161 | 162 | c.JuliaMagics.highlight = False # default: True 163 | 164 | To enable `Revise.jl `_ 165 | automatically, use 166 | 167 | .. code-block:: python 168 | 169 | c.JuliaMagics.revise = True # default: False 170 | 171 | Virtual environments 172 | ~~~~~~~~~~~~~~~~~~~~ 173 | 174 | PyJulia can be used in Python virtual environments created by 175 | ``virtualenv``, ``venv``, and any tools wrapping them such as 176 | ``pipenv``, provided that Python executable used in such environments 177 | are linked to identical libpython used by PyCall. If this is not the 178 | case, initializing PyJulia (e.g., ``import julia.Main``) prints an 179 | informative error message with detected paths to libpython. See `PyCall 180 | documentation `_ for how to 181 | configure Python executable. 182 | 183 | Note that Python environment created by ``conda`` is not supported. 184 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | [metadata] 4 | license_file = LICENSE.md 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from io import open # for Python 2 (identical to builtin in Python 3) 5 | 6 | from setuptools import find_packages, setup 7 | 8 | 9 | def pyload(name): 10 | ns = {} 11 | with open(name, encoding="utf-8") as f: 12 | exec(compile(f.read(), name, "exec"), ns) 13 | return ns 14 | 15 | 16 | # In case it's Python 2: 17 | try: 18 | execfile 19 | except NameError: 20 | pass 21 | else: 22 | 23 | def pyload(path): 24 | ns = {} 25 | execfile(path, ns) 26 | return ns 27 | 28 | 29 | repo_root = os.path.abspath(os.path.dirname(__file__)) 30 | 31 | with open(os.path.join(repo_root, "README.md"), encoding="utf-8") as f: 32 | long_description = f.read() 33 | # https://packaging.python.org/guides/making-a-pypi-friendly-readme/ 34 | 35 | ns = pyload(os.path.join(repo_root, "src", "julia", "release.py")) 36 | version = ns["__version__"] 37 | 38 | # fmt: off 39 | setup(name='julia', 40 | version=version, 41 | description="Julia/Python bridge with IPython support.", 42 | long_description=long_description, 43 | long_description_content_type="text/markdown", 44 | author='The Julia and IPython development teams.', 45 | author_email='julia@julialang.org', 46 | license='MIT', 47 | keywords='julia python', 48 | classifiers=[ 49 | # How mature is this project? Common values are 50 | # 3 - Alpha 51 | # 4 - Beta 52 | # 5 - Production/Stable 53 | 'Development Status :: 3 - Alpha', 54 | 55 | # Indicate who your project is intended for 56 | #'Intended Audience :: Developers', 57 | 58 | 'License :: OSI Approved :: MIT License', 59 | 60 | # Specify the Python versions you support here. In particular, ensure 61 | # that you indicate whether you support Python 2, Python 3 or both. 62 | 'Programming Language :: Python :: 3', 63 | 'Programming Language :: Python :: 3.4', 64 | 'Programming Language :: Python :: 3.5', 65 | 'Programming Language :: Python :: 3.6', 66 | 'Programming Language :: Python :: 3.7', 67 | 'Programming Language :: Python :: 3.8', 68 | 'Programming Language :: Python :: 3.9', 69 | 'Programming Language :: Python :: 3.10', 70 | 'Programming Language :: Python :: 3.11', 71 | 'Programming Language :: Python :: 3.12', 72 | ], 73 | url='http://julialang.org', 74 | project_urls={ 75 | "Source": "https://github.com/JuliaPy/pyjulia", 76 | "Tracker": "https://github.com/JuliaPy/pyjulia/issues", 77 | "Documentation": "https://pyjulia.readthedocs.io", 78 | }, 79 | packages=find_packages("src"), 80 | package_dir={"": "src"}, 81 | package_data={"julia": ["*.jl"]}, 82 | python_requires=">=3.4", 83 | extras_require={ 84 | # Update `ci/test-upload/tox.ini` when "test" is changed: 85 | "test": [ 86 | "numpy", 87 | "ipython", 88 | # pytest 4.4 for pytest.skip in doctest: 89 | # https://github.com/pytest-dev/pytest/pull/4927 90 | "pytest>=4.4", 91 | "mock", 92 | ], 93 | }, 94 | entry_points={ 95 | "console_scripts": [ 96 | "julia-py = julia.julia_py:main", 97 | "python-jl = julia.python_jl:main", 98 | ], 99 | }, 100 | # We bundle Julia scripts etc. inside `julia` directory. Thus, 101 | # this directory must exist in the file system (not in a zip 102 | # file): 103 | zip_safe=False, 104 | ) 105 | -------------------------------------------------------------------------------- /src/julia/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import JuliaError 2 | from .core import LegacyJulia as Julia 3 | from .ipy.revise import disable_revise, enable_revise 4 | from .release import __version__ 5 | from .tools import install 6 | -------------------------------------------------------------------------------- /src/julia/api.py: -------------------------------------------------------------------------------- 1 | from .core import Julia, JuliaError, JuliaInfo, LibJulia 2 | -------------------------------------------------------------------------------- /src/julia/compile.jl: -------------------------------------------------------------------------------- 1 | compiler_env, script, output, base_sysimage = ARGS 2 | 3 | if VERSION < v"0.7-" 4 | error("Unsupported Julia version $VERSION") 5 | end 6 | 7 | const Pkg = 8 | Base.require(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) 9 | 10 | Pkg.activate(compiler_env) 11 | @info "Loading PackageCompiler..." 12 | using PackageCompiler 13 | 14 | @info "Installing PyCall..." 15 | Pkg.activate(".") 16 | Pkg.add([ 17 | Pkg.PackageSpec( 18 | name = "PyCall", 19 | uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0", 20 | ), 21 | ]) 22 | 23 | if VERSION >= v"1.5-" 24 | mktempdir() do dir 25 | tmpimg = joinpath(dir, basename(output)) 26 | @info "Compiling a temporary system image without `PyCall`..." 27 | create_sysimage( 28 | Symbol[]; 29 | sysimage_path = tmpimg, 30 | project = ".", 31 | base_sysimage = isempty(base_sysimage) ? nothing : base_sysimage, 32 | ) 33 | @info "Compiling system image..." 34 | create_sysimage( 35 | [:PyCall]; 36 | sysimage_path = output, 37 | project = ".", 38 | precompile_execution_file = script, 39 | base_sysimage = tmpimg, 40 | ) 41 | end 42 | else 43 | @info "Compiling system image..." 44 | create_sysimage( 45 | [:PyCall], 46 | sysimage_path = output, 47 | project = ".", 48 | precompile_execution_file = script, 49 | base_sysimage = isempty(base_sysimage) ? nothing : base_sysimage, 50 | ) 51 | end 52 | 53 | @info "System image is created at $output" 54 | 55 | # Notes on two-stage system image monkey-patching for Julia 1.5 56 | # 57 | # Naive `@eval Base`-based monkey-patching stopped working as of Julia 1.5 58 | # presumably because @eval-ing to another module during precompilation now 59 | # throws an error: https://github.com/JuliaLang/julia/pull/35410 60 | # 61 | # We workaround this by creating the system image in two stages: 62 | # 63 | # 1. The first stage is monkey-patching as done before but without 64 | # PyCall. This way, we don't get the error because julia does not 65 | # try to precompile any packages. 66 | # 67 | # 2. The second stage is the inclusion of PyCall. At this point, we are 68 | # in a monkey-patched system image. So, it's possible to precompile 69 | # PyCall now. 70 | # 71 | # Note that even importing PackageCompiler.jl itself inside julia-py is 72 | # problematic because (normally) it has to be precompiled. We avoid this 73 | # problem by running PackageCompiler.jl under --compiled-modules=no. This is 74 | # safe to do thanks to PackageCompiler.do_ensurecompiled 75 | # (https://github.com/JuliaLang/PackageCompiler.jl/blob/3f0f4d882c560c4e4ccc6ab9a8b51ced380bb0d5/src/PackageCompiler.jl#L181-L188) 76 | # using a custom function PackageCompiler.get_julia_cmd 77 | # (https://github.com/JuliaLang/PackageCompiler.jl/blob/3f0f4d882c560c4e4ccc6ab9a8b51ced380bb0d5/src/PackageCompiler.jl#L113-L116) 78 | # (instead of Base.julia_cmd) and ignores --compiled-modules=no of the current 79 | # process. 80 | -------------------------------------------------------------------------------- /src/julia/find_libpython.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Locate libpython associated with this Python executable. 5 | """ 6 | 7 | # License 8 | # 9 | # Copyright 2018, Takafumi Arakaki 10 | # 11 | # Permission is hereby granted, free of charge, to any person obtaining 12 | # a copy of this software and associated documentation files (the 13 | # "Software"), to deal in the Software without restriction, including 14 | # without limitation the rights to use, copy, modify, merge, publish, 15 | # distribute, sublicense, and/or sell copies of the Software, and to 16 | # permit persons to whom the Software is furnished to do so, subject to 17 | # the following conditions: 18 | # 19 | # The above copyright notice and this permission notice shall be 20 | # included in all copies or substantial portions of the Software. 21 | # 22 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 23 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 24 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 25 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 26 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 27 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 28 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 29 | 30 | from __future__ import absolute_import, print_function 31 | 32 | import ctypes.util 33 | import functools 34 | import os 35 | import sys 36 | import sysconfig 37 | from logging import getLogger # see `julia.core.logger` 38 | 39 | logger = getLogger("find_libpython") 40 | 41 | is_windows = os.name == "nt" 42 | is_apple = sys.platform == "darwin" 43 | 44 | SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") 45 | if SHLIB_SUFFIX is None: 46 | if is_windows: 47 | SHLIB_SUFFIX = ".dll" 48 | else: 49 | SHLIB_SUFFIX = ".so" 50 | if is_apple: 51 | # sysconfig.get_config_var("SHLIB_SUFFIX") can be ".so" in macOS. 52 | # Let's not use the value from sysconfig. 53 | SHLIB_SUFFIX = ".dylib" 54 | 55 | 56 | def linked_libpython(): 57 | """ 58 | Find the linked libpython using dladdr (in *nix). 59 | 60 | Returns 61 | ------- 62 | path : str or None 63 | A path to linked libpython. Return `None` if statically linked. 64 | """ 65 | if is_windows: 66 | return _linked_libpython_windows() 67 | return _linked_libpython_unix() 68 | 69 | 70 | class Dl_info(ctypes.Structure): 71 | _fields_ = [ 72 | ("dli_fname", ctypes.c_char_p), 73 | ("dli_fbase", ctypes.c_void_p), 74 | ("dli_sname", ctypes.c_char_p), 75 | ("dli_saddr", ctypes.c_void_p), 76 | ] 77 | 78 | 79 | # fmt: off 80 | 81 | 82 | def _linked_libpython_unix(): 83 | libdl = ctypes.CDLL(ctypes.util.find_library("dl")) 84 | libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] 85 | libdl.dladdr.restype = ctypes.c_int 86 | 87 | dlinfo = Dl_info() 88 | retcode = libdl.dladdr( 89 | ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p), 90 | ctypes.pointer(dlinfo)) 91 | if retcode == 0: # means error 92 | return None 93 | path = os.path.realpath(dlinfo.dli_fname.decode()) 94 | if not os.path.exists(path): 95 | return None 96 | if path == os.path.realpath(sys.executable): 97 | return None 98 | return path 99 | 100 | 101 | def _linked_libpython_windows(): 102 | """ 103 | Based on: https://stackoverflow.com/a/16659821 104 | """ 105 | from ctypes.wintypes import HANDLE, LPWSTR, DWORD 106 | 107 | GetModuleFileName = ctypes.windll.kernel32.GetModuleFileNameW 108 | GetModuleFileName.argtypes = [HANDLE, LPWSTR, DWORD] 109 | GetModuleFileName.restype = DWORD 110 | 111 | MAX_PATH = 260 112 | try: 113 | buf = ctypes.create_unicode_buffer(MAX_PATH) 114 | GetModuleFileName(ctypes.pythonapi._handle, buf, MAX_PATH) 115 | return buf.value 116 | except (ValueError, OSError): 117 | return None 118 | 119 | 120 | 121 | def library_name(name, suffix=SHLIB_SUFFIX, is_windows=is_windows): 122 | """ 123 | Convert a file basename `name` to a library name (no "lib" and ".so" etc.) 124 | 125 | >>> library_name("libpython3.7m.so") # doctest: +SKIP 126 | 'python3.7m' 127 | >>> library_name("libpython3.7m.so", suffix=".so", is_windows=False) 128 | 'python3.7m' 129 | >>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False) 130 | 'python3.7m' 131 | >>> library_name("python37.dll", suffix=".dll", is_windows=True) 132 | 'python37' 133 | """ 134 | if not is_windows and name.startswith("lib"): 135 | name = name[len("lib"):] 136 | if suffix and name.endswith(suffix): 137 | name = name[:-len(suffix)] 138 | return name 139 | 140 | 141 | def append_truthy(list, item): 142 | if item: 143 | list.append(item) 144 | 145 | 146 | def uniquifying(items): 147 | """ 148 | Yield items while excluding the duplicates and preserving the order. 149 | 150 | >>> list(uniquifying([1, 2, 1, 2, 3])) 151 | [1, 2, 3] 152 | """ 153 | seen = set() 154 | for x in items: 155 | if x not in seen: 156 | yield x 157 | seen.add(x) 158 | 159 | 160 | def uniquified(func): 161 | """ Wrap iterator returned from `func` by `uniquifying`. """ 162 | @functools.wraps(func) 163 | def wrapper(*args, **kwds): 164 | return uniquifying(func(*args, **kwds)) 165 | return wrapper 166 | 167 | 168 | @uniquified 169 | def candidate_names(suffix=SHLIB_SUFFIX): 170 | """ 171 | Iterate over candidate file names of libpython. 172 | 173 | Yields 174 | ------ 175 | name : str 176 | Candidate name libpython. 177 | """ 178 | LDLIBRARY = sysconfig.get_config_var("LDLIBRARY") 179 | if LDLIBRARY: 180 | yield LDLIBRARY 181 | 182 | LIBRARY = sysconfig.get_config_var("LIBRARY") 183 | if LIBRARY: 184 | yield os.path.splitext(LIBRARY)[0] + suffix 185 | 186 | dlprefix = "" if is_windows else "lib" 187 | sysdata = dict( 188 | v=sys.version_info, 189 | # VERSION is X.Y in Linux/macOS and XY in Windows: 190 | VERSION=(sysconfig.get_config_var("VERSION") or 191 | "{v.major}.{v.minor}".format(v=sys.version_info)), 192 | ABIFLAGS=(sysconfig.get_config_var("ABIFLAGS") or 193 | sysconfig.get_config_var("abiflags") or ""), 194 | ) 195 | 196 | for stem in [ 197 | "python{VERSION}{ABIFLAGS}".format(**sysdata), 198 | "python{VERSION}".format(**sysdata), 199 | "python{v.major}".format(**sysdata), 200 | "python", 201 | ]: 202 | yield dlprefix + stem + suffix 203 | 204 | 205 | 206 | @uniquified 207 | def candidate_paths(suffix=SHLIB_SUFFIX): 208 | """ 209 | Iterate over candidate paths of libpython. 210 | 211 | Yields 212 | ------ 213 | path : str or None 214 | Candidate path to libpython. The path may not be a fullpath 215 | and may not exist. 216 | """ 217 | 218 | yield linked_libpython() 219 | 220 | # List candidates for directories in which libpython may exist 221 | lib_dirs = [] 222 | append_truthy(lib_dirs, sysconfig.get_config_var('LIBPL')) 223 | append_truthy(lib_dirs, sysconfig.get_config_var('srcdir')) 224 | append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR")) 225 | 226 | # LIBPL seems to be the right config_var to use. It is the one 227 | # used in python-config when shared library is not enabled: 228 | # https://github.com/python/cpython/blob/v3.7.0/Misc/python-config.in#L55-L57 229 | # 230 | # But we try other places just in case. 231 | 232 | if is_windows: 233 | lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) 234 | else: 235 | lib_dirs.append(os.path.join( 236 | os.path.dirname(os.path.dirname(sys.executable)), 237 | "lib")) 238 | 239 | # For macOS: 240 | append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX")) 241 | 242 | lib_dirs.append(sys.exec_prefix) 243 | lib_dirs.append(os.path.join(sys.exec_prefix, "lib")) 244 | 245 | lib_basenames = list(candidate_names(suffix=suffix)) 246 | 247 | for directory in lib_dirs: 248 | for basename in lib_basenames: 249 | yield os.path.join(directory, basename) 250 | 251 | # In macOS and Windows, ctypes.util.find_library returns a full path: 252 | for basename in lib_basenames: 253 | yield ctypes.util.find_library(library_name(basename)) 254 | 255 | # Possibly useful links: 256 | # * https://packages.ubuntu.com/bionic/amd64/libpython3.6/filelist 257 | # * https://github.com/Valloric/ycmd/issues/518 258 | # * https://github.com/Valloric/ycmd/pull/519 259 | 260 | 261 | def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): 262 | """ 263 | Normalize shared library `path` to a real path. 264 | 265 | If `path` is not a full path, `None` is returned. If `path` does 266 | not exists, append `SHLIB_SUFFIX` and check if it exists. 267 | Finally, the path is canonicalized by following the symlinks. 268 | 269 | Parameters 270 | ---------- 271 | path : str or None 272 | A candidate path to a shared library. 273 | 274 | Returns 275 | ------- 276 | path : str or None 277 | Normalized existing path or `None`. 278 | """ 279 | if not path: 280 | return None 281 | if not os.path.isabs(path): 282 | return None 283 | if os.path.exists(path): 284 | return os.path.realpath(path) 285 | if os.path.exists(path + suffix): 286 | return os.path.realpath(path + suffix) 287 | if is_apple: 288 | return normalize_path(_remove_suffix_apple(path), 289 | suffix=".so", is_apple=False) 290 | return None 291 | 292 | 293 | def _remove_suffix_apple(path): 294 | """ 295 | Strip off .so or .dylib. 296 | 297 | >>> _remove_suffix_apple("libpython.so") 298 | 'libpython' 299 | >>> _remove_suffix_apple("libpython.dylib") 300 | 'libpython' 301 | >>> _remove_suffix_apple("libpython3.7") 302 | 'libpython3.7' 303 | """ 304 | if path.endswith(".dylib"): 305 | return path[:-len(".dylib")] 306 | if path.endswith(".so"): 307 | return path[:-len(".so")] 308 | return path 309 | 310 | 311 | @uniquified 312 | def finding_libpython(): 313 | """ 314 | Iterate over existing libpython paths. 315 | 316 | The first item is likely to be the best one. 317 | 318 | Yields 319 | ------ 320 | path : str 321 | Existing path to a libpython. 322 | """ 323 | logger.debug("is_windows = %s", is_windows) 324 | logger.debug("is_apple = %s", is_apple) 325 | for path in candidate_paths(): 326 | logger.debug("Candidate: %s", path) 327 | normalized = normalize_path(path) 328 | if normalized: 329 | logger.debug("Found: %s", normalized) 330 | yield normalized 331 | else: 332 | logger.debug("Not found.") 333 | 334 | 335 | def find_libpython(): 336 | """ 337 | Return a path (`str`) to libpython or `None` if not found. 338 | 339 | Parameters 340 | ---------- 341 | path : str or None 342 | Existing path to the (supposedly) correct libpython. 343 | """ 344 | for path in finding_libpython(): 345 | return os.path.realpath(path) 346 | 347 | 348 | def print_all(items): 349 | for x in items: 350 | print(x) 351 | 352 | 353 | def cli_find_libpython(cli_op, verbose): 354 | import logging 355 | # Importing `logging` module here so that using `logging.debug` 356 | # instead of `logger.debug` outside of this function becomes an 357 | # error. 358 | 359 | if verbose: 360 | logging.basicConfig( 361 | format="%(levelname)s %(message)s", 362 | level=logging.DEBUG) 363 | 364 | if cli_op == "list-all": 365 | print_all(finding_libpython()) 366 | elif cli_op == "candidate-names": 367 | print_all(candidate_names()) 368 | elif cli_op == "candidate-paths": 369 | print_all(p for p in candidate_paths() if p and os.path.isabs(p)) 370 | else: 371 | path = find_libpython() 372 | if path is None: 373 | return 1 374 | print(path, end="") 375 | 376 | 377 | def main(args=None): 378 | import argparse 379 | parser = argparse.ArgumentParser( 380 | description=__doc__) 381 | parser.add_argument( 382 | "--verbose", "-v", action="store_true", 383 | help="Print debugging information.") 384 | 385 | group = parser.add_mutually_exclusive_group() 386 | group.add_argument( 387 | "--list-all", 388 | action="store_const", dest="cli_op", const="list-all", 389 | help="Print list of all paths found.") 390 | group.add_argument( 391 | "--candidate-names", 392 | action="store_const", dest="cli_op", const="candidate-names", 393 | help="Print list of candidate names of libpython.") 394 | group.add_argument( 395 | "--candidate-paths", 396 | action="store_const", dest="cli_op", const="candidate-paths", 397 | help="Print list of candidate paths of libpython.") 398 | 399 | ns = parser.parse_args(args) 400 | parser.exit(cli_find_libpython(**vars(ns))) 401 | 402 | 403 | if __name__ == "__main__": 404 | main() 405 | -------------------------------------------------------------------------------- /src/julia/install-packagecompiler.jl: -------------------------------------------------------------------------------- 1 | compiler_env, = ARGS 2 | 3 | if VERSION < v"0.7-" 4 | error("Unsupported Julia version $VERSION") 5 | end 6 | 7 | const Pkg = 8 | Base.require(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) 9 | 10 | function cat_build_log(pkg) 11 | modpath = Base.locate_package(pkg) 12 | if modpath !== nothing 13 | logfile = joinpath(dirname(modpath), "..", "deps", "build.log") 14 | if isfile(logfile) 15 | print(stderr, read(logfile, String)) 16 | return 17 | end 18 | end 19 | @error "build.log for $pkg not found" 20 | end 21 | 22 | Pkg.activate(compiler_env) 23 | @info "Installing PackageCompiler..." 24 | 25 | Pkg.add([ 26 | Pkg.PackageSpec( 27 | name = "PackageCompiler", 28 | uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d", 29 | version = "1", 30 | ) 31 | ]) 32 | cat_build_log(Base.PkgId( 33 | Base.UUID("9b87118b-4619-50d2-8e1e-99f35a4d4d9d"), 34 | "PackageCompiler")) 35 | 36 | @info "Loading PackageCompiler..." 37 | using PackageCompiler 38 | 39 | @info "PackageCompiler is successfully installed at $compiler_env" 40 | -------------------------------------------------------------------------------- /src/julia/install.jl: -------------------------------------------------------------------------------- 1 | OP, python, libpython = ARGS 2 | 3 | # Special exit codes for this script. 4 | # (see: https://www.tldp.org/LDP/abs/html/exitcodes.html) 5 | code_no_precompile_needed = 113 6 | 7 | 8 | if VERSION < v"0.7.0" 9 | error("Unsupported Julia version $VERSION") 10 | end 11 | 12 | const Pkg = 13 | Base.require(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) 14 | const InteractiveUtils = Base.require(Base.PkgId( 15 | Base.UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240"), 16 | "InteractiveUtils", 17 | )) 18 | 19 | @info "Julia version info" 20 | InteractiveUtils.versioninfo(verbose=true) 21 | 22 | @info "Julia executable: $(Base.julia_cmd().exec[1])" 23 | 24 | pkgid = Base.PkgId(Base.UUID(0x438e738f_606a_5dbb_bf0a_cddfbfd45ab0), "PyCall") 25 | pycall_is_installed = Base.locate_package(pkgid) !== nothing 26 | 27 | @info "Trying to import PyCall..." 28 | 29 | module DummyPyCall 30 | python = nothing 31 | libpython = nothing 32 | end 33 | 34 | try 35 | # `import PyCall` cannot be caught? 36 | global PyCall = Base.require(pkgid) 37 | catch err 38 | @error "`import PyCall` failed" exception=(err, catch_backtrace()) 39 | global PyCall = DummyPyCall 40 | end 41 | 42 | 43 | ENV["PYTHON"] = python 44 | 45 | # TODO: warn if some relevant JULIA_* environment variables are set 46 | # TODO: use PackageSpec to specify PyCall's UUID 47 | 48 | function build_pycall() 49 | modpath = Base.locate_package(pkgid) 50 | pkgdir = joinpath(dirname(modpath), "..") 51 | 52 | if VERSION >= v"1.1.0-rc1" 53 | @info """Run `Pkg.build("PyCall"; verbose=true)`""" 54 | Pkg.build("PyCall"; verbose=true) 55 | else 56 | @info """Run `Pkg.build("PyCall")`""" 57 | Pkg.build("PyCall") 58 | logfile = joinpath(pkgdir, "deps", "build.log") 59 | if isfile(logfile) 60 | @info "Build log in $logfile" 61 | print(stderr, read(logfile, String)) 62 | end 63 | end 64 | depsfile = joinpath(pkgdir, "deps", "deps.jl") 65 | if isfile(depsfile) 66 | @info "`$depsfile`" 67 | print(stderr, read(depsfile, String)) 68 | else 69 | @error "Missing `deps.jl` file at: `$depsfile`" 70 | end 71 | end 72 | 73 | if OP == "build" 74 | build_pycall() 75 | elseif PyCall.python == python || PyCall.libpython == libpython 76 | @info """ 77 | PyCall is already installed and compatible with Python executable. 78 | 79 | PyCall: 80 | python: $(PyCall.python) 81 | libpython: $(PyCall.libpython) 82 | Python: 83 | python: $python 84 | libpython: $libpython 85 | """ 86 | exit(code_no_precompile_needed) 87 | else 88 | if PyCall.python !== nothing 89 | if isempty(libpython) 90 | @warn """ 91 | PyCall is already installed. However, you may have trouble using 92 | this Python executable because it is statically linked to libpython. 93 | 94 | For more information, see: 95 | https://pyjulia.readthedocs.io/en/latest/troubleshooting.html 96 | 97 | Python executable: 98 | $python 99 | Julia executable: 100 | $(Base.julia_cmd().exec[1]) 101 | """ 102 | exit(code_no_precompile_needed) 103 | else 104 | @info """ 105 | PyCall is already installed but not compatible with this Python 106 | executable. Re-building PyCall... 107 | """ 108 | build_pycall() 109 | end 110 | elseif pycall_is_installed 111 | @info """ 112 | PyCall is already installed but importing it failed. 113 | Re-building PyCall may fix the issue... 114 | """ 115 | build_pycall() 116 | else 117 | @info "Installing PyCall..." 118 | Pkg.add("PyCall") 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /src/julia/ipy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JuliaPy/pyjulia/f30de4e235ce1705b2473f12b9b3b307a8a78ab8/src/julia/ipy/__init__.py -------------------------------------------------------------------------------- /src/julia/ipy/monkeypatch_completer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Monkey-patch `IPCompleter` to make code completion work in ``%%julia``. 3 | 4 | This is done by monkey-patching because it looks like there is no 5 | immediate plan for an API to do this: 6 | https://github.com/ipython/ipython/pull/10722 7 | """ 8 | 9 | from __future__ import absolute_import, print_function 10 | 11 | import re 12 | 13 | from IPython.core.completer import Completion, IPCompleter 14 | 15 | 16 | class JuliaCompleter(object): 17 | def __init__(self, julia=None): 18 | from julia import Julia 19 | 20 | self.julia = Julia() if julia is None else julia 21 | self.magic_re = re.compile(r".*(\s|^)%%?julia\s*") 22 | # With this regexp, "=%julia Cha" won't work. But maybe 23 | # it's better to be conservative here. 24 | 25 | @property 26 | def jlcomplete(self): 27 | from julia.Main._PyJuliaHelper import completions 28 | 29 | return completions 30 | 31 | def julia_completions(self, full_text, offset): 32 | self.last_text = full_text 33 | match = self.magic_re.match(full_text) 34 | if not match: 35 | return [] 36 | prefix_len = match.end() 37 | jl_pos = offset - prefix_len 38 | jl_code = full_text[prefix_len:] 39 | texts, (jl_start, jl_end), should_complete = self.jlcomplete(jl_code, jl_pos) 40 | start = jl_start - 1 + prefix_len 41 | end = jl_end + prefix_len 42 | completions = [Completion(start, end, txt) for txt in texts] 43 | self.last_completions = completions 44 | # if not should_complete: 45 | # return [] 46 | return completions 47 | 48 | 49 | class IPCompleterPatcher(object): 50 | def __init__(self): 51 | from julia.Base import VERSION 52 | 53 | if (VERSION.major, VERSION.minor) < (0, 7): 54 | return 55 | 56 | self.patch_ipcompleter(IPCompleter, JuliaCompleter()) 57 | 58 | def patch_ipcompleter(self, IPCompleter, jlcompleter): 59 | orig__completions = IPCompleter._completions 60 | 61 | def _completions(self, full_text, offset, **kwargs): 62 | completions = jlcompleter.julia_completions(full_text, offset) 63 | if completions: 64 | return completions 65 | else: 66 | return orig__completions(self, full_text, offset, **kwargs) 67 | 68 | IPCompleter._completions = _completions 69 | 70 | self.orig__completions = orig__completions 71 | self.patched__completions = _completions 72 | self.IPCompleter = IPCompleter 73 | 74 | 75 | # Make it work with reload: 76 | try: 77 | PATCHER 78 | except NameError: 79 | PATCHER = None 80 | 81 | 82 | def patch_ipcompleter(): 83 | global PATCHER 84 | if PATCHER is not None: 85 | return 86 | PATCHER = IPCompleterPatcher() 87 | 88 | 89 | # TODO: write `unpatch_ipcompleter` 90 | -------------------------------------------------------------------------------- /src/julia/ipy/monkeypatch_interactiveshell.py: -------------------------------------------------------------------------------- 1 | """ 2 | Monkey-patch `TerminalInteractiveShell` to highlight code in ``%%julia``. 3 | """ 4 | 5 | from __future__ import absolute_import, print_function 6 | 7 | from IPython.terminal.interactiveshell import TerminalInteractiveShell 8 | from prompt_toolkit.lexers import PygmentsLexer 9 | from pygments.lexers import JuliaLexer 10 | 11 | 12 | class TerminalInteractiveShellPatcher(object): 13 | def __init__(self): 14 | self.patch_extra_prompt_options(TerminalInteractiveShell) 15 | 16 | def patch_extra_prompt_options(self, TerminalInteractiveShell): 17 | orig__extra_prompt_options = TerminalInteractiveShell._extra_prompt_options 18 | self.orig__extra_prompt_options = orig__extra_prompt_options 19 | 20 | def _extra_prompt_options(self): 21 | options = orig__extra_prompt_options(self) 22 | options["lexer"].magic_lexers["julia"] = PygmentsLexer(JuliaLexer) 23 | return options 24 | 25 | TerminalInteractiveShell._extra_prompt_options = _extra_prompt_options 26 | 27 | 28 | # Make it work with reload: 29 | try: 30 | PATCHER 31 | except NameError: 32 | PATCHER = None 33 | 34 | 35 | def patch_interactiveshell(ip): 36 | global PATCHER 37 | if PATCHER is not None: 38 | return 39 | if isinstance(ip, TerminalInteractiveShell): 40 | PATCHER = TerminalInteractiveShellPatcher() 41 | 42 | 43 | # TODO: write `unpatch_interactiveshell` 44 | -------------------------------------------------------------------------------- /src/julia/ipy/revise.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import warnings 4 | 5 | revise_errors_limit = 1 6 | 7 | 8 | def enable_revise(): 9 | """ 10 | (Re-)enable Revise.jl integration. 11 | 12 | IPython magic must be loaded with ``JuliaMagics.revise = True`` option. 13 | """ 14 | global revise_errors 15 | revise_errors = 0 16 | 17 | 18 | def disable_revise(): 19 | """ 20 | Disable Revise.jl integration. 21 | """ 22 | global revise_errors 23 | revise_errors = revise_errors_limit 24 | 25 | 26 | def make_revise_wrapper(revise): 27 | def revise_wrapper(): 28 | global revise_errors 29 | 30 | if revise_errors >= revise_errors_limit: 31 | return 32 | 33 | try: 34 | revise() 35 | except Exception as err: 36 | warnings.warn(str(err)) 37 | revise_errors += 1 38 | if revise_errors >= revise_errors_limit: 39 | warnings.warn( 40 | "Turning off Revise.jl." 41 | " Run `julia.enable_revise()` to re-enable it." 42 | ) 43 | else: 44 | revise_errors = 0 45 | 46 | return revise_wrapper 47 | 48 | 49 | def register_revise_hook(ip): 50 | global revise_errors 51 | 52 | try: 53 | from julia.Revise import revise 54 | except ImportError: 55 | warnings.warn( 56 | "Failed to import Revise.jl." 57 | ' Install it with `using Pkg; Pkg.add("Revise")` in Julia REPL.' 58 | ) 59 | return 60 | 61 | revise_errors = 0 62 | ip.events.register("pre_execute", make_revise_wrapper(revise)) 63 | -------------------------------------------------------------------------------- /src/julia/julia_py.py: -------------------------------------------------------------------------------- 1 | """ 2 | Launch Julia through PyJulia. 3 | 4 | Currently, `julia-py` is primary used internally for supporting 5 | `julia.sysimage` command line interface. Using `julia-py` like normal 6 | Julia program requires `--sysimage` to be set to the system image 7 | created by `julia.sysimage`. 8 | 9 | Example:: 10 | 11 | $ python3 -m julia.sysimage sys.so 12 | $ julia-py --sysimage sys.so 13 | """ 14 | 15 | from __future__ import absolute_import, print_function 16 | 17 | import argparse 18 | import os 19 | import sys 20 | from logging import getLogger # see `.core.logger` 21 | 22 | from .api import JuliaInfo, LibJulia 23 | from .core import enable_debug, which 24 | from .tools import julia_py_executable 25 | 26 | logger = getLogger("julia") 27 | 28 | 29 | def julia_py(julia, pyjulia_debug, jl_args): 30 | if pyjulia_debug: 31 | enable_debug() 32 | 33 | julia = which(julia) 34 | os.environ["_PYJULIA_JULIA"] = julia 35 | os.environ["_PYJULIA_JULIA_PY"] = julia_py_executable() 36 | os.environ["_PYJULIA_PATCH_JL"] = patch_jl_path = os.path.join( 37 | os.path.dirname(os.path.realpath(__file__)), "patch.jl" 38 | ) 39 | 40 | juliainfo = JuliaInfo.load(julia=julia) 41 | api = LibJulia.from_juliainfo(juliainfo) 42 | api.init_julia(jl_args) 43 | code = 1 44 | if juliainfo.version_info >= (1, 5, 0): 45 | logger.debug("Skipping `__init__()` hacks in `julia` %s", juliainfo.version_raw) 46 | else: 47 | logger.debug("Calling `Base.PCRE.__init__()`") 48 | if not api.jl_eval_string(b"Base.PCRE.__init__()"): 49 | print( 50 | "julia-py: Error while calling `Base.PCRE.__init__()`", file=sys.stderr 51 | ) 52 | sys.exit(code) 53 | logger.debug("Calling `Random.__init__()`") 54 | if not api.jl_eval_string( 55 | b""" 56 | Base.require( 57 | Base.PkgId( 58 | Base.UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c"), 59 | "Random", 60 | ), 61 | ).__init__() 62 | """ 63 | ): 64 | print("julia-py: Error while calling `Random.__init__()`", file=sys.stderr) 65 | sys.exit(code) 66 | logger.debug("Loading %s", patch_jl_path) 67 | if not api.jl_eval_string(b"""Base.include(Main, ENV["_PYJULIA_PATCH_JL"])"""): 68 | print("julia-py: Error in", patch_jl_path, file=sys.stderr) 69 | sys.exit(code) 70 | logger.debug("Calling `Base._start()`") 71 | if api.jl_eval_string(b"Base.invokelatest(Base._start)"): 72 | code = 0 73 | logger.debug("Calling `jl_atexit_hook(%s)`", code) 74 | api.jl_atexit_hook(code) 75 | logger.debug("Exiting with code %s", code) 76 | sys.exit(code) 77 | 78 | 79 | class CustomFormatter( 80 | argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter 81 | ): 82 | pass 83 | 84 | 85 | def is_pyjulia_in_julia_debug(julia_debug=os.environ.get("JULIA_DEBUG", "")): 86 | """ 87 | Parse `JULIA_DEBUG` and return the debug flag. 88 | 89 | >>> is_pyjulia_in_julia_debug("") 90 | False 91 | >>> is_pyjulia_in_julia_debug("Main,loading") 92 | False 93 | >>> is_pyjulia_in_julia_debug("pyjulia") 94 | True 95 | >>> is_pyjulia_in_julia_debug("all") 96 | True 97 | >>> is_pyjulia_in_julia_debug("all,!pyjulia") 98 | False 99 | 100 | Ref: 101 | https://github.com/JuliaLang/julia/pull/32432 102 | """ 103 | syms = list(filter(None, map(str.strip, julia_debug.split(",")))) 104 | if "pyjulia" in syms: 105 | return True 106 | elif "all" in syms and "!pyjulia" not in syms: 107 | return True 108 | return False 109 | 110 | 111 | def parse_args(args, **kwargs): 112 | options = dict( 113 | prog="julia-py", 114 | usage="%(prog)s [--julia JULIA] [--pyjulia-debug] [...]", 115 | formatter_class=CustomFormatter, 116 | description=__doc__, 117 | ) 118 | options.update(kwargs) 119 | parser = argparse.ArgumentParser(**options) 120 | parser.add_argument( 121 | "--julia", 122 | default=os.environ.get("_PYJULIA_JULIA", "julia"), 123 | help=""" 124 | Julia `executable` used by PyJulia. 125 | """, 126 | ) 127 | parser.add_argument( 128 | "--pyjulia-debug", 129 | action="store_true", 130 | default=is_pyjulia_in_julia_debug(), 131 | help=""" 132 | Print PyJulia's debugging messages to standard error. It is 133 | automatically set if `pyjulia` is in the environment variable 134 | `JULIA_DEBUG` (e.g., `JULIA_DEBUG=pyjulia,Main`). 135 | """, 136 | ) 137 | parser.add_argument( 138 | "--no-pyjulia-debug", 139 | dest="pyjulia_debug", 140 | action="store_false", 141 | help=""" 142 | Toggle off `--pyjulia_debug`. 143 | """, 144 | ) 145 | ns, jl_args = parser.parse_known_args(args) 146 | ns.jl_args = jl_args 147 | return ns 148 | 149 | 150 | def main(args=None, **kwargs): 151 | julia_py(**vars(parse_args(args, **kwargs))) 152 | 153 | 154 | if __name__ == "__main__": 155 | main() 156 | -------------------------------------------------------------------------------- /src/julia/juliainfo.jl: -------------------------------------------------------------------------------- 1 | println(VERSION) 2 | println(VERSION.major) 3 | println(VERSION.minor) 4 | println(VERSION.patch) 5 | 6 | VERSION < v"0.7.0" && exit() 7 | 8 | const Libdl = 9 | Base.require(Base.PkgId(Base.UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb"), "Libdl")) 10 | const Pkg = 11 | Base.require(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) 12 | 13 | println(Base.Sys.BINDIR) 14 | println(Libdl.dlpath(string("lib", splitext(Base.julia_exename())[1]))) 15 | println(unsafe_string(Base.JLOptions().image_file)) 16 | 17 | pkg = Base.PkgId(Base.UUID(0x438e738f_606a_5dbb_bf0a_cddfbfd45ab0), "PyCall") 18 | modpath = Base.locate_package(pkg) 19 | if modpath !== nothing 20 | PyCall_depsfile = joinpath(dirname(modpath),"..","deps","deps.jl") 21 | if isfile(PyCall_depsfile) 22 | include(PyCall_depsfile) 23 | println(pyprogramname) 24 | println(libpython) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /src/julia/juliainfo.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import os 4 | import subprocess 5 | import sys 6 | import warnings 7 | from logging import getLogger # see `.core.logger` 8 | 9 | from .find_libpython import linked_libpython 10 | 11 | try: 12 | from os.path import samefile 13 | except ImportError: 14 | # For Python < 3.2 in Windows: 15 | def samefile(f1, f2): 16 | a = os.path.realpath(os.path.normcase(f1)) 17 | b = os.path.realpath(os.path.normcase(f2)) 18 | return a == b 19 | 20 | 21 | logger = getLogger("julia") 22 | 23 | 24 | class JuliaInfo(object): 25 | """ 26 | Information required for initializing Julia runtime. 27 | 28 | Examples 29 | -------- 30 | >>> from julia.api import JuliaInfo 31 | >>> info = JuliaInfo.load() 32 | >>> info = JuliaInfo.load(julia="julia") # equivalent 33 | >>> info = JuliaInfo.load(julia="PATH/TO/julia") # doctest: +SKIP 34 | >>> info.julia 35 | 'julia' 36 | >>> info.sysimage # doctest: +SKIP 37 | '/home/user/julia/lib/julia/sys.so' 38 | >>> info.python # doctest: +SKIP 39 | '/usr/bin/python3' 40 | >>> info.is_compatible_python() # doctest: +SKIP 41 | True 42 | 43 | Attributes 44 | ---------- 45 | julia : str 46 | Path to a Julia executable from which information was retrieved. 47 | bindir : str 48 | ``Sys.BINDIR`` of `julia`. 49 | libjulia_path : str 50 | Path to libjulia. 51 | sysimage : str 52 | Path to system image. 53 | python : str 54 | Python executable with which PyCall.jl is configured. 55 | libpython_path : str 56 | libpython path used by PyCall.jl. 57 | """ 58 | 59 | @classmethod 60 | def load(cls, julia="julia", **popen_kwargs): 61 | """ 62 | Get basic information from `julia`. 63 | """ 64 | 65 | juliainfo_script = os.path.join( 66 | os.path.dirname(os.path.realpath(__file__)), "juliainfo.jl" 67 | ) 68 | proc = subprocess.Popen( 69 | [julia, "--startup-file=no", juliainfo_script], 70 | stdout=subprocess.PIPE, 71 | stderr=subprocess.PIPE, 72 | universal_newlines=True, 73 | **popen_kwargs 74 | ) 75 | 76 | stdout, stderr = proc.communicate() 77 | retcode = proc.wait() 78 | if retcode != 0: 79 | logger.debug("STDOUT from %s:\n%s", julia, stdout) 80 | logger.debug("STDERR from %s:\n%s", julia, stderr) 81 | if sys.version_info[0] < 3: 82 | output = "\n".join(["STDOUT:", stdout, "STDERR:", stderr]) 83 | raise subprocess.CalledProcessError( 84 | retcode, [julia, "-e", "..."], output 85 | ) 86 | else: 87 | raise subprocess.CalledProcessError( 88 | retcode, [julia, "-e", "..."], stdout, stderr 89 | ) 90 | 91 | stderr = stderr.strip() 92 | if stderr: 93 | warnings.warn("{} warned:\n{}".format(julia, stderr)) 94 | 95 | args = stdout.rstrip().split("\n") 96 | 97 | return cls(julia, *args) 98 | 99 | def __init__( 100 | self, 101 | julia, 102 | version_raw, 103 | version_major, 104 | version_minor, 105 | version_patch, 106 | bindir=None, 107 | libjulia_path=None, 108 | sysimage=None, 109 | python=None, 110 | libpython_path=None, 111 | ): 112 | self.julia = julia 113 | self.bindir = bindir 114 | self.libjulia_path = libjulia_path 115 | self.sysimage = sysimage 116 | 117 | version_major = int(version_major) 118 | version_minor = int(version_minor) 119 | version_patch = int(version_patch) 120 | self.version_raw = version_raw 121 | self.version_major = version_major 122 | self.version_minor = version_minor 123 | self.version_patch = version_patch 124 | self.version_info = (version_major, version_minor, version_patch) 125 | 126 | self.python = python 127 | self.libpython_path = libpython_path 128 | 129 | logger.debug("pyprogramname = %s", python) 130 | logger.debug("sys.executable = %s", sys.executable) 131 | logger.debug("bindir = %s", bindir) 132 | logger.debug("libjulia_path = %s", libjulia_path) 133 | 134 | def is_pycall_built(self): 135 | return bool(self.libpython_path) 136 | 137 | def is_compatible_python(self): 138 | """ 139 | Check if python used by PyCall.jl is compatible with `sys.executable`. 140 | """ 141 | return self.libpython_path and is_compatible_exe(self.libpython_path) 142 | 143 | 144 | def is_compatible_exe(jl_libpython): 145 | """ 146 | Determine if `libpython` is compatible with this Python. 147 | 148 | Current Python executable is considered compatible if it is dynamically 149 | linked to libpython and both of them are using identical libpython. If 150 | this function returns `True`, PyJulia use the same precompilation cache 151 | of PyCall.jl used by Julia itself. 152 | """ 153 | py_libpython = linked_libpython() 154 | logger.debug("py_libpython = %s", py_libpython) 155 | logger.debug("jl_libpython = %s", jl_libpython) 156 | dynamically_linked = py_libpython is not None 157 | return dynamically_linked and samefile(py_libpython, jl_libpython) 158 | # `py_libpython is not None` here for checking if this Python 159 | # executable is dynamically linked or not (`py_libpython is None` 160 | # if it's statically linked). `jl_libpython` may be `None` if 161 | # libpython used for PyCall is removed so we can't expect 162 | # `jl_libpython` to be a `str` always. 163 | -------------------------------------------------------------------------------- /src/julia/magic.py: -------------------------------------------------------------------------------- 1 | """ 2 | ========================== 3 | Julia magics for IPython 4 | ========================== 5 | 6 | {JULIAMAGICS_DOC} 7 | 8 | Usage 9 | ===== 10 | 11 | ``%%julia`` 12 | 13 | {JULIA_DOC} 14 | """ 15 | 16 | # ---------------------------------------------------------------------------- 17 | # Imports 18 | # ---------------------------------------------------------------------------- 19 | 20 | from __future__ import absolute_import, print_function 21 | 22 | import inspect 23 | import sys 24 | import warnings 25 | 26 | from IPython.core.magic import Magics, line_cell_magic, magics_class 27 | from traitlets import Bool, Enum 28 | 29 | from .core import Julia, JuliaError 30 | from .tools import redirect_output_streams 31 | 32 | try: 33 | unicode 34 | except NameError: 35 | unicode = str 36 | 37 | try: 38 | from IPython.core.magic import no_var_expand 39 | except ImportError: 40 | 41 | def no_var_expand(f): 42 | return f 43 | 44 | 45 | # ---------------------------------------------------------------------------- 46 | # Main classes 47 | # ---------------------------------------------------------------------------- 48 | 49 | 50 | @magics_class 51 | class JuliaMagics(Magics): 52 | """A set of magics useful for interactive work with Julia.""" 53 | 54 | highlight = Bool( 55 | True, 56 | config=True, 57 | help=""" 58 | Enable highlighting in `%%julia` magic by monkey-patching 59 | IPython internal (`TerminalInteractiveShell`). 60 | """, 61 | ) 62 | completion = Bool( 63 | True, 64 | config=True, 65 | help=""" 66 | Enable code completion in `%julia` and `%%julia` magics by 67 | monkey-patching IPython internal (`IPCompleter`). 68 | """, 69 | ) 70 | redirect_output_streams = Enum( 71 | ["auto", True, False], 72 | "auto", 73 | config=True, 74 | help=""" 75 | Connect Julia's stdout and stderr to Python's standard stream. 76 | "auto" (default) means to do so only in Jupyter. 77 | """, 78 | ) 79 | revise = Bool( 80 | False, 81 | config=True, 82 | help=""" 83 | Enable Revise.jl integration. Revise.jl must be installed 84 | before using this option (run `using Pkg; Pkg.add("Revise")`). 85 | """, 86 | ) 87 | 88 | def __init__(self, shell): 89 | """ 90 | Parameters 91 | ---------- 92 | shell : IPython shell 93 | 94 | """ 95 | 96 | super(JuliaMagics, self).__init__(shell) 97 | print("Initializing Julia interpreter. This may take some time...", end="") 98 | # Flush, otherwise the Julia startup will keep stdout buffered 99 | sys.stdout.flush() 100 | self._julia = Julia(init_julia=True) 101 | print() 102 | 103 | @no_var_expand 104 | @line_cell_magic 105 | def julia(self, line, cell=None): 106 | """ 107 | Execute code in Julia, and pull some of the results back into the 108 | Python namespace. 109 | """ 110 | src = unicode(line if cell is None else cell) 111 | 112 | caller_frame = inspect.currentframe() 113 | if caller_frame is None: 114 | caller_frame = sys._getframe(3) # May not work. 115 | 116 | # We assume the caller's frame is the first parent frame not in the 117 | # IPython module. This seems to work with IPython back to ~v5, and 118 | # is at least somewhat immune to future IPython internals changes, 119 | # although by no means guaranteed to be perfect. 120 | while any( 121 | ( 122 | caller_frame.f_globals.get("__name__").startswith("IPython"), 123 | caller_frame.f_globals.get("__name__").startswith("julia"), 124 | ) 125 | ): 126 | caller_frame = caller_frame.f_back 127 | 128 | return_value = "nothing" if src.strip().endswith(";") else "" 129 | 130 | return self._julia.eval( 131 | """ 132 | _PyJuliaHelper.@prepare_for_pyjulia_call begin 133 | begin %s end 134 | %s 135 | end 136 | """ 137 | % (src, return_value) 138 | )(self.shell.user_ns, caller_frame.f_locals) 139 | 140 | 141 | # Add to the global docstring the class information. 142 | __doc__ = __doc__.format( 143 | JULIAMAGICS_DOC=" " * 8 + JuliaMagics.__doc__, 144 | JULIA_DOC=" " * 8 + JuliaMagics.julia.__doc__, 145 | ) 146 | 147 | 148 | def should_redirect_output_streams(): 149 | try: 150 | OutStream = sys.modules["ipykernel"].iostream.OutStream 151 | except (KeyError, AttributeError): 152 | return False 153 | return isinstance(sys.stdout, OutStream) 154 | 155 | 156 | # ---------------------------------------------------------------------------- 157 | # IPython registration entry point. 158 | # ---------------------------------------------------------------------------- 159 | 160 | 161 | def load_ipython_extension(ip): 162 | """Load the extension in IPython.""" 163 | 164 | # This is equivalent to `ip.register_magics(JuliaMagics)` (but it 165 | # let us access the instance of `JuliaMagics`): 166 | magics = JuliaMagics(shell=ip) 167 | ip.register_magics(magics) 168 | 169 | template = "Incompatible upstream libraries. Got ImportError: {}" 170 | if magics.highlight: 171 | try: 172 | from .ipy.monkeypatch_interactiveshell import patch_interactiveshell 173 | except ImportError as err: 174 | warnings.warn(template.format(err)) 175 | else: 176 | patch_interactiveshell(ip) 177 | 178 | if magics.completion: 179 | try: 180 | from .ipy.monkeypatch_completer import patch_ipcompleter 181 | except ImportError as err: 182 | warnings.warn(template.format(err)) 183 | else: 184 | patch_ipcompleter() 185 | 186 | if magics.redirect_output_streams is True or ( 187 | magics.redirect_output_streams == "auto" and should_redirect_output_streams() 188 | ): 189 | redirect_output_streams() 190 | 191 | if magics.revise: 192 | from .ipy.revise import register_revise_hook 193 | 194 | register_revise_hook(ip) 195 | -------------------------------------------------------------------------------- /src/julia/options.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import textwrap 4 | 5 | 6 | class OptionDescriptor(object): 7 | @property 8 | def dataname(self): 9 | return "_" + self.name 10 | 11 | def __get__(self, instance, owner): 12 | if instance is None: 13 | return self 14 | return getattr(instance, self.dataname, self.default) 15 | 16 | def cli_argument_name(self): 17 | name = {"bindir": "home"}.get(self.name, self.name) 18 | return "--" + name.replace("_", "-") 19 | 20 | def cli_argument_spec(self): 21 | return dict(help="Julia's ``{}`` option.".format(self.cli_argument_name())) 22 | # TODO: parse help from `options_docs`. 23 | 24 | 25 | class String(OptionDescriptor): 26 | def __init__(self, name, default=None): 27 | self.name = name 28 | self.default = default 29 | 30 | def __set__(self, instance, value): 31 | if instance is None: 32 | raise AttributeError(self.name) 33 | if value == self.default or isinstance(value, str): 34 | setattr(instance, self.dataname, value) 35 | else: 36 | raise ValueError( 37 | "Option {!r} only accepts `str`. Got: {!r}".format(self.name, value) 38 | ) 39 | 40 | def _domain(self): # used in test 41 | return str 42 | 43 | 44 | class IntEtc(OptionDescriptor): 45 | def __init__(self, name, *, etc={}): 46 | self.name = name 47 | self.default = etc 48 | 49 | def __set__(self, instance, value): 50 | if instance is None: 51 | raise AttributeError(self.name) 52 | elif value in {None, *self.default} or isinstance(value, int): 53 | setattr(instance, self.dataname, value) 54 | else: 55 | if self.default: 56 | part = " or " + " ".join(map(str, self.default)) 57 | else: 58 | part = "" 59 | raise ValueError( 60 | f"Option {self.name} only accepts integers{part}. Got: {value}" 61 | ) 62 | 63 | def _domain(self): 64 | return {int, *self.default} 65 | 66 | def cli_argument_spec(self): 67 | return dict( 68 | super(IntEtc, self).cli_argument_spec(), 69 | choices=list(self.default) + ["1", "2", "3", "..."], 70 | ) 71 | 72 | 73 | class Choices(OptionDescriptor): 74 | def __init__(self, name, choicemap, default=None): 75 | self.name = name 76 | self.choicemap = choicemap 77 | self.default = default 78 | 79 | def __set__(self, instance, value): 80 | if instance is None: 81 | raise AttributeError(self.name) 82 | if value == self.default: 83 | setattr(instance, self.dataname, value) 84 | elif value in self.choicemap: 85 | setattr(instance, self.dataname, self.choicemap[value]) 86 | else: 87 | raise ValueError( 88 | "Option {!r} does not accept {!r}".format(self.name, value) 89 | ) 90 | 91 | def _domain(self): # used in test 92 | return set(self.choicemap) 93 | 94 | def cli_argument_spec(self): 95 | return dict( 96 | super(Choices, self).cli_argument_spec(), 97 | choices=list(self.choicemap.values()), 98 | ) 99 | 100 | 101 | def yes_no_etc(*etc): 102 | choicemap = {True: "yes", False: "no", "yes": "yes", "no": "no"} 103 | for v in etc: 104 | choicemap[v] = v 105 | return choicemap 106 | 107 | 108 | options_docs = """ 109 | bindir: str 110 | Set location of `julia` executable relative to which we find 111 | system image (``sys.so``). It is inferred from `runtime` if not 112 | given. Equivalent to ``--home`` of the Julia CLI. 113 | 114 | check_bounds: {True, False, 'yes', 'no'} 115 | Emit bounds checks always or never (ignoring declarations). 116 | `True` and `False` are synonym of ``'yes'`` and ``'no'``, respectively. 117 | This applies to all other options. 118 | 119 | compile: {True, False, 'yes', 'no', 'all', 'min'} 120 | Enable or disable JIT compiler, or request exhaustive compilation. 121 | 122 | compiled_modules: {True, False, 'yes', 'no'} 123 | Enable or disable incremental precompilation of modules. 124 | 125 | depwarn: {True, False, 'yes', 'no', 'error'} 126 | Enable or disable syntax and method deprecation warnings ("error" 127 | turns warnings into errors). 128 | 129 | inline: {True, False, 'yes', 'no'} 130 | Control whether inlining is permitted, including overriding 131 | @inline declarations. 132 | 133 | optimize: {0, 1, 2, 3} 134 | Set the optimization level (default level is 2 if unspecified or 3 135 | if used without a level). 136 | 137 | sysimage: str 138 | Start up with the given system image file. 139 | 140 | warn_overwrite: {True, False, 'yes', 'no'} 141 | Enable or disable method overwrite warnings. 142 | 143 | min_optlevel: {0, 1, 2, 3} 144 | Lower bound on the optimization level. 145 | 146 | threads: {int, 'auto'} 147 | How many threads to use. 148 | """ 149 | 150 | 151 | class JuliaOptions(object): 152 | """ 153 | Julia options validator. 154 | 155 | Attributes 156 | ---------- 157 | """ 158 | 159 | __doc__ = textwrap.dedent(__doc__) + options_docs 160 | 161 | # `options_docs` defined above must be updated when changing the 162 | # list of options supported by `JuliaOptions`. `test_options_docs` 163 | # tests that `options_docs` matches with the definition of 164 | # `JuliaOptions` 165 | 166 | sysimage = String("sysimage") 167 | bindir = String("bindir") 168 | compiled_modules = Choices("compiled_modules", yes_no_etc()) 169 | compile = Choices("compile", yes_no_etc("all", "min")) 170 | depwarn = Choices("depwarn", yes_no_etc("error")) 171 | warn_overwrite = Choices("warn_overwrite", yes_no_etc()) 172 | min_optlevel = Choices("min_optlevel", dict(zip(range(4), map(str, range(4))))) 173 | optimize = Choices("optimize", dict(zip(range(4), map(str, range(4))))) 174 | inline = Choices("inline", yes_no_etc()) 175 | check_bounds = Choices("check_bounds", yes_no_etc()) 176 | threads = IntEtc("threads", etc={"auto"}) 177 | 178 | def __init__(self, **kwargs): 179 | unsupported = [] 180 | for (name, value) in kwargs.items(): 181 | if self.is_supported(name): 182 | setattr(self, name, value) 183 | else: 184 | unsupported.append(name) 185 | if unsupported: 186 | raise TypeError( 187 | "Unsupported Julia option(s): {}".format(", ".join(unsupported)) 188 | ) 189 | 190 | @classmethod 191 | def is_supported(cls, name): 192 | return isinstance(getattr(cls, name, None), OptionDescriptor) 193 | 194 | def is_specified(self, name): 195 | desc = getattr(self.__class__, name) 196 | if isinstance(desc, OptionDescriptor): 197 | return getattr(self, name) != desc.default 198 | return False 199 | 200 | def specified(self): 201 | for name in dir(self.__class__): 202 | if self.is_specified(name): 203 | yield getattr(self.__class__, name), getattr(self, name) 204 | 205 | def as_args(self): 206 | args = [] 207 | for (desc, value) in self.specified(): 208 | if value is None: 209 | ... 210 | elif len(desc.cli_argument_name()) == 1: 211 | args.append(desc.cli_argument_name() + str(value)) 212 | else: 213 | args.append(desc.cli_argument_name() + "=" + str(value)) 214 | return args 215 | 216 | @classmethod 217 | def supported_options(cls): 218 | for name in dir(cls): 219 | if cls.is_supported(name): 220 | yield getattr(cls, name) 221 | 222 | 223 | def parse_jl_options(options): 224 | """ 225 | Parse --home and --sysimage options. 226 | 227 | Examples 228 | -------- 229 | >>> ns = parse_jl_options(["--home", "PATH/TO/HOME"]) 230 | >>> ns 231 | Namespace(home='PATH/TO/HOME', sysimage=None) 232 | >>> ns.home 233 | 'PATH/TO/HOME' 234 | >>> parse_jl_options([]) 235 | Namespace(home=None, sysimage=None) 236 | >>> parse_jl_options(["-HHOME", "--sysimage=PATH/TO/sys.so"]) 237 | Namespace(home='HOME', sysimage='PATH/TO/sys.so') 238 | """ 239 | import argparse 240 | 241 | def exit(*_): 242 | raise Exception("`exit` must not be called") 243 | 244 | def error(message): 245 | raise RuntimeError(message) 246 | 247 | parser = argparse.ArgumentParser() 248 | parser.add_argument("--home", "-H") 249 | parser.add_argument("--sysimage", "-J") 250 | 251 | parser.exit = exit 252 | parser.error = error 253 | ns, _ = parser.parse_known_args(options) 254 | return ns 255 | -------------------------------------------------------------------------------- /src/julia/patch.jl: -------------------------------------------------------------------------------- 1 | try 2 | julia_py = ENV["_PYJULIA_JULIA_PY"] 3 | 4 | if Base.julia_cmd().exec[1] == julia_py 5 | @debug "Already monkey-patched. Skipping..." julia_py getpid() 6 | else 7 | @debug "Monkey-patching..." julia_py getpid() 8 | 9 | # Monkey patch `Base.package_slug` 10 | # 11 | # This is used for generating the set of precompilation cache paths 12 | # for PyJulia different to the standard Julia runtime. 13 | # 14 | # See also: 15 | # * Suggestion: Use different precompilation cache path for different 16 | # system image -- https://github.com/JuliaLang/julia/pull/29914 17 | # 18 | if VERSION < v"1.4.0-DEV.389" 19 | Base.eval( 20 | Base, 21 | quote 22 | function package_slug(uuid::UUID, p::Int = 5) 23 | crc = _crc32c(uuid) 24 | crc = _crc32c(unsafe_string(JLOptions().image_file), crc) 25 | crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) 26 | crc = _crc32c($julia_py, crc) 27 | return slug(crc, p) 28 | end 29 | end, 30 | ) 31 | else 32 | Base.eval(Base, quote 33 | function package_slug(uuid::UUID, p::Int = 5) 34 | crc = _crc32c(uuid) 35 | crc = _crc32c($julia_py, crc) 36 | return slug(crc, p) 37 | end 38 | end) 39 | end 40 | 41 | # Monkey patch `Base.julia_exename`. 42 | # 43 | # This is required for propagating the monkey patches to subprocesses. 44 | # This is important especially for the subprocesses used for 45 | # precompilation. 46 | # 47 | # See also: 48 | # * Request: Add an API for configuring julia_cmd -- 49 | # https://github.com/JuliaLang/julia/pull/30065 50 | # 51 | Base.eval(Base, quote 52 | julia_exename() = $julia_py 53 | end) 54 | @assert Base.julia_cmd().exec[1] == julia_py 55 | 56 | @debug "Successfully monkey-patched" julia_py getpid() 57 | end 58 | catch err 59 | @error "Failed to monkey-patch `julia`" exception = (err, catch_backtrace()) getpid() 60 | rethrow() 61 | end 62 | -------------------------------------------------------------------------------- /src/julia/precompile.jl: -------------------------------------------------------------------------------- 1 | using PyCall 2 | -------------------------------------------------------------------------------- /src/julia/pseudo_python_cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pseudo Python command line interface. 3 | 4 | It tries to mimic a subset of Python CLI: 5 | https://docs.python.org/3/using/cmdline.html 6 | """ 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import code 11 | import copy 12 | import runpy 13 | import sys 14 | import traceback 15 | from collections import namedtuple 16 | 17 | try: 18 | from types import SimpleNamespace 19 | except ImportError: 20 | from argparse import Namespace as SimpleNamespace 21 | 22 | 23 | ARGUMENT_HELP = """ 24 | positional arguments: 25 | script path to file (default: None) 26 | args arguments passed to program in sys.argv[1:] 27 | 28 | optional arguments: 29 | -h, --help show this help message and exit 30 | -i inspect interactively after running script. 31 | --version, -V Print the Python version number and exit. 32 | -VV is not supported. 33 | -c COMMAND Execute the Python code in COMMAND. 34 | -m MODULE Search sys.path for the named MODULE and execute its contents 35 | as the __main__ module. 36 | """ 37 | 38 | 39 | def python(module, command, script, args, interactive): 40 | if command: 41 | sys.argv[0] = "-c" 42 | 43 | assert sys.argv 44 | sys.argv[1:] = args 45 | if script: 46 | sys.argv[0] = script 47 | 48 | banner = "" 49 | try: 50 | if command: 51 | scope = {} 52 | exec(command, scope) 53 | elif module: 54 | scope = runpy.run_module(module, run_name="__main__", alter_sys=True) 55 | elif script == "-": 56 | source = sys.stdin.read() 57 | exec(compile(source, "", "exec"), scope) 58 | elif script: 59 | scope = runpy.run_path(script, run_name="__main__") 60 | else: 61 | interactive = True 62 | scope = None 63 | banner = None # show banner 64 | except Exception: 65 | if not interactive: 66 | raise 67 | traceback.print_exc() 68 | 69 | if interactive: 70 | code.interact(banner=banner, local=scope) 71 | 72 | 73 | ArgDest = namedtuple("ArgDest", "dest names default") 74 | Optional = namedtuple("Optional", "name is_long argdest nargs action terminal") 75 | Result = namedtuple("Result", "option values") 76 | 77 | 78 | class PyArgumentParser(object): 79 | 80 | """ 81 | `ArgumentParser`-like parser with "terminal option" support. 82 | 83 | Major differences: 84 | 85 | * Formatted help has to be provided to `description`. 86 | * Many options for `.add_argument` are not supported. 87 | * Especially, there is no positional argument support: all positional 88 | arguments go into `ns.args`. 89 | * `.add_argument` can take boolean option `terminal` (default: `False`) 90 | to stop parsing after consuming the given option. 91 | """ 92 | 93 | def __init__(self, prog=None, usage="%(prog)s [options] [args]", description=""): 94 | self.prog = sys.argv[0] if prog is None else prog 95 | self.usage = usage 96 | self.description = description 97 | 98 | self._dests = ["args"] 99 | self._argdests = [ArgDest("args", (), [])] 100 | self._options = [] 101 | 102 | self.add_argument("--help", "-h", "-?", action="store_true") 103 | 104 | def format_usage(self): 105 | return "usage: " + self.usage % {"prog": self.prog} 106 | 107 | # Once we drop Python 2, we can do: 108 | """ 109 | def add_argument(self, name, *alt, dest=None, nargs=None, action=None, 110 | default=None, terminal=False): 111 | """ 112 | 113 | def add_argument(self, name, *alt, **kwargs): 114 | return self._add_argument_impl(name, alt, **kwargs) 115 | 116 | # fmt: off 117 | 118 | def _add_argument_impl(self, name, alt, dest=None, nargs=None, action=None, 119 | default=None, terminal=False): 120 | if dest is None: 121 | if name.startswith("--"): 122 | dest = name[2:] 123 | elif not name.startswith("-"): 124 | dest = name 125 | else: 126 | raise ValueError(name) 127 | 128 | if not name.startswith("-"): 129 | raise NotImplementedError( 130 | "Positional arguments are not supported." 131 | " All positional arguments will be stored in `ns.args`.") 132 | if terminal and action is not None: 133 | raise NotImplementedError( 134 | "Terminal option is assumed to have argument." 135 | " Non-`None` action={} is not supported".format()) 136 | 137 | if nargs is not None and action is not None: 138 | raise TypeError("`nargs` and `action` are mutually exclusive") 139 | if action == "store_true": 140 | nargs = 0 141 | assert nargs is None or isinstance(nargs, int) 142 | assert action in (None, "store_true") 143 | 144 | assert dest not in self._dests 145 | self._dests.append(dest) 146 | 147 | argdest = ArgDest( 148 | dest=dest, 149 | names=(name,) + alt, 150 | default=default, 151 | ) 152 | self._argdests.append(argdest) 153 | 154 | for arg in (name,) + alt: 155 | self._options.append(Optional( 156 | name=arg, 157 | is_long=arg.startswith("--"), 158 | argdest=argdest, 159 | nargs=nargs, 160 | action=action, 161 | terminal=terminal, 162 | )) 163 | 164 | def parse_args(self, args): 165 | ns = SimpleNamespace(**{ 166 | argdest.dest: copy.copy(argdest.default) 167 | for argdest in self._argdests 168 | }) 169 | args_iter = iter(args) 170 | self._parse_until_terminal(ns, args_iter) 171 | ns.args.extend(args_iter) 172 | 173 | if ns.help: 174 | self.print_help() 175 | self.exit() 176 | del ns.help 177 | 178 | return ns 179 | 180 | def _parse_until_terminal(self, ns, args_iter): 181 | seen = set() 182 | for a in args_iter: 183 | 184 | results = self._find_matches(a) 185 | if not results: 186 | ns.args.append(a) 187 | break 188 | 189 | for i, res in enumerate(results): 190 | dest = res.option.argdest.dest 191 | if dest in seen: 192 | self._usage_and_error( 193 | "{} provided more than twice" 194 | .format(", ".join(res.option.argdest.names))) 195 | seen.add(dest) 196 | 197 | num_args = res.option.nargs 198 | if num_args is None: 199 | num_args = 1 200 | while len(res.values) < num_args: 201 | try: 202 | res.values.append(next(args_iter)) 203 | except StopIteration: 204 | self.error(self.format_usage()) 205 | 206 | if res.option.action == "store_true": 207 | setattr(ns, dest, True) 208 | else: 209 | value = res.values 210 | if res.option.nargs is None: 211 | value, = value 212 | setattr(ns, dest, value) 213 | 214 | if res.option.terminal: 215 | assert i == len(results) - 1 216 | return 217 | 218 | def _find_matches(self, arg): 219 | """ 220 | Return a list of `.Result`. 221 | 222 | If value presents in `arg` (i.e., ``--long-option=value``), it 223 | becomes the element of `.Result.values` (a list). Otherwise, 224 | this list has to be filled by the caller (`_parse_until_terminal`). 225 | """ 226 | for opt in self._options: 227 | if arg == opt.name: 228 | return [Result(opt, [])] 229 | elif arg.startswith(opt.name): 230 | # i.e., len(arg) > len(opt.name): 231 | if opt.is_long and arg[len(opt.name)] == "=": 232 | return [Result(opt, [arg[len(opt.name) + 1:]])] 233 | elif not opt.is_long: 234 | if opt.nargs != 0: 235 | return [Result(opt, [arg[len(opt.name):]])] 236 | else: 237 | results = [Result(opt, [])] 238 | rest = "-" + arg[len(opt.name):] 239 | results.extend(self._find_matches(rest)) 240 | return results 241 | # arg="-ih" -> rest="-h" 242 | return [] 243 | # fmt: on 244 | 245 | def print_usage(self, file=None): 246 | print(self.format_usage(), file=file or sys.stdout) 247 | 248 | def print_help(self): 249 | self.print_usage() 250 | print() 251 | print(self.description) 252 | 253 | def exit(self, status=0): 254 | sys.exit(status) 255 | 256 | def _usage_and_error(self, message): 257 | self.print_usage(sys.stderr) 258 | print(file=sys.stderr) 259 | self.error(message) 260 | 261 | def error(self, message): 262 | print(message, file=sys.stderr) 263 | self.exit(2) 264 | 265 | 266 | def make_parser(description=__doc__ + ARGUMENT_HELP): 267 | parser = PyArgumentParser( 268 | prog=None if sys.argv[0] else "python", 269 | usage="%(prog)s [option] ... [-c cmd | -m mod | script | -] [args]", 270 | description=description, 271 | ) 272 | 273 | parser.add_argument("-i", dest="interactive", action="store_true") 274 | parser.add_argument("--version", "-V", action="store_true") 275 | parser.add_argument("-c", dest="command", terminal=True) 276 | parser.add_argument("-m", dest="module", terminal=True) 277 | 278 | return parser 279 | 280 | 281 | def parse_args_with(parser, args): 282 | ns = parser.parse_args(args) 283 | 284 | if ns.command and ns.module: 285 | parser.error("-c and -m are mutually exclusive") 286 | if ns.version: 287 | print("Python {0}.{1}.{2}".format(*sys.version_info)) 288 | parser.exit() 289 | del ns.version 290 | 291 | ns.script = None 292 | if (not (ns.command or ns.module)) and ns.args: 293 | ns.script = ns.args[0] 294 | ns.args = ns.args[1:] 295 | 296 | return ns 297 | 298 | 299 | def parse_args(args): 300 | return parse_args_with(make_parser(), args) 301 | 302 | 303 | def main(args=None): 304 | if args is None: 305 | args = sys.argv[1:] 306 | try: 307 | ns = parse_args(args) 308 | python(**vars(ns)) 309 | except SystemExit as err: 310 | return err.code 311 | except Exception: 312 | traceback.print_exc() 313 | return 1 314 | 315 | 316 | if __name__ == "__main__": 317 | sys.exit(main()) 318 | -------------------------------------------------------------------------------- /src/julia/pyjulia_helper.jl: -------------------------------------------------------------------------------- 1 | module _PyJuliaHelper 2 | 3 | const REPL = 4 | Base.require(Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL")) 5 | const PyCall = 6 | Base.require(Base.PkgId(Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0"), "PyCall")) 7 | const MacroTools = Base.require(Base.PkgId( 8 | Base.UUID("1914dd2f-81c6-5fcd-8719-6d5c9610ff09"), 9 | "MacroTools", 10 | )) 11 | 12 | using .PyCall 13 | using .PyCall: Py_eval_input, Py_file_input, pyeval_ 14 | using .MacroTools: isexpr, walk 15 | 16 | """ 17 | fullnamestr(m) 18 | 19 | # Examples 20 | ```jldoctest 21 | julia> fullnamestr(Base.Enums) 22 | "Base.Enums" 23 | ``` 24 | """ 25 | fullnamestr(m) = join(fullname(m), ".") 26 | 27 | isdefinedstr(parent, member) = isdefined(parent, Symbol(member)) 28 | 29 | function completions(str, pos) 30 | ret, ran, should_complete = REPL.completions(str, Int(pos)) 31 | return ( 32 | map(REPL.completion_text, ret), 33 | (first(ran), last(ran)), 34 | should_complete, 35 | ) 36 | end 37 | 38 | 39 | # takes an expression like `$foo + 1` and turns it into a pyfunction 40 | # `(globals,locals) -> convert(PyAny, pyeval_("foo",globals,locals,PyAny)) + 1` 41 | # so that Python code can call it and just pass the appropriate globals/locals 42 | # dicts to perform the interpolation. 43 | macro prepare_for_pyjulia_call(ex) 44 | 45 | # f(x, quote_depth) should return a transformed expression x and whether to 46 | # recurse into the new expression. quote_depth keeps track of how deep 47 | # inside of nested quote objects we arepyeval 48 | function stoppable_walk(f, x, quote_depth=1) 49 | (fx, recurse) = f(x, quote_depth) 50 | if isexpr(fx,:quote) 51 | quote_depth += 1 52 | end 53 | if isexpr(fx,:$) 54 | quote_depth -= 1 55 | end 56 | walk(fx, (recurse ? (x -> stoppable_walk(f,x,quote_depth)) : identity), identity) 57 | end 58 | 59 | function make_pyeval(globals, locals, expr::Union{String,Symbol}, options...) 60 | code = string(expr) 61 | T = length(options) == 1 && 'o' in options[1] ? PyObject : PyAny 62 | input_type = '\n' in code ? Py_file_input : Py_eval_input 63 | :($convert($T, $pyeval_($code, $globals, $locals, $input_type))) 64 | end 65 | 66 | function insert_pyevals(globals, locals, ex) 67 | stoppable_walk(ex) do x, quote_depth 68 | if quote_depth==1 && isexpr(x, :$) 69 | if x.args[1] isa Symbol 70 | make_pyeval(globals, locals, x.args[1]), false 71 | else 72 | error("""syntax error in: \$($(string(x.args[1]))) 73 | Use py"..." instead of \$(...) for interpolating Python expressions.""") 74 | end 75 | elseif quote_depth==1 && isexpr(x, :macrocall) 76 | if x.args[1]==Symbol("@py_str") 77 | # in Julia 0.7+, x.args[2] is a LineNumberNode, so filter it out 78 | # in a way that's compatible with Julia 0.6: 79 | make_pyeval(globals, locals, filter(s->(s isa String), x.args[2:end])...), false 80 | else 81 | x, false 82 | end 83 | else 84 | x, true 85 | end 86 | end 87 | end 88 | 89 | esc(quote 90 | $pyfunction( 91 | (globals, locals)->Base.eval(Main, $insert_pyevals(globals, locals, $(QuoteNode(ex)))), 92 | $PyObject, $PyObject 93 | ) 94 | end) 95 | end 96 | 97 | 98 | module IOPiper 99 | 100 | const orig_stdin = Ref{IO}() 101 | const orig_stdout = Ref{IO}() 102 | const orig_stderr = Ref{IO}() 103 | 104 | function __init__() 105 | orig_stdin[] = stdin 106 | orig_stdout[] = stdout 107 | orig_stderr[] = stderr 108 | end 109 | 110 | """ 111 | num_utf8_trailing(d::Vector{UInt8}) 112 | 113 | If `d` ends with an incomplete UTF8-encoded character, return the number of trailing incomplete bytes. 114 | Otherwise, return `0`. 115 | 116 | Taken from IJulia.jl. 117 | """ 118 | function num_utf8_trailing(d::Vector{UInt8}) 119 | i = length(d) 120 | # find last non-continuation byte in d: 121 | while i >= 1 && ((d[i] & 0xc0) == 0x80) 122 | i -= 1 123 | end 124 | i < 1 && return 0 125 | c = d[i] 126 | # compute number of expected UTF-8 bytes starting at i: 127 | n = c <= 0x7f ? 1 : c < 0xe0 ? 2 : c < 0xf0 ? 3 : 4 128 | nend = length(d) + 1 - i # num bytes from i to end 129 | return nend == n ? 0 : nend 130 | end 131 | 132 | function pipe_stream(sender::IO, receiver, buf::IO = IOBuffer()) 133 | try 134 | while !eof(sender) 135 | nb = bytesavailable(sender) 136 | write(buf, read(sender, nb)) 137 | 138 | # Taken from IJulia.send_stream: 139 | d = take!(buf) 140 | n = num_utf8_trailing(d) 141 | dextra = d[end-(n-1):end] 142 | resize!(d, length(d) - n) 143 | s = String(copy(d)) 144 | 145 | write(buf, dextra) 146 | receiver(s) # check isvalid(String, s)? 147 | end 148 | catch e 149 | if !isa(e, InterruptException) 150 | rethrow() 151 | end 152 | pipe_stream(sender, receiver, buf) 153 | end 154 | end 155 | 156 | const read_stdout = Ref{Base.PipeEndpoint}() 157 | const read_stderr = Ref{Base.PipeEndpoint}() 158 | 159 | function pipe_std_outputs(out_receiver, err_receiver) 160 | global readout_task 161 | global readerr_task 162 | read_stdout[], = redirect_stdout() 163 | readout_task = @async pipe_stream(read_stdout[], out_receiver) 164 | read_stderr[], = redirect_stderr() 165 | readerr_task = @async pipe_stream(read_stderr[], err_receiver) 166 | end 167 | 168 | end # module 169 | 170 | end # module 171 | -------------------------------------------------------------------------------- /src/julia/pytestplugin.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import sys 4 | 5 | import pytest 6 | 7 | from .options import JuliaOptions 8 | 9 | _USING_DEFAULT_SETUP = True 10 | 11 | 12 | def pytest_addoption(parser): 13 | import os 14 | 15 | # Note: the help strings have to be synchronized manually with 16 | # ../../docs/source/pytest.rst 17 | 18 | parser.addoption( 19 | "--no-julia", 20 | action="store_false", 21 | dest="julia", 22 | default=True, 23 | help="Skip tests that require julia.", 24 | ) 25 | parser.addoption( 26 | "--julia", 27 | action="store_true", 28 | dest="julia", 29 | default=True, 30 | help="Undo `--no-julia`; i.e., run tests that require julia.", 31 | ) 32 | parser.addoption( 33 | "--julia-runtime", 34 | help=""" 35 | Julia executable to be used. Defaults to environment variable 36 | `$PYJULIA_TEST_RUNTIME`. 37 | """, 38 | default=os.getenv("PYJULIA_TEST_RUNTIME", "julia"), 39 | ) 40 | 41 | for desc in JuliaOptions.supported_options(): 42 | parser.addoption( 43 | "--julia-{}".format(desc.cli_argument_name().lstrip("-")), 44 | **desc.cli_argument_spec() 45 | ) 46 | 47 | 48 | def pytest_sessionstart(session): 49 | from .core import LibJulia, JuliaInfo, Julia, enable_debug 50 | 51 | options = JuliaOptions() 52 | for desc in JuliaOptions.supported_options(): 53 | cli_option = "--julia-{}".format(desc.cli_argument_name().lstrip("-")) 54 | desc.__set__(options, session.config.getoption(cli_option)) 55 | 56 | julia_runtime = session.config.getoption("julia_runtime") 57 | 58 | global _USING_DEFAULT_SETUP 59 | _USING_DEFAULT_SETUP = not (julia_runtime != "julia" or options.as_args()) 60 | 61 | if not session.config.getoption("julia"): 62 | return 63 | 64 | enable_debug() 65 | global _JULIA_INFO 66 | _JULIA_INFO = info = JuliaInfo.load(julia=julia_runtime) 67 | 68 | if not info.is_pycall_built(): 69 | print( 70 | """ 71 | PyCall is not installed or built. Run the following code in Python REPL: 72 | 73 | >>> import julia 74 | >>> julia.install() 75 | 76 | See: 77 | https://pyjulia.readthedocs.io/en/latest/installation.html 78 | """, 79 | file=sys.stderr, 80 | ) 81 | pytest.exit("PyCall not built", returncode=1) 82 | 83 | if ( 84 | options.compiled_modules != "no" 85 | and not info.is_compatible_python() 86 | and info.version_info >= (0, 7) 87 | ): 88 | print( 89 | """ 90 | PyJulia does not fully support this combination of Julia and Python. 91 | Try: 92 | 93 | * Pass `--julia-compiled-modules=no` option to disable 94 | precompilation cache. 95 | 96 | * Use `--julia-runtime` option to specify different Julia 97 | executable. 98 | 99 | * Pass `--no-julia` to run tests that do not rely on Julia 100 | runtime. 101 | """, 102 | file=sys.stderr, 103 | ) 104 | pytest.exit("incompatible runtimes", returncode=1) 105 | 106 | api = LibJulia.from_juliainfo(info) 107 | api.init_julia(options) 108 | 109 | 110 | # Initialize Julia runtime as soon as possible (or more precisely 111 | # before importing any additional Python modules) to avoid, e.g., 112 | # incompatibility of `libstdc++`. 113 | # 114 | # See: 115 | # https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_sessionstart 116 | 117 | 118 | def pytest_configure(config): 119 | config.addinivalue_line( 120 | "markers", "julia: mark tests to be skipped with --no-julia." 121 | ) 122 | # https://docs.pytest.org/en/latest/writing_plugins.html#registering-markers 123 | # https://docs.pytest.org/en/latest/mark.html#registering-marks 124 | 125 | 126 | @pytest.fixture(scope="session") 127 | def julia(request): 128 | """pytest fixture for providing a `Julia` instance.""" 129 | if not request.config.getoption("julia"): 130 | pytest.skip("--no-julia is given.") 131 | 132 | from .core import Julia 133 | 134 | return Julia() 135 | 136 | 137 | @pytest.fixture(scope="session") 138 | def juliainfo(julia): 139 | """pytest fixture for providing `JuliaInfo` instance.""" 140 | return _JULIA_INFO 141 | 142 | 143 | def pytest_runtest_setup(item): 144 | if not item.config.getoption("julia"): 145 | for mark in item.iter_markers("julia"): 146 | pytest.skip("--no-julia is given.") 147 | 148 | if not _USING_DEFAULT_SETUP: 149 | for mark in item.iter_markers("pyjulia__using_default_setup"): 150 | pytest.skip( 151 | "using non-default setup (e.g., --julia- is given)" 152 | ) 153 | -------------------------------------------------------------------------------- /src/julia/python_jl.py: -------------------------------------------------------------------------------- 1 | """ 2 | Python interpreter inside a Julia process. 3 | 4 | This command line interface mimics a basic subset of Python program so that 5 | Python program involving calls to Julia functions can be run in a *Julia* 6 | process. This avoids the known problem with pre-compilation cache in 7 | Debian-based distribution such as Ubuntu and Python executable installed by 8 | Conda in Linux. 9 | 10 | Although this script has -i option and it can do a basic REPL, contrl-c may 11 | crash the whole process. Consider using IPython >= 7 which can be launched 12 | by:: 13 | 14 | python-jl -m IPython 15 | 16 | .. NOTE:: 17 | 18 | For this command to work, Python environment with which PyCall.jl is 19 | configured has to have PyJulia installed. 20 | """ 21 | 22 | from __future__ import absolute_import, print_function 23 | 24 | import sys 25 | 26 | from .pseudo_python_cli import ARGUMENT_HELP, make_parser, parse_args_with 27 | from .utils import execprog 28 | 29 | PYJL_ARGUMENT_HELP = ( 30 | ARGUMENT_HELP 31 | + """ 32 | --julia JULIA Julia runtime to be used. (default: julia) 33 | """ 34 | ) 35 | 36 | script_jl = """ 37 | import PyCall 38 | 39 | let code = PyCall.pyimport("julia.pseudo_python_cli")[:main](ARGS) 40 | if code isa Integer 41 | exit(code) 42 | end 43 | end 44 | """ 45 | 46 | 47 | def remove_julia_options(args): 48 | """ 49 | Remove options used in this Python process. 50 | 51 | >>> list(remove_julia_options(["a", "b", "c"])) 52 | ['a', 'b', 'c'] 53 | >>> list(remove_julia_options(["a", "--julia", "julia", "b", "c"])) 54 | ['a', 'b', 'c'] 55 | >>> list(remove_julia_options(["a", "b", "c", "--julia=julia"])) 56 | ['a', 'b', 'c'] 57 | """ 58 | it = iter(args) 59 | for a in it: 60 | if a == "--julia": 61 | try: 62 | next(it) 63 | except StopIteration: 64 | return 65 | continue 66 | elif a.startswith("--julia="): 67 | continue 68 | yield a 69 | 70 | 71 | def parse_pyjl_args(args): 72 | """ 73 | Return a pair of parsed result and "unused" arguments. 74 | 75 | Returns 76 | ------- 77 | ns : argparse.Namespace 78 | Parsed result. Only `ns.julia` is relevant here. 79 | unused_args : list 80 | Arguments to be parsed (again) by `.pseudo_python_cli.main`. 81 | 82 | Examples 83 | -------- 84 | >>> ns, unused_args = parse_pyjl_args([]) 85 | >>> ns.julia 86 | 'julia' 87 | >>> unused_args 88 | [] 89 | >>> ns, unused_args = parse_pyjl_args( 90 | ... ["--julia", "julia-dev", "-i", "-c", "import julia"]) 91 | >>> ns.julia 92 | 'julia-dev' 93 | >>> unused_args 94 | ['-i', '-c', 'import julia'] 95 | """ 96 | # Mix the options we need in this Python process with the Python 97 | # arguments to be parsed in the "subprocess". This way, we get a 98 | # parse error right now without initializing Julia runtime and 99 | # importing PyCall.jl etc. to get an extra speedup for the 100 | # abnormal case (including -h/--help and -V/--version). 101 | parser = make_parser(description=__doc__ + PYJL_ARGUMENT_HELP) 102 | parser.add_argument("--julia", default="julia") 103 | 104 | ns = parse_args_with(parser, args) 105 | unused_args = list(remove_julia_options(args)) 106 | return ns, unused_args 107 | 108 | 109 | def main(args=None): 110 | if args is None: 111 | args = sys.argv[1:] 112 | ns, unused_args = parse_pyjl_args(args) 113 | julia = ns.julia 114 | execprog([julia, "-e", script_jl, "--"] + unused_args) 115 | 116 | 117 | if __name__ == "__main__": 118 | main() 119 | -------------------------------------------------------------------------------- /src/julia/release.py: -------------------------------------------------------------------------------- 1 | # This file is executed via setup.py and imported via __init__.py 2 | 3 | __version__ = "0.6.2" 4 | # For Python versioning scheme, see: 5 | # https://www.python.org/dev/peps/pep-0440/#version-scheme 6 | -------------------------------------------------------------------------------- /src/julia/runtests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run tests for PyJulia. 3 | """ 4 | 5 | from __future__ import absolute_import, print_function 6 | 7 | import argparse 8 | import sys 9 | 10 | from .utils import execprog 11 | 12 | try: 13 | from shlex import quote 14 | except ImportError: 15 | from pipes import quote # Python 2.7 16 | 17 | 18 | class ApplicationError(RuntimeError): 19 | pass 20 | 21 | 22 | required_pytest = (3, 9) 23 | """ 24 | Required pytest version. 25 | 26 | This is a very loose lower bound because we abort `runtests` CLI if 27 | this does not match. 28 | """ 29 | 30 | msg_test_dependencies = """ 31 | Test dependencies are not installed. 32 | 33 | To run `julia.runtests`, use the following command to install `pytest`: 34 | {} -m pip install "julia[test]" 35 | 36 | Note that you may need to add option `--user` after `install`. 37 | """.format( 38 | quote(sys.executable) 39 | ).strip() 40 | 41 | 42 | def check_test_dependencies(): 43 | # See `extras_require` in setup.py 44 | try: 45 | import numpy 46 | import IPython.testing.tools # may require `mock` 47 | except ImportError as err: 48 | print(err, file=sys.stderr) 49 | raise ApplicationError(msg_test_dependencies) 50 | 51 | try: 52 | import pytest 53 | except ImportError as err: 54 | print(err, file=sys.stderr) 55 | raise ApplicationError(msg_test_dependencies) 56 | 57 | major, minor, _ = pytest.__version__.split(".", 2) 58 | if (int(major), int(minor)) < required_pytest: 59 | raise ApplicationError(msg_test_dependencies) 60 | 61 | 62 | def runtests(pytest_args, dry_run): 63 | check_test_dependencies() 64 | 65 | # TODO: Detect segfault and report. 66 | # TODO: Maybe integrate this script with `with_rebuilt`? 67 | cmd = [ 68 | sys.executable, 69 | "-m", 70 | "julia.with_rebuilt", 71 | "--", 72 | sys.executable, 73 | "-m", 74 | "pytest", 75 | "-p", 76 | "pytester", 77 | "-p", 78 | "julia.pytestplugin", 79 | "--doctest-modules", 80 | "--runpytest=subprocess", 81 | "--pyargs", 82 | "julia", 83 | ] 84 | cmd.extend(pytest_args) 85 | if dry_run: 86 | print(*map(quote, cmd)) 87 | return 88 | execprog(cmd) 89 | 90 | 91 | class CustomFormatter( 92 | argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter 93 | ): 94 | pass 95 | 96 | 97 | def main(args=None): 98 | parser = argparse.ArgumentParser( 99 | formatter_class=CustomFormatter, description=__doc__ 100 | ) 101 | parser.add_argument( 102 | "--dry-run", 103 | action="store_true", 104 | help=""" 105 | Print the command to be executed instead of actually running 106 | it. 107 | """, 108 | ) 109 | parser.add_argument( 110 | "pytest_args", 111 | nargs="*", 112 | help=""" 113 | Command line arguments to be passed to pytest. 114 | """, 115 | ) 116 | ns, pytest_args = parser.parse_known_args(args) 117 | if ns.pytest_args and pytest_args: 118 | parser.error( 119 | "Ambiguous arguments. Use `--` to separate pytest options" 120 | " from options for julia.runtests." 121 | ) 122 | if pytest_args: 123 | ns.pytest_args = pytest_args 124 | try: 125 | runtests(**vars(ns)) 126 | except ApplicationError as err: 127 | print(err, file=sys.stderr) 128 | sys.exit(1) 129 | 130 | 131 | if __name__ == "__main__": 132 | main() 133 | -------------------------------------------------------------------------------- /src/julia/sysimage.py: -------------------------------------------------------------------------------- 1 | """ 2 | Build system image. 3 | 4 | Example:: 5 | 6 | python3 -m julia.sysimage sys.so 7 | 8 | Generated system image can be passed to ``sysimage`` option of 9 | `julia.api.Julia`. 10 | 11 | .. note:: 12 | 13 | This script is not tested on Windows. 14 | """ 15 | 16 | from __future__ import absolute_import, print_function 17 | 18 | import argparse 19 | import os 20 | import shutil 21 | import subprocess 22 | import sys 23 | import tempfile 24 | from contextlib import contextmanager 25 | from logging import getLogger # see `.core.logger` 26 | 27 | from .core import enable_debug 28 | from .tools import _julia_version, julia_py_executable 29 | 30 | logger = getLogger("julia.sysimage") 31 | 32 | 33 | class KnownError(RuntimeError): 34 | pass 35 | 36 | 37 | def script_path(name): 38 | return os.path.join(os.path.dirname(os.path.realpath(__file__)), name) 39 | 40 | 41 | def install_packagecompiler_cmd(julia, compiler_env): 42 | cmd = [julia] 43 | if sys.stdout.isatty(): 44 | cmd.append("--color=yes") 45 | cmd.append(script_path("install-packagecompiler.jl")) 46 | cmd.append(compiler_env) 47 | return cmd 48 | 49 | 50 | def build_sysimage_cmd(julia_py, julia, compile_args): 51 | cmd = [julia_py, "--julia", julia] 52 | if _julia_version(julia) >= (1, 5, 0): 53 | # Avoid precompiling PackageCompiler.jl. See the notes in compile.jl. 54 | cmd.append("--compiled-modules=no") 55 | if sys.stdout.isatty(): 56 | cmd.append("--color=yes") 57 | cmd.append(script_path("compile.jl")) 58 | cmd.extend(compile_args) 59 | return cmd 60 | 61 | 62 | def check_call(cmd, **kwargs): 63 | logger.debug("Run %s", cmd) 64 | subprocess.check_call(cmd, **kwargs) 65 | 66 | 67 | @contextmanager 68 | def temporarydirectory(**kwargs): 69 | path = tempfile.mkdtemp(**kwargs) 70 | try: 71 | yield path 72 | finally: 73 | shutil.rmtree(path, ignore_errors=True) 74 | 75 | 76 | def build_sysimage( 77 | output, 78 | julia="julia", 79 | script=script_path("precompile.jl"), 80 | debug=False, 81 | compiler_env="", 82 | base_sysimage=None, 83 | ): 84 | if debug: 85 | enable_debug() 86 | 87 | if output.endswith(".a"): 88 | raise KnownError("Output file must not have extension .a") 89 | 90 | julia_py = julia_py_executable() 91 | 92 | with temporarydirectory(prefix="tmp.pyjulia.sysimage.") as path: 93 | if not compiler_env: 94 | compiler_env = os.path.join(path, "compiler_env") 95 | # Not using julia-py to install PackageCompiler to reduce 96 | # method re-definition warnings: 97 | check_call(install_packagecompiler_cmd(julia, compiler_env), cwd=path) 98 | 99 | # Arguments to ./compile.jl script: 100 | compile_args = [ 101 | compiler_env, 102 | # script -- ./precompile.jl by default 103 | os.path.realpath(script), 104 | # output -- path to sys.o file 105 | os.path.realpath(output), 106 | # optional base system image to build on 107 | "" if base_sysimage is None else os.path.realpath(base_sysimage), 108 | ] 109 | 110 | check_call(build_sysimage_cmd(julia_py, julia, compile_args), cwd=path) 111 | 112 | 113 | class CustomFormatter( 114 | argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter 115 | ): 116 | pass 117 | 118 | 119 | def main(args=None): 120 | parser = argparse.ArgumentParser( 121 | formatter_class=CustomFormatter, description=__doc__ 122 | ) 123 | parser.add_argument("--julia", default="julia") 124 | parser.add_argument("--debug", action="store_true", help="Print debug log.") 125 | parser.add_argument( 126 | "--script", 127 | default=script_path("precompile.jl"), 128 | help="Path to Julia script with precompile instructions.", 129 | ) 130 | parser.add_argument( 131 | "--compiler-env", 132 | default="", 133 | help=""" 134 | Path to a Julia project with PackageCompiler to be used for 135 | system image compilation. Create a temporary environment with 136 | appropriate PackageCompiler by default or when an empty string 137 | is given. 138 | """, 139 | ) 140 | parser.add_argument( 141 | "--base-sysimage", 142 | help=""" 143 | Path to a Julia system image to build on rather than the default 144 | Julia system image. 145 | """, 146 | ) 147 | parser.add_argument("output", help="Path to new system image file sys.o.") 148 | ns = parser.parse_args(args) 149 | try: 150 | build_sysimage(**vars(ns)) 151 | except (KnownError, subprocess.CalledProcessError) as err: 152 | print(err, file=sys.stderr) 153 | sys.exit(1) 154 | 155 | 156 | if __name__ == "__main__": 157 | main() 158 | -------------------------------------------------------------------------------- /src/julia/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JuliaPy/pyjulia/f30de4e235ce1705b2473f12b9b3b307a8a78ab8/src/julia/tests/__init__.py -------------------------------------------------------------------------------- /src/julia/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture(scope="session") 5 | def Main(julia): 6 | """pytest fixture for providing a Julia `Main` name space.""" 7 | from julia import Main 8 | 9 | return Main 10 | -------------------------------------------------------------------------------- /src/julia/tests/test_compatible_exe.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import shutil 5 | import subprocess 6 | import sys 7 | import tempfile 8 | import textwrap 9 | from contextlib import contextmanager 10 | 11 | import pytest 12 | 13 | import julia 14 | from julia.core import which 15 | 16 | is_linux = sys.platform.startswith("linux") 17 | is_windows = os.name == "nt" 18 | is_apple = sys.platform == "darwin" 19 | 20 | 21 | def discover_other_pythons(): 22 | this_version = subprocess.check_output( 23 | [sys.executable, "--version"], universal_newlines=True, stderr=subprocess.STDOUT 24 | ) 25 | 26 | candidate_names = ["python", "python3"] + [ 27 | "python3.{}".format(i) for i in range(20) 28 | ] 29 | found = {} 30 | for python in filter(None, map(which, candidate_names)): 31 | try: 32 | version = subprocess.check_output( 33 | [python, "--version"], universal_newlines=True, stderr=subprocess.STDOUT 34 | ) 35 | except Exception: 36 | continue 37 | if "Python" in version and version != this_version: 38 | found[version] = python 39 | 40 | return sorted(found.values()) 41 | 42 | 43 | def _get_paths(path): 44 | return list(filter(None, path.split(":"))) 45 | 46 | 47 | def get_incompatible_pythons( 48 | env=os.environ.get("PYJULIA_TEST_INCOMPATIBLE_PYTHONS", "") 49 | ): 50 | # Environment variable PYJULIA_TEST_INCOMPATIBLE_PYTHONS is the 51 | # :-separated list of Python executables incompatible with the 52 | # current Python: 53 | if env == "no": 54 | return [] 55 | paths = _get_paths(env) 56 | if paths: 57 | return paths 58 | if is_windows: 59 | # In Windows, we need to detect different word size. Skipping 60 | # the tests for now... 61 | return [] 62 | if os.environ.get("CI", "false") == "true": 63 | return list(discover_other_pythons()) 64 | return [] 65 | 66 | 67 | incompatible_pythons = get_incompatible_pythons() 68 | 69 | 70 | try: 71 | from types import SimpleNamespace 72 | except ImportError: 73 | # Python 2: 74 | from argparse import Namespace as SimpleNamespace 75 | 76 | 77 | def _run_fallback(args, input=None, **kwargs): 78 | # A port of subprocess.run just enough to run the tests. 79 | process = subprocess.Popen(args, stdin=subprocess.PIPE, **kwargs) 80 | stdout, stderr = process.communicate(input) 81 | retcode = process.wait() 82 | return SimpleNamespace(args=args, stdout=stdout, stderr=stderr, returncode=retcode) 83 | 84 | 85 | try: 86 | from subprocess import run 87 | except ImportError: 88 | run = _run_fallback 89 | 90 | 91 | @contextmanager 92 | def tmpdir_if(should): 93 | if should: 94 | path = tempfile.mkdtemp(prefix="tmp-pyjulia-test") 95 | try: 96 | yield path 97 | finally: 98 | shutil.rmtree(path, ignore_errors=True) 99 | else: 100 | yield None 101 | 102 | 103 | def runcode(code, python=None, check=False, env=None, **kwargs): 104 | """Run `code` in `python`.""" 105 | env = (env or os.environ).copy() 106 | 107 | with tmpdir_if(python) as path: 108 | if path is not None: 109 | # Make PyJulia importable. 110 | shutil.copytree( 111 | os.path.dirname(os.path.realpath(julia.__file__)), 112 | os.path.join(path, "julia"), 113 | ) 114 | env["PYTHONPATH"] = path 115 | proc = run( 116 | [python or sys.executable], 117 | input=textwrap.dedent(code), 118 | stdout=subprocess.PIPE, 119 | stderr=subprocess.PIPE, 120 | universal_newlines=True, 121 | env=env, 122 | **kwargs 123 | ) 124 | print("# --- Code evaluated:") 125 | print(code) 126 | print_completed_proc(proc) 127 | if check: 128 | assert proc.returncode == 0 129 | return proc 130 | 131 | 132 | def print_completed_proc(proc): 133 | # Print output (pytest will hide it by default): 134 | print("Ran:", *proc.args) 135 | if proc.stdout: 136 | print("# --- STDOUT from", *proc.args) 137 | print(proc.stdout) 138 | if proc.stderr: 139 | print("# --- STDERR from", *proc.args) 140 | print(proc.stderr) 141 | print("# ---") 142 | 143 | 144 | def is_dynamically_linked(executable): 145 | """ 146 | Check if Python `executable` is (likely to be) dynamically linked. 147 | 148 | It returns three possible values: 149 | 150 | * `True`: Likely that it's dynamically linked. 151 | * `False`: Likely that it's statically linked. 152 | * `None`: Unsupported platform. 153 | 154 | It's only "likely" since the check is by simple occurrence of a 155 | some substrings like "libpython". For example, if there is 156 | another library existing on the path containing "libpython", this 157 | function may return false-positive. 158 | """ 159 | path = which(executable) 160 | assert os.path.exists(path) 161 | if is_linux and which("ldd"): 162 | proc = run(["ldd", path], stdout=subprocess.PIPE, universal_newlines=True) 163 | print_completed_proc(proc) 164 | return "libpython" in proc.stdout 165 | elif is_apple and which("otool"): 166 | proc = run( 167 | ["otool", "-L", path], stdout=subprocess.PIPE, universal_newlines=True 168 | ) 169 | print_completed_proc(proc) 170 | return ( 171 | "libpython" in proc.stdout 172 | or "/Python" in proc.stdout 173 | or "/.Python" in proc.stdout 174 | ) 175 | # TODO: support Windows 176 | return None 177 | 178 | 179 | @pytest.mark.parametrize("python", incompatible_pythons) 180 | def test_incompatible_python(python, julia): 181 | python = which(python) 182 | proc = runcode( 183 | """ 184 | import os 185 | from julia import Julia 186 | Julia(runtime=os.getenv("PYJULIA_TEST_RUNTIME"), debug=True) 187 | """, 188 | python, 189 | ) 190 | 191 | assert proc.returncode == 1 192 | assert "It seems your Julia and PyJulia setup are not supported." in proc.stderr 193 | dynamic = is_dynamically_linked(python) 194 | if dynamic is True: 195 | assert "`libpython` have to match" in proc.stderr 196 | elif dynamic is False: 197 | assert "is statically linked to libpython" in proc.stderr 198 | 199 | 200 | @pytest.mark.parametrize( 201 | "python", 202 | [ 203 | p 204 | for p in filter(None, map(which, incompatible_pythons)) 205 | if is_dynamically_linked(p) is False 206 | ], 207 | ) 208 | def test_statically_linked(python): 209 | """ 210 | Simulate the case PyCall is configured with statically linked Python. 211 | 212 | In this case, `find_libpython()` would return the path identical 213 | to the one in PyCall's deps.jl. `is_compatible_exe` should reject 214 | it. 215 | """ 216 | python = which(python) 217 | runcode( 218 | """ 219 | from __future__ import print_function 220 | from julia.core import enable_debug 221 | from julia.find_libpython import find_libpython 222 | from julia.juliainfo import is_compatible_exe 223 | 224 | enable_debug() 225 | assert not is_compatible_exe(find_libpython()) 226 | """, 227 | python, 228 | check=True, 229 | ) 230 | -------------------------------------------------------------------------------- /src/julia/tests/test_core.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import array 4 | import math 5 | import subprocess 6 | import sys 7 | from types import ModuleType 8 | 9 | import pytest 10 | 11 | from julia import JuliaError 12 | from julia.core import jl_name, py_name 13 | 14 | from .utils import retry_failing_if_windows 15 | 16 | python_version = sys.version_info 17 | 18 | 19 | def test_call(julia): 20 | julia._call("1 + 1") 21 | julia._call("sqrt(2.0)") 22 | 23 | 24 | def test_eval(julia): 25 | assert julia.eval("1 + 1") == 2 26 | assert julia.eval("sqrt(2.0)") == math.sqrt(2.0) 27 | assert julia.eval("PyObject(1)") == 1 28 | assert julia.eval("PyObject(1000)") == 1000 29 | assert julia.eval("PyObject((1, 2, 3))") == (1, 2, 3) 30 | 31 | 32 | def test_call_error(julia): 33 | msg = "Error with message" 34 | with pytest.raises(JuliaError) as excinfo: 35 | julia._call('error("{}")'.format(msg)) 36 | assert msg in str(excinfo.value) 37 | 38 | 39 | def test_call_julia_function_with_python_args(Main): 40 | assert list(Main.map(Main.uppercase, array.array("u", ["a", "b", "c"]))) == [ 41 | "A", 42 | "B", 43 | "C", 44 | ] 45 | assert list(Main.map(Main.floor, [1.1, 2.2, 3.3])) == [1.0, 2.0, 3.0] 46 | assert Main.cos(0) == 1.0 47 | 48 | 49 | def test_call_julia_with_python_callable(Main): 50 | def add(a, b): 51 | return a + b 52 | 53 | assert list(Main.map(lambda x: x * x, [1, 2, 3])) == [1, 4, 9] 54 | assert all( 55 | x == y 56 | for x, y in zip( 57 | [11, 11, 11], Main.map(lambda x: x + 1, array.array("I", [10, 10, 10])) 58 | ) 59 | ) 60 | assert Main.reduce(add, [1, 2, 3]) == 6 61 | 62 | 63 | def test_call_python_with_julia_args(julia): 64 | assert sum(julia.eval("(1, 2, 3)")) == 6 65 | assert list(map(julia.eval("x->x^2"), [1, 2, 3])) == [1, 4, 9] 66 | 67 | 68 | def test_import_julia_functions(julia): 69 | if python_version.major < 3 or ( 70 | python_version.major == 3 and python_version.minor < 3 71 | ): 72 | import julia.sum as julia_sum 73 | 74 | assert julia_sum([1, 2, 3]) == 6 75 | else: 76 | pass 77 | 78 | 79 | def test_import_julia_module_existing_function(julia): 80 | from julia import Base 81 | 82 | assert Base.mod(2, 2) == 0 83 | 84 | 85 | def test_from_import_existing_julia_function(julia): 86 | from julia.Base import divrem 87 | 88 | assert divrem(7, 3) == (2, 1) 89 | 90 | 91 | def test_import_julia_module_non_existing_name(julia): 92 | from julia import Base 93 | 94 | with pytest.raises(AttributeError): 95 | Base.spamspamspam 96 | 97 | 98 | def test_from_import_non_existing_julia_name(julia): 99 | try: 100 | from Base import spamspamspam 101 | except ImportError: 102 | pass 103 | else: 104 | assert not spamspamspam 105 | 106 | 107 | def test_julia_module_bang(julia): 108 | from julia.Base import Channel, put_b, take_b 109 | 110 | chan = Channel(1) 111 | sent = 123 112 | put_b(chan, sent) 113 | received = take_b(chan) 114 | assert sent == received 115 | 116 | 117 | def test_import_julia_submodule(julia): 118 | from julia.Base import Enums 119 | 120 | assert isinstance(Enums, ModuleType) 121 | assert Enums.__name__ == "julia.Base.Enums" 122 | assert julia.fullname(Enums) == "Base.Enums" 123 | 124 | 125 | def test_getattr_submodule(Main): 126 | assert Main._PyJuliaHelper.IOPiper.__name__ == "julia.Main._PyJuliaHelper.IOPiper" 127 | 128 | 129 | def test_star_import_julia_module(julia, tmp_path): 130 | # Create a Python module __pyjulia_star_import_test 131 | path = tmp_path / "__pyjulia_star_import_test.py" 132 | path.write_text("from julia.Base.Enums import *") 133 | sys.path.insert(0, str(tmp_path)) 134 | 135 | import __pyjulia_star_import_test 136 | 137 | __pyjulia_star_import_test.Enum 138 | 139 | 140 | def test_main_module(julia, Main): 141 | Main.x = x = 123456 142 | assert julia.eval("x") == x 143 | 144 | 145 | def test_module_all(julia): 146 | from julia import Base 147 | 148 | assert "resize_b" in Base.__all__ 149 | 150 | 151 | def test_module_dir(julia): 152 | from julia import Base 153 | 154 | assert "resize_b" in dir(Base) 155 | 156 | 157 | @pytest.mark.pyjulia__using_default_setup 158 | @pytest.mark.julia 159 | def test_import_without_setup(): 160 | check_import_without_setup() 161 | 162 | 163 | @retry_failing_if_windows 164 | def check_import_without_setup(): 165 | command = [sys.executable, "-c", "from julia import Base"] 166 | print("Executing:", *command) 167 | subprocess.check_call(command) 168 | 169 | 170 | # TODO: this causes a segfault 171 | """ 172 | def test_import_julia_modules(julia): 173 | import julia.PyCall as pycall 174 | assert pycall.pyeval('2 * 3') == 6 175 | """ 176 | 177 | 178 | @pytest.mark.parametrize("name", ["normal", "resize!"]) 179 | def test_jlpy_identity(name): 180 | assert jl_name(py_name(name)) == name 181 | 182 | 183 | @pytest.mark.parametrize("name", ["normal", "resize_b"]) 184 | def test_pyjl_identity(name): 185 | assert py_name(jl_name(name)) == name 186 | -------------------------------------------------------------------------------- /src/julia/tests/test_find_libpython.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | 4 | from julia.find_libpython import finding_libpython, linked_libpython 5 | 6 | try: 7 | unicode 8 | except NameError: 9 | unicode = str # for Python 3 10 | 11 | 12 | def test_finding_libpython_yield_type(): 13 | paths = list(finding_libpython()) 14 | assert set(map(type, paths)) <= {str, unicode} 15 | 16 | 17 | # In a statically linked Python executable, no paths may be found. So 18 | # let's just check returned type of finding_libpython. 19 | 20 | 21 | def determine_if_statically_linked(): 22 | """Determines if this python executable is statically linked""" 23 | if not sys.platform.startswith("linux"): 24 | # Assuming that Windows and OS X are generally always 25 | # dynamically linked. Note that this is not the case in 26 | # Python installed via conda: 27 | # https://github.com/JuliaPy/pyjulia/issues/150#issuecomment-432912833 28 | # However, since we do not use conda in our CI, this function 29 | # is OK to use in tests. 30 | return False 31 | lddoutput = subprocess.check_output(["ldd", sys.executable]) 32 | return not (b"libpython" in lddoutput) 33 | 34 | 35 | def test_linked_libpython(): 36 | # TODO: Special-case conda (check `sys.version`). See the above 37 | # comments in `determine_if_statically_linked. 38 | if not determine_if_statically_linked(): 39 | assert linked_libpython() is not None 40 | -------------------------------------------------------------------------------- /src/julia/tests/test_install.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | import pytest 5 | 6 | from julia import install 7 | 8 | from .utils import only_in_ci 9 | 10 | 11 | @only_in_ci 12 | def test_noop(juliainfo): 13 | install(julia=juliainfo.julia) 14 | 15 | 16 | @only_in_ci 17 | def test_rebuild_broken_pycall(juliainfo): 18 | if juliainfo.version_info < (0, 7): 19 | pytest.skip("Julia >= 0.7 required") 20 | 21 | subprocess.check_call( 22 | [ 23 | juliainfo.julia, 24 | "--startup-file=no", 25 | "-e", 26 | """using Pkg; Pkg.develop("PyCall")""", 27 | ] 28 | ) 29 | 30 | # Remove ~/.julia/dev/PyCall/deps/deps.jl 31 | depsjl = os.path.join( 32 | os.path.expanduser("~"), ".julia", "dev", "PyCall", "deps", "deps.jl" 33 | ) 34 | if os.path.exists(depsjl): 35 | print("Removing", depsjl) 36 | os.remove(depsjl) 37 | 38 | # julia.install() should fix it: 39 | install(julia=juliainfo.julia) 40 | 41 | assert os.path.exists(depsjl) 42 | 43 | 44 | @only_in_ci 45 | def test_add_pycall(juliainfo): 46 | if juliainfo.version_info < (0, 7): 47 | pytest.skip("Julia >= 0.7 required") 48 | 49 | # Try to remove PyCall 50 | subprocess.call( 51 | [juliainfo.julia, "--startup-file=no", "-e", """using Pkg; Pkg.rm("PyCall")"""] 52 | ) 53 | 54 | # julia.install() should add PyCall: 55 | install(julia=juliainfo.julia) 56 | -------------------------------------------------------------------------------- /src/julia/tests/test_juliainfo.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | import pytest 5 | 6 | from julia.core import JuliaInfo, which 7 | 8 | 9 | def dummy_juliainfo(**kwargs): 10 | defaults = dict( 11 | julia="julia", 12 | bindir="/dummy/bin", 13 | libjulia_path="/dummy/libjulia.so", 14 | sysimage="/dummy/sys.so", 15 | version_raw="1.1.1", 16 | version_major="1", 17 | version_minor="1", 18 | version_patch="1", 19 | ) 20 | return JuliaInfo(**dict(defaults, **kwargs)) 21 | 22 | 23 | def check_core_juliainfo(jlinfo): 24 | assert os.path.exists(jlinfo.bindir) 25 | assert os.path.exists(jlinfo.libjulia_path) 26 | assert os.path.exists(jlinfo.sysimage) 27 | 28 | 29 | def test_juliainfo_normal(): 30 | jlinfo = JuliaInfo.load(os.getenv("PYJULIA_TEST_RUNTIME", "julia")) 31 | check_core_juliainfo(jlinfo) 32 | assert os.path.exists(jlinfo.python) 33 | # Note: jlinfo.libpython is probably not a full path so we are not 34 | # testing it here. 35 | 36 | 37 | def test_is_compatible_exe_without_pycall(): 38 | jlinfo = dummy_juliainfo() 39 | jlinfo.libpython_path = None 40 | assert not jlinfo.is_compatible_python() 41 | 42 | 43 | def test_juliainfo_without_pycall(tmpdir): 44 | """ 45 | `juliainfo` should not fail even when PyCall.jl is not installed. 46 | """ 47 | 48 | runtime = os.getenv("PYJULIA_TEST_RUNTIME", "julia") 49 | 50 | depot = subprocess.check_output( 51 | [ 52 | runtime, 53 | "--startup-file=no", 54 | "-e", 55 | """ 56 | paths = [ARGS[1], DEPOT_PATH[2:end]...] 57 | print(join(paths, Sys.iswindows() ? ';' : ':')) 58 | """, 59 | str(tmpdir), 60 | ], 61 | universal_newlines=True, 62 | ).strip() 63 | 64 | jlinfo = JuliaInfo.load(runtime, env=dict(os.environ, JULIA_DEPOT_PATH=depot)) 65 | 66 | check_core_juliainfo(jlinfo) 67 | assert jlinfo.python is None 68 | assert jlinfo.libpython_path is None 69 | assert not jlinfo.is_pycall_built() 70 | assert not jlinfo.is_compatible_python() 71 | 72 | 73 | @pytest.mark.skipif(not which("false"), reason="false command not found") 74 | def test_juliainfo_failure(): 75 | with pytest.raises(subprocess.CalledProcessError) as excinfo: 76 | JuliaInfo.load(julia="false") 77 | assert excinfo.value.cmd[0] == "false" 78 | assert excinfo.value.returncode == 1 79 | assert isinstance(excinfo.value.output, str) 80 | -------------------------------------------------------------------------------- /src/julia/tests/test_juliaoptions.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from julia.core import JuliaOptions 4 | 5 | 6 | # fmt: off 7 | @pytest.mark.parametrize("kwargs, args", [ 8 | ({}, []), 9 | (dict(compiled_modules=None), []), 10 | (dict(compiled_modules=False), ["--compiled-modules=no"]), 11 | (dict(compiled_modules="no"), ["--compiled-modules=no"]), 12 | (dict(depwarn="error"), ["--depwarn=error"]), 13 | (dict(sysimage="PATH"), ["--sysimage=PATH"]), 14 | (dict(bindir="PATH"), ["--home=PATH"]), 15 | (dict(optimize=3), ["--optimize=3"]), 16 | (dict(threads=4), ["--threads=4"]), 17 | (dict(min_optlevel=2), ["--min-optlevel=2"]), 18 | (dict(threads="auto", optimize=3), ["--optimize=3", '--threads=auto']), 19 | (dict(optimize=3, threads="auto"), ["--optimize=3", '--threads=auto']), # passed order doesn't matter 20 | (dict(compiled_modules=None, depwarn="yes"), ["--depwarn=yes"]), 21 | ]) 22 | # fmt: on 23 | def test_as_args(kwargs, args): 24 | assert JuliaOptions(**kwargs).as_args() == args 25 | 26 | 27 | @pytest.mark.parametrize( 28 | "kwargs", 29 | [ 30 | dict(compiled_modules="invalid value"), 31 | dict(bindir=123456789), 32 | ], 33 | ) 34 | def test_valueerror(kwargs): 35 | with pytest.raises(ValueError) as excinfo: 36 | JuliaOptions(**kwargs) 37 | assert "Option" in str(excinfo.value) 38 | assert "accept" in str(excinfo.value) 39 | 40 | 41 | # fmt: off 42 | @pytest.mark.parametrize("kwargs", [ 43 | dict(invalid_option=None), 44 | dict(invalid_option_1=None, invalid_option_2=None), 45 | ]) 46 | # fmt: on 47 | def test_unsupported(kwargs): 48 | with pytest.raises(TypeError) as excinfo: 49 | JuliaOptions(**kwargs) 50 | assert "Unsupported Julia option(s): " in str(excinfo.value) 51 | for key in kwargs: 52 | assert key in str(excinfo.value) 53 | -------------------------------------------------------------------------------- /src/julia/tests/test_libjulia.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from julia.core import JuliaInfo 4 | from julia.tests.utils import retry_failing_if_windows 5 | 6 | from .test_compatible_exe import runcode 7 | 8 | juliainfo = JuliaInfo.load() 9 | 10 | 11 | @pytest.mark.skipif("juliainfo.version_info < (0, 7)") 12 | @pytest.mark.julia 13 | def test_compiled_modules_no(): 14 | check_compiled_modules_no() 15 | 16 | 17 | @retry_failing_if_windows 18 | def check_compiled_modules_no(): 19 | runcode( 20 | """ 21 | from julia.core import Julia 22 | 23 | Julia(debug=True, compiled_modules=False) 24 | 25 | from julia import Main 26 | use_compiled_modules = Main.eval("Base.JLOptions().use_compiled_modules") 27 | 28 | print("use_compiled_modules =", use_compiled_modules) 29 | assert use_compiled_modules == 0 30 | """, 31 | check=True, 32 | ) 33 | 34 | 35 | @pytest.mark.skipif("not juliainfo.is_compatible_python()") 36 | @pytest.mark.julia 37 | def test_custom_sysimage(tmpdir): 38 | sysimage = str(tmpdir.join("sys.so")) 39 | runcode( 40 | """ 41 | from shutil import copyfile 42 | from julia.core import LibJulia, JuliaInfo, enable_debug 43 | 44 | enable_debug() 45 | info = JuliaInfo.load() 46 | 47 | sysimage = {!r} 48 | copyfile(info.sysimage, sysimage) 49 | 50 | api = LibJulia.load() 51 | api.init_julia(["--sysimage", sysimage]) 52 | 53 | from julia import Main 54 | actual = Main.eval("unsafe_string(Base.JLOptions().image_file)") 55 | 56 | print("actual =", actual) 57 | print("sysimage =", sysimage) 58 | assert actual == sysimage 59 | """.format( 60 | sysimage 61 | ), 62 | check=True, 63 | ) 64 | 65 | 66 | @pytest.mark.julia 67 | def test_non_existing_sysimage(tmpdir): 68 | proc = runcode( 69 | """ 70 | import sys 71 | from julia.core import enable_debug, LibJulia 72 | 73 | enable_debug() 74 | 75 | api = LibJulia.load() 76 | try: 77 | api.init_julia(["--sysimage", "sys.so"]) 78 | except RuntimeError as err: 79 | print(err) 80 | assert "System image" in str(err) 81 | assert "does not exist" in str(err) 82 | sys.exit(55) 83 | """, 84 | cwd=str(tmpdir), 85 | ) 86 | assert proc.returncode == 55 87 | -------------------------------------------------------------------------------- /src/julia/tests/test_magic.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from textwrap import dedent 3 | 4 | import pytest 5 | from IPython.testing import globalipapp 6 | 7 | from julia import magic 8 | 9 | 10 | @pytest.fixture 11 | def julia_magics(julia): 12 | return magic.JuliaMagics(shell=globalipapp.get_ipython()) 13 | 14 | 15 | # fmt: off 16 | 17 | 18 | @pytest.fixture 19 | def run_cell(julia_magics): 20 | # a more convenient way to run strings (possibly with magic) as if they were 21 | # an IPython cell 22 | def run_cell_impl(cell): 23 | cell = dedent(cell).strip() 24 | if cell.startswith("%%"): 25 | return julia_magics.shell.run_cell_magic("julia","",cell.replace("%%julia","").strip()) 26 | else: 27 | exec_result = julia_magics.shell.run_cell(cell) 28 | if exec_result.error_in_exec: 29 | raise exec_result.error_in_exec 30 | else: 31 | return exec_result.result 32 | return run_cell_impl 33 | 34 | 35 | def test_register_magics(julia): 36 | magic.load_ipython_extension(globalipapp.get_ipython()) 37 | 38 | 39 | def test_success_line(julia_magics): 40 | ans = julia_magics.julia('1') 41 | assert ans == 1 42 | 43 | 44 | def test_success_cell(julia_magics): 45 | ans = julia_magics.julia(None, '2') 46 | assert ans == 2 47 | 48 | 49 | def test_failure_line(julia_magics): 50 | with pytest.raises(Exception): 51 | julia_magics.julia('pop!([])') 52 | 53 | 54 | def test_failure_cell(julia_magics): 55 | with pytest.raises(Exception): 56 | julia_magics.julia(None, '1 += 1') 57 | 58 | 59 | # In IPython, $x does a string interpolation handled by IPython itself for 60 | # *line* magic, which prior to IPython 7.3 could not be turned off. However, 61 | # even prior to IPython 7.3, *cell* magic never did the string interpolation, so 62 | # below, any time we need to test $x interpolation, do it as cell magic so it 63 | # works on IPython < 7.3 64 | 65 | def test_interp_var(run_cell): 66 | run_cell("x=1") 67 | assert run_cell(""" 68 | %%julia 69 | $x 70 | """) == 1 71 | 72 | def test_interp_expr(run_cell): 73 | assert run_cell(""" 74 | x=1 75 | %julia py"x+1" 76 | """) == 2 77 | 78 | def test_bad_interp(run_cell): 79 | with pytest.raises(Exception): 80 | assert run_cell(""" 81 | %%julia 82 | $(x+1) 83 | """) 84 | 85 | def test_string_interp(run_cell): 86 | run_cell("foo='python'") 87 | assert run_cell(""" 88 | %%julia 89 | foo="julia" 90 | "$foo", "$($foo)" 91 | """) == ('julia','python') 92 | 93 | def test_expr_interp(run_cell): 94 | run_cell("foo='python'") 95 | assert run_cell(""" 96 | %%julia 97 | foo="julia" 98 | :($foo), :($($foo)) 99 | """) == ('julia','python') 100 | 101 | def test_expr_py_interp(run_cell): 102 | assert "baz" in str(run_cell(""" 103 | %julia :(py"baz") 104 | """)) 105 | 106 | def test_macro_esc(run_cell): 107 | assert run_cell(""" 108 | %%julia 109 | x = 1 110 | @eval y = $x 111 | y 112 | """) == 1 113 | 114 | def test_type_conversion(run_cell): 115 | assert run_cell(""" 116 | %julia py"1" isa Integer && py"1"o isa PyObject 117 | """) == True 118 | 119 | def test_local_scope(run_cell): 120 | assert run_cell(""" 121 | x = "global" 122 | def f(): 123 | x = "local" 124 | ret = %julia py"x" 125 | return ret 126 | f() 127 | """) == "local" 128 | 129 | def test_global_scope(run_cell): 130 | assert run_cell(""" 131 | x = "global" 132 | def f(): 133 | ret = %julia py"x" 134 | return ret 135 | f() 136 | """) == "global" 137 | 138 | def test_noretvalue(run_cell): 139 | assert run_cell(""" 140 | %%julia 141 | 1+2; 142 | """) is None 143 | 144 | 145 | # fmt: on 146 | def test_revise_error(): 147 | from julia.ipy import revise 148 | 149 | counter = [0] 150 | 151 | def throw(): 152 | counter[0] += 1 153 | raise RuntimeError("fake revise error") 154 | 155 | revise_wrapper = revise.make_revise_wrapper(throw) 156 | 157 | revise.revise_errors = 0 158 | try: 159 | assert revise.revise_errors_limit == 1 160 | 161 | with pytest.warns(UserWarning) as record1: 162 | revise_wrapper() # called 163 | assert len(record1) == 2 164 | assert "fake revise error" in record1[0].message.args[0] 165 | assert "Turning off Revise.jl" in record1[1].message.args[0] 166 | 167 | revise_wrapper() # not called 168 | 169 | assert counter[0] == 1 170 | assert revise.revise_errors == 1 171 | finally: 172 | revise.revise_errors = 0 173 | 174 | 175 | @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 2 not supported") 176 | def test_completions(julia): 177 | from IPython.core.completer import provisionalcompleter 178 | from julia.ipy.monkeypatch_completer import JuliaCompleter 179 | 180 | jc = JuliaCompleter(julia) 181 | t = "%julia Base.si" 182 | with provisionalcompleter(): 183 | completions = jc.julia_completions(t, len(t)) 184 | assert {"sin", "sign", "sizehint!"} <= {c.text for c in completions} 185 | -------------------------------------------------------------------------------- /src/julia/tests/test_options.py: -------------------------------------------------------------------------------- 1 | from julia.options import JuliaOptions, options_docs, parse_jl_options 2 | 3 | 4 | def parse_options_docs(docs): 5 | optdefs = {} 6 | for line in docs.splitlines(): 7 | if line.startswith(" ") or not line: 8 | continue 9 | 10 | name, domain = line.split(":", 1) 11 | assert name not in optdefs 12 | optdefs[name] = {"domain": eval(domain, {})} 13 | return optdefs 14 | 15 | 16 | def test_options_docs(): 17 | """ 18 | Ensure that `JuliaOptions` and `JuliaOptions.__doc__` agree. 19 | """ 20 | 21 | optdefs = parse_options_docs(options_docs) 22 | for desc in JuliaOptions.supported_options(): 23 | odef = optdefs.pop(desc.name) 24 | assert odef["domain"] == desc._domain() 25 | assert not optdefs 26 | 27 | 28 | def test_parse_jl_options(): 29 | opts = parse_jl_options( 30 | ["--home", "/home", "--sysimage", "/sys/image", "--optimize", "3"] 31 | ) 32 | assert opts.home == "/home" 33 | assert opts.sysimage == "/sys/image" 34 | -------------------------------------------------------------------------------- /src/julia/tests/test_plugin.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from julia.core import which 6 | 7 | pytest_plugins = ["pytester"] 8 | 9 | is_windows = os.name == "nt" 10 | userhome = os.path.expanduser("~") 11 | 12 | 13 | def test__using_default_setup(testdir, request, monkeypatch): 14 | if request.config.getoption("runpytest") != "subprocess": 15 | raise ValueError("Need `-p pytester --runpytest=subprocess` options.") 16 | monkeypatch.delenv("PYJULIA_TEST_RUNTIME", raising=False) 17 | 18 | # create a temporary conftest.py file 19 | testdir.makeini( 20 | """ 21 | [pytest] 22 | addopts = 23 | -p julia.pytestplugin 24 | """ 25 | ) 26 | 27 | testdir.makepyfile( 28 | """ 29 | import pytest 30 | 31 | @pytest.mark.pyjulia__using_default_setup 32 | def test(): 33 | pass 34 | """ 35 | ) 36 | 37 | args = ("-p", "julia.pytestplugin", "--no-julia") 38 | r0 = testdir.runpytest(*args) 39 | r0.assert_outcomes(passed=1) 40 | 41 | r1 = testdir.runpytest("--julia-runtime", which("julia"), *args) 42 | r1.assert_outcomes(skipped=1) 43 | 44 | r2 = testdir.runpytest("--julia-inline=yes", *args) 45 | r2.assert_outcomes(skipped=1) 46 | 47 | 48 | @pytest.mark.skipif( 49 | is_windows, reason="cannot run on Windows; symlink is used inside test" 50 | ) 51 | def test_undo_no_julia(testdir, request, julia): 52 | if request.config.getoption("runpytest") != "subprocess": 53 | raise ValueError("Need `-p pytester --runpytest=subprocess` options.") 54 | 55 | # TODO: Support `JULIA_DEPOT_PATH`; or a better approach would be 56 | # to not depend on user's depot at all. 57 | testdepot = os.path.join(str(testdir.tmpdir), ".julia") 58 | userdepot = os.path.join(userhome, ".julia") 59 | os.symlink(userdepot, testdepot) 60 | 61 | # create a temporary conftest.py file 62 | testdir.makeini( 63 | """ 64 | [pytest] 65 | addopts = 66 | -p julia.pytestplugin --no-julia 67 | """ 68 | ) 69 | 70 | testdir.makepyfile( 71 | """ 72 | import pytest 73 | 74 | @pytest.mark.julia 75 | def test(): 76 | pass 77 | """ 78 | ) 79 | 80 | r0 = testdir.runpytest() 81 | r0.assert_outcomes(skipped=1) 82 | 83 | r1 = testdir.runpytest("--julia") 84 | r1.assert_outcomes(passed=1) 85 | -------------------------------------------------------------------------------- /src/julia/tests/test_pseudo_python_cli.py: -------------------------------------------------------------------------------- 1 | import shlex 2 | 3 | import pytest 4 | 5 | from julia.pseudo_python_cli import parse_args 6 | 7 | 8 | def make_dict(**kwargs): 9 | ns = parse_args([]) 10 | return dict(vars(ns), **kwargs) 11 | 12 | 13 | # fmt: off 14 | @pytest.mark.parametrize("args, desired", [ 15 | ("-m json.tool -h", make_dict(module="json.tool", args=["-h"])), 16 | ("-mjson.tool -h", make_dict(module="json.tool", args=["-h"])), 17 | ("-imjson.tool -h", 18 | make_dict(interactive=True, module="json.tool", args=["-h"])), 19 | ("-m ipykernel install --user --name NAME --display-name DISPLAY_NAME", 20 | make_dict(module="ipykernel", 21 | args=shlex.split("install --user --name NAME" 22 | " --display-name DISPLAY_NAME"))), 23 | ("-m ipykernel_launcher -f FILE", 24 | make_dict(module="ipykernel_launcher", 25 | args=shlex.split("-f FILE"))), 26 | ("-", make_dict(script="-")), 27 | ("- a", make_dict(script="-", args=["a"])), 28 | ("script", make_dict(script="script")), 29 | ("script a", make_dict(script="script", args=["a"])), 30 | ("script -m", make_dict(script="script", args=["-m"])), 31 | ("script -c 1", make_dict(script="script", args=["-c", "1"])), 32 | ("script -h 1", make_dict(script="script", args=["-h", "1"])), 33 | ]) 34 | # fmt: on 35 | def test_valid_args(args, desired): 36 | ns = parse_args(shlex.split(args)) 37 | actual = vars(ns) 38 | assert actual == desired 39 | 40 | 41 | # fmt: off 42 | @pytest.mark.parametrize("args", [ 43 | "-m", 44 | "-c", 45 | "-i -m", 46 | "-h -m", 47 | "-V -m", 48 | ]) 49 | # fmt: on 50 | def test_invalid_args(args, capsys): 51 | with pytest.raises(SystemExit) as exc_info: 52 | parse_args(shlex.split(args)) 53 | assert exc_info.value.code != 0 54 | 55 | captured = capsys.readouterr() 56 | assert "usage:" in captured.err 57 | assert not captured.out 58 | 59 | 60 | # fmt: off 61 | @pytest.mark.parametrize("args", [ 62 | "-h", 63 | "-i --help", 64 | "-h -i", 65 | "-hi", 66 | "-ih", 67 | "-Vh", 68 | "-hV", 69 | "-h -m json.tool", 70 | "-h -mjson.tool", 71 | ]) 72 | # fmt: on 73 | def test_help_option(args, capsys): 74 | with pytest.raises(SystemExit) as exc_info: 75 | parse_args(shlex.split(args)) 76 | assert exc_info.value.code == 0 77 | 78 | captured = capsys.readouterr() 79 | assert "usage:" in captured.out 80 | assert not captured.err 81 | 82 | 83 | # fmt: off 84 | @pytest.mark.parametrize("args", [ 85 | "-V", 86 | "--version", 87 | "-V -i", 88 | "-Vi", 89 | "-iV", 90 | "-V script", 91 | "-V script -h", 92 | ]) 93 | # fmt: on 94 | def test_version_option(args, capsys): 95 | with pytest.raises(SystemExit) as exc_info: 96 | parse_args(shlex.split(args)) 97 | assert exc_info.value.code == 0 98 | 99 | captured = capsys.readouterr() 100 | assert "Python " in captured.out 101 | assert not captured.err 102 | -------------------------------------------------------------------------------- /src/julia/tests/test_python_jl.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shlex 3 | import subprocess 4 | import sys 5 | from textwrap import dedent 6 | 7 | import pytest 8 | 9 | from julia.core import which 10 | from julia.python_jl import parse_pyjl_args 11 | from julia.utils import is_apple 12 | 13 | from .utils import skip_in_github_actions_windows 14 | 15 | PYJULIA_TEST_REBUILD = os.environ.get("PYJULIA_TEST_REBUILD", "no") == "yes" 16 | 17 | python_jl_required = pytest.mark.skipif( 18 | os.environ.get("PYJULIA_TEST_PYTHON_JL_IS_INSTALLED", "no") != "yes" 19 | and not which("python-jl") 20 | # Skip for Python 2. (This is just a quick fix. Python 2 21 | # support should be removed soon.) 22 | or sys.version_info[0] < 3, 23 | reason="python-jl command not found", 24 | ) 25 | 26 | # fmt: off 27 | 28 | 29 | @pytest.mark.parametrize("args", [ 30 | "-h", 31 | "-i --help", 32 | "--julia false -h", 33 | "--julia false -i --help", 34 | ]) 35 | def test_help_option(args, capsys): 36 | with pytest.raises(SystemExit) as exc_info: 37 | parse_pyjl_args(shlex.split(args)) 38 | assert exc_info.value.code == 0 39 | 40 | captured = capsys.readouterr() 41 | assert "usage:" in captured.out 42 | 43 | 44 | quick_pass_cli_args = [ 45 | "-h", 46 | "-i --help", 47 | "-V", 48 | "--version -c 1/0", 49 | ] 50 | 51 | 52 | @python_jl_required 53 | @pytest.mark.parametrize("args", quick_pass_cli_args) 54 | def test_cli_quick_pass(args): 55 | subprocess.check_output( 56 | ["python-jl"] + shlex.split(args), 57 | ) 58 | 59 | 60 | @python_jl_required 61 | @pytest.mark.skipif( 62 | not which("false"), 63 | reason="false command not found") 64 | @pytest.mark.parametrize("args", quick_pass_cli_args) 65 | def test_cli_quick_pass_no_julia(args): 66 | subprocess.check_output( 67 | ["python-jl", "--julia", "false"] + shlex.split(args), 68 | ) 69 | 70 | 71 | @python_jl_required 72 | @skip_in_github_actions_windows 73 | @pytest.mark.skipif( 74 | # This test makes sense only when PyJulia is importable by 75 | # `PyCall.python`. Thus, it is safe to run this test only when 76 | # `PYJULIA_TEST_REBUILD=yes`; i.e., PyCall is using this Python 77 | # executable. 78 | not PYJULIA_TEST_REBUILD, 79 | reason="PYJULIA_TEST_REBUILD=yes is not set") 80 | def test_cli_import(juliainfo): 81 | code = """ 82 | from julia import Base 83 | Base.banner() 84 | from julia import Main 85 | Main.x = 1 86 | assert Main.x == 1 87 | """ 88 | args = ["--julia", juliainfo.julia, "-c", dedent(code)] 89 | output = subprocess.check_output( 90 | ["python-jl"] + args, 91 | universal_newlines=True) 92 | assert "julialang.org" in output 93 | 94 | # Embedded julia does not have usual the Main.eval and Main.include. 95 | # Main.eval is Core.eval. Let's test that we are not relying on this 96 | # special behavior. 97 | # 98 | # See also: https://github.com/JuliaLang/julia/issues/28825 99 | -------------------------------------------------------------------------------- /src/julia/tests/test_runtests.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | from textwrap import dedent 4 | 5 | from .test_compatible_exe import run 6 | 7 | 8 | def test_runtests_failure(tmp_path): 9 | testfile = tmp_path / "test.py" 10 | testcode = """ 11 | def test_THIS_TEST_MUST_FAIL(): 12 | assert False 13 | """ 14 | testfile.write_text(dedent(testcode)) 15 | 16 | proc = run( 17 | [ 18 | sys.executable, 19 | "-m", 20 | "julia.runtests", 21 | "--", 22 | str(testfile), 23 | "--no-julia", 24 | "-k", 25 | "test_THIS_TEST_MUST_FAIL", 26 | ], 27 | stdout=subprocess.PIPE, 28 | stderr=subprocess.STDOUT, 29 | universal_newlines=True, 30 | ) 31 | assert proc.returncode == 1 32 | assert "1 failed" in proc.stdout 33 | -------------------------------------------------------------------------------- /src/julia/tests/test_sysimage.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | from subprocess import check_call 4 | 5 | import pytest 6 | 7 | from julia.sysimage import build_sysimage 8 | 9 | from ..tools import build_pycall 10 | from .test_compatible_exe import runcode 11 | from .utils import only_in_ci, skip_in_apple, skip_in_windows 12 | 13 | 14 | def skip_early_julia_versions(juliainfo): 15 | if juliainfo.version_info < (1, 3, 1): 16 | pytest.skip("Julia < 1.3.1 is not supported") 17 | 18 | 19 | def skip_julia_nightly(juliainfo): 20 | if juliainfo.version_info >= (1, 8): 21 | pytest.skip("custom sysimage with Julia >= 1.8 (nightly) is not supported") 22 | 23 | 24 | def assert_sample_julia_code_runs(juliainfo, sysimage_path): 25 | very_random_string = "4903dc03-950f-4a54-98a3-c57a354b62df" 26 | proc = runcode( 27 | """ 28 | from julia.api import Julia 29 | 30 | sysimage_path = {sysimage_path!r} 31 | very_random_string = {very_random_string!r} 32 | jl = Julia( 33 | debug=True, 34 | sysimage=sysimage_path, 35 | runtime={juliainfo.julia!r}, 36 | ) 37 | 38 | from julia import Main 39 | Main.println(very_random_string) 40 | """.format( 41 | juliainfo=juliainfo, 42 | sysimage_path=sysimage_path, 43 | very_random_string=very_random_string, 44 | ) 45 | ) 46 | assert very_random_string in proc.stdout 47 | 48 | 49 | @pytest.mark.julia 50 | @only_in_ci 51 | @skip_in_windows 52 | @skip_in_apple 53 | @pytest.mark.parametrize("with_pycall_cache", [False, True]) 54 | def test_build_and_load(tmpdir, juliainfo, with_pycall_cache): 55 | skip_early_julia_versions(juliainfo) 56 | skip_julia_nightly(juliainfo) 57 | 58 | if with_pycall_cache: 59 | build_pycall(julia=juliainfo.julia) 60 | check_call([juliainfo.julia, "--startup-file=no", "-e", "using PyCall"]) 61 | else: 62 | # TODO: don't remove user's compile cache 63 | cachepath = os.path.join( 64 | os.path.expanduser("~"), 65 | ".julia", 66 | "compiled", 67 | "v{}.{}".format(juliainfo.version_major, juliainfo.version_minor), 68 | "PyCall", 69 | ) 70 | shutil.rmtree(cachepath) 71 | 72 | sysimage_path = str(tmpdir.join("sys.so")) 73 | build_sysimage(sysimage_path, julia=juliainfo.julia) 74 | 75 | assert_sample_julia_code_runs(juliainfo, sysimage_path) 76 | 77 | 78 | @pytest.mark.julia 79 | @only_in_ci 80 | @skip_in_windows # Avoid "LVM ERROR: out of memory" 81 | @skip_in_apple 82 | def test_build_with_basesysimage_and_load(tmpdir, juliainfo): 83 | skip_early_julia_versions(juliainfo) 84 | skip_julia_nightly(juliainfo) 85 | 86 | sysimage_path = str(tmpdir.join("sys.so")) 87 | base_sysimage_path = juliainfo.sysimage 88 | build_sysimage( 89 | sysimage_path, julia=juliainfo.julia, base_sysimage=base_sysimage_path 90 | ) 91 | 92 | assert_sample_julia_code_runs(juliainfo, sysimage_path) 93 | -------------------------------------------------------------------------------- /src/julia/tests/test_tools.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import platform 4 | import sysconfig 5 | 6 | import pytest 7 | 8 | from julia.tools import _non_default_julia_warning_message, julia_py_executable 9 | 10 | fake_user = "fa9a5150-8e17-11ea-3f8d-ff1e5ae4a251" 11 | posix_user_sample_path = os.path.join(os.sep, "home", fake_user, ".local", "bin") 12 | standard_sample_path = os.path.join(os.sep, "usr", "bin") 13 | 14 | 15 | def get_path_mock_scheme(scheme): 16 | # We'll ignore the other alternate install scheme types and just handle "posix_user" and None (standard install). 17 | # We use sample scripts paths as reported by sysconfig.get_path("scripts","posix_user") and 18 | # sysconfig.get_path("scripts") on a Linux system. 19 | assert scheme in ("posix_user", None) 20 | if scheme is None: 21 | # standard 22 | return standard_sample_path 23 | else: 24 | # posix_user 25 | return posix_user_sample_path 26 | 27 | 28 | def julia_py_with_command_extension(): 29 | extension = ".cmd" if platform.system() == "Windows" else "" 30 | return "julia-py" + extension 31 | 32 | 33 | def glob_mock(path=None): 34 | # we're only handling the case when the glob is ".../julia-py*" or nothing 35 | # if path is None then return empty list - this is indicator that we don't want to "find" any files matching a glob 36 | if path is None: 37 | return [] 38 | else: 39 | return [os.path.join(os.path.dirname(path), julia_py_with_command_extension())] 40 | 41 | 42 | def test_find_julia_py_executable_by_scheme(monkeypatch): 43 | # could extend this to test different kinds of scheme 44 | # right now we just fake the "posix_user" scheme and standard scheme, giving two paths to look in 45 | 46 | monkeypatch.setattr(sysconfig, "get_scheme_names", lambda: ("posix_user",)) 47 | monkeypatch.setattr( 48 | sysconfig, "get_path", lambda x, scheme=None: get_path_mock_scheme(scheme) 49 | ) 50 | monkeypatch.setattr(glob, "glob", glob_mock) 51 | 52 | jp = julia_py_executable() 53 | 54 | assert jp == os.path.join(posix_user_sample_path, julia_py_with_command_extension()) 55 | 56 | 57 | def test_find_julia_py_executable_standard(monkeypatch): 58 | # as though we only have standard install available, or didn't find julia-py in any alternate install location 59 | 60 | monkeypatch.setattr(sysconfig, "get_scheme_names", lambda: ()) 61 | monkeypatch.setattr( 62 | sysconfig, "get_path", lambda x, scheme=None: get_path_mock_scheme(scheme) 63 | ) 64 | monkeypatch.setattr(glob, "glob", glob_mock) 65 | 66 | jp = julia_py_executable() 67 | 68 | assert jp == os.path.join(standard_sample_path, julia_py_with_command_extension()) 69 | 70 | 71 | def test_find_julia_py_executable_not_found(monkeypatch): 72 | # look in posix_user and standard locations but don't find anything 73 | 74 | monkeypatch.setattr(sysconfig, "get_scheme_names", lambda: ("posix_user",)) 75 | monkeypatch.setattr( 76 | sysconfig, "get_path", lambda x, scheme=None: get_path_mock_scheme(scheme) 77 | ) 78 | monkeypatch.setattr(glob, "glob", lambda x: glob_mock()) 79 | 80 | with pytest.raises(RuntimeError) as excinfo: 81 | julia_py_executable() 82 | 83 | assert "``julia-py`` executable is not found" in str(excinfo.value) 84 | 85 | 86 | def test_non_default_julia_warning_message(): 87 | msg = _non_default_julia_warning_message("julia1.5") 88 | assert "Julia(runtime='julia1.5')" in msg 89 | -------------------------------------------------------------------------------- /src/julia/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests which can be done without loading `libjulia`. 3 | """ 4 | 5 | import os 6 | 7 | import pytest 8 | 9 | from julia.core import UnsupportedPythonError 10 | 11 | from .test_compatible_exe import runcode 12 | from .utils import _retry_on_failure, retry_failing_if_windows 13 | 14 | try: 15 | from types import SimpleNamespace 16 | except ImportError: 17 | from argparse import Namespace as SimpleNamespace # Python 2 18 | 19 | 20 | def dummy_juliainfo(): 21 | somepath = os.devnull # some random path 22 | return SimpleNamespace(julia="julia", python=somepath, libpython_path=somepath) 23 | 24 | 25 | def test_unsupported_python_error_statically_linked(): 26 | jlinfo = dummy_juliainfo() 27 | err = UnsupportedPythonError(jlinfo) 28 | err.statically_linked = True 29 | assert "is statically linked" in str(err) 30 | 31 | 32 | def test_unsupported_python_error_dynamically_linked(): 33 | jlinfo = dummy_juliainfo() 34 | err = UnsupportedPythonError(jlinfo) 35 | err.statically_linked = False 36 | assert "have to match exactly" in str(err) 37 | 38 | 39 | def test_retry_on_failure(): 40 | c = [0] 41 | 42 | def f(): 43 | c[0] += 1 44 | assert c[0] >= 2 45 | 46 | _retry_on_failure(f) 47 | assert c[0] == 2 48 | 49 | 50 | @pytest.mark.pyjulia__using_default_setup 51 | @pytest.mark.julia 52 | def test_atexit(): 53 | check_atexit() 54 | 55 | 56 | @retry_failing_if_windows 57 | def check_atexit(): 58 | proc = runcode( 59 | ''' 60 | import os 61 | from julia import Julia 62 | jl = Julia(runtime=os.getenv("PYJULIA_TEST_RUNTIME"), debug=True) 63 | 64 | jl_atexit = jl.eval(""" 65 | function(f) 66 | atexit(() -> f()) 67 | end 68 | """) 69 | 70 | @jl_atexit 71 | def _(): 72 | print("atexit called") 73 | ''' 74 | ) 75 | assert "atexit called" in proc.stdout 76 | -------------------------------------------------------------------------------- /src/julia/tests/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | import traceback 6 | from functools import wraps 7 | 8 | import pytest 9 | 10 | is_windows = os.name == "nt" 11 | is_apple = sys.platform == "darwin" 12 | in_github_actions = os.environ.get("GITHUB_ACTIONS", "false").lower() == "true" 13 | 14 | only_in_ci = pytest.mark.skipif( 15 | os.environ.get("CI", "false").lower() != "true", reason="CI=true not set" 16 | ) 17 | """ 18 | Tests that are too destructive or slow to run with casual `tox` call. 19 | """ 20 | 21 | skip_in_windows = pytest.mark.skipif(is_windows, reason="Running in Windows") 22 | """ 23 | Tests that are known to fail in Windows. 24 | """ 25 | 26 | skip_in_apple = pytest.mark.skipif(is_apple, reason="Running in macOS") 27 | """ 28 | Tests that are known to fail in macOS. 29 | """ 30 | 31 | skip_in_github_actions_windows = pytest.mark.skipif( 32 | is_windows and in_github_actions, reason="Running in Windows in GitHub Actions" 33 | ) 34 | """ 35 | Tests that are known to fail in Windows in GitHub Actions. 36 | """ 37 | 38 | 39 | def _retry_on_failure(*fargs, **kwargs): 40 | f = fargs[0] 41 | args = fargs[1:] 42 | for i in range(10): 43 | try: 44 | return f(*args, **kwargs) 45 | except Exception: 46 | print() 47 | print("{}-th try of {} failed".format(i, f)) 48 | traceback.print_exc() 49 | return f(*args, **kwargs) 50 | 51 | 52 | def retry_failing_if_windows(test): 53 | """ 54 | Retry upon test failure if in Windows. 55 | 56 | This is an ugly workaround for occasional STATUS_ACCESS_VIOLATION failures 57 | in Windows: https://github.com/JuliaPy/pyjulia/issues/462 58 | """ 59 | if not is_windows: 60 | return test 61 | 62 | @wraps(test) 63 | def repeater(*args, **kwargs): 64 | _retry_on_failure(test, *args, **kwargs) 65 | 66 | return repeater 67 | -------------------------------------------------------------------------------- /src/julia/tools.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import glob 4 | import os 5 | import re 6 | import subprocess 7 | import sys 8 | import sysconfig 9 | 10 | from .core import JuliaNotFound, which 11 | from .find_libpython import linked_libpython 12 | 13 | 14 | class PyCallInstallError(RuntimeError): 15 | def __init__(self, op, output=None): 16 | self.op = op 17 | self.output = output 18 | 19 | def __str__(self): 20 | if self.output: 21 | return "{} PyCall failed with output:\n\n{}".format(self.op, self.output) 22 | else: 23 | return """\ 24 | {} PyCall failed. 25 | 26 | ** Important information from Julia may be printed before Python's Traceback ** 27 | 28 | Some useful information may also be stored in the build log file 29 | `~/.julia/packages/PyCall/*/deps/build.log`. 30 | """.format( 31 | self.op 32 | ) 33 | 34 | 35 | def _julia_version(julia): 36 | output = subprocess.check_output([julia, "--version"], universal_newlines=True) 37 | match = re.search(r"([0-9]+)\.([0-9]+)\.([0-9]+)", output) 38 | if match: 39 | return tuple(int(match.group(i + 1)) for i in range(3)) 40 | else: 41 | return (0, 0, 0) 42 | 43 | 44 | def _non_default_julia_warning_message(julia): 45 | # Avoid confusion like 46 | # https://github.com/JuliaPy/pyjulia/issues/416 47 | return ( 48 | "PyCall is setup for non-default Julia runtime (executable) `{julia}`.\n" 49 | "To use this Julia runtime, PyJulia has to be initialized first by\n" 50 | " from julia import Julia\n" 51 | " Julia(runtime={julia!r})" 52 | ).format(julia=julia) 53 | 54 | 55 | def build_pycall(julia="julia", python=sys.executable, **kwargs): 56 | # Passing `python` to force build (OP="build") 57 | install(julia=julia, python=python, **kwargs) 58 | 59 | 60 | def install(julia="julia", color="auto", python=None, quiet=False): 61 | """ 62 | install(*, julia="julia", color="auto") 63 | Install Julia packages required by PyJulia in `julia`. 64 | 65 | This function installs and/or re-builds PyCall if necessary. It 66 | also makes sure to build PyCall in a way compatible with this 67 | Python executable (if possible). 68 | 69 | Keyword Arguments 70 | ----------------- 71 | julia : str 72 | Julia executable (default: "julia") 73 | color : "auto", False or True 74 | Use colorful output if `True`. "auto" (default) to detect it 75 | automatically. 76 | """ 77 | if which(julia) is None: 78 | raise JuliaNotFound(julia, kwargname="julia") 79 | 80 | libpython = linked_libpython() or "" 81 | 82 | julia_cmd = [julia, "--startup-file=no"] 83 | if quiet: 84 | color = False 85 | if color == "auto": 86 | color = sys.stdout.isatty() 87 | if color: 88 | # `--color=auto` doesn't work? 89 | julia_cmd.append("--color=yes") 90 | """ 91 | if _julia_version(julia) >= (1, 1): 92 | julia_cmd.append("--color=auto") 93 | else: 94 | julia_cmd.append("--color=yes") 95 | """ 96 | 97 | OP = "build" if python else "install" 98 | install_cmd = julia_cmd + [ 99 | "--", 100 | os.path.join(os.path.dirname(os.path.realpath(__file__)), "install.jl"), 101 | OP, 102 | python or sys.executable, 103 | libpython, 104 | ] 105 | 106 | kwargs = {} 107 | if quiet: 108 | kwargs.update( 109 | stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True 110 | ) 111 | proc = subprocess.Popen(install_cmd, **kwargs) 112 | output, _ = proc.communicate() 113 | returncode = proc.returncode 114 | 115 | if returncode == 113: # code_no_precompile_needed 116 | return 117 | elif returncode != 0: 118 | raise PyCallInstallError("Installing", output) 119 | 120 | if not quiet: 121 | print(file=sys.stderr) 122 | print("Precompiling PyCall...", file=sys.stderr) 123 | sys.stderr.flush() 124 | precompile_cmd = julia_cmd + ["-e", "using PyCall"] 125 | returncode = subprocess.call(precompile_cmd) 126 | if returncode != 0: 127 | raise PyCallInstallError("Precompiling") 128 | if not quiet: 129 | print("Precompiling PyCall... DONE", file=sys.stderr) 130 | print("PyCall is installed and built successfully.", file=sys.stderr) 131 | if julia != "julia": 132 | print(file=sys.stderr) 133 | print(_non_default_julia_warning_message(julia), file=sys.stderr) 134 | sys.stderr.flush() 135 | 136 | 137 | def make_receiver(io): 138 | def receiver(s): 139 | io.write(s) 140 | io.flush() 141 | 142 | return receiver 143 | 144 | 145 | def redirect_output_streams(): 146 | """ 147 | Redirect Julia's stdout and stderr to Python's counter parts. 148 | """ 149 | 150 | from .Main._PyJuliaHelper.IOPiper import pipe_std_outputs 151 | 152 | pipe_std_outputs(make_receiver(sys.stdout), make_receiver(sys.stderr)) 153 | 154 | # TODO: Invoking `redirect_output_streams()` in terminal IPython 155 | # terminates the whole Python process. Find out why. 156 | 157 | 158 | def julia_py_executable(): 159 | """ 160 | Path to ``julia-py`` executable installed for this Python executable. 161 | """ 162 | 163 | # try to find installed julia-py script - check scripts folders under different installation schemes 164 | # we check the alternate schemes first, at most one of which should give us a julia-py script 165 | # if the environment variable `PYTHONPATH` is set, we additionally check whether the script is there 166 | # if no candidate in an alternate scheme, try the standard install location 167 | # see https://docs.python.org/3/install/index.html#alternate-installation 168 | scripts_paths = [ 169 | *[ 170 | sysconfig.get_path("scripts", scheme) 171 | for scheme in sysconfig.get_scheme_names() 172 | ], 173 | *[ 174 | os.path.join(pypath, "bin") 175 | for pypath in os.environ.get("PYTHONPATH", "").split(os.pathsep) 176 | if pypath 177 | ], 178 | ] 179 | scripts_paths.append(sysconfig.get_path("scripts")) 180 | 181 | for scripts_path in scripts_paths: 182 | stempath = os.path.join(scripts_path, "julia-py") 183 | candidates = {os.path.basename(p): p for p in glob.glob(stempath + "*")} 184 | if candidates: 185 | break 186 | 187 | if not candidates: 188 | raise RuntimeError( 189 | "``julia-py`` executable is not found for Python installed at {}".format( 190 | scripts_paths 191 | ) 192 | ) 193 | 194 | for basename in ["julia-py", "julia-py.exe", "julia-py.cmd"]: 195 | try: 196 | return candidates[basename] 197 | except KeyError: 198 | continue 199 | 200 | raise RuntimeError( 201 | """\ 202 | ``julia-py`` with following unrecognized extension(s) are found. 203 | Please report it at https://github.com/JuliaPy/pyjulia/issues 204 | with the full traceback. 205 | Files found: 206 | """ 207 | + " \n".join(sorted(candidates)) 208 | ) 209 | -------------------------------------------------------------------------------- /src/julia/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import os 4 | import subprocess 5 | import sys 6 | 7 | is_linux = sys.platform.startswith("linux") 8 | is_windows = os.name == "nt" 9 | is_apple = sys.platform == "darwin" 10 | 11 | 12 | def _execprog_os(cmd): 13 | os.execvp(cmd[0], cmd) 14 | 15 | 16 | def _execprog_subprocess(cmd): 17 | sys.exit(subprocess.call(cmd)) 18 | 19 | 20 | if is_windows: 21 | # https://bugs.python.org/issue19124 22 | execprog = _execprog_subprocess 23 | else: 24 | execprog = _execprog_os 25 | 26 | 27 | PYCALL_PKGID = """\ 28 | Base.PkgId(Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0"), "PyCall")""" 29 | -------------------------------------------------------------------------------- /src/julia/with_rebuilt.py: -------------------------------------------------------------------------------- 1 | """ 2 | (Maybe) Re-build PyCall.jl to test ``exe_differs=False`` path. 3 | 4 | ``Pkg.build("PyCall")`` is run on Julia side when the environment 5 | variable `PYJULIA_TEST_REBUILD` is set to ``yes``. 6 | """ 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import os 11 | import signal 12 | import subprocess 13 | import sys 14 | from contextlib import contextmanager 15 | 16 | from .core import JuliaInfo 17 | from .tools import build_pycall 18 | 19 | # fmt: off 20 | 21 | 22 | @contextmanager 23 | def maybe_rebuild(rebuild, julia): 24 | if rebuild: 25 | info = JuliaInfo.load(julia) 26 | 27 | print('Building PyCall.jl with PYTHON =', sys.executable) 28 | sys.stdout.flush() 29 | build_pycall(julia=julia, python=sys.executable) 30 | try: 31 | yield 32 | finally: 33 | if info.python: 34 | # Use str to avoid "TypeError: environment can only 35 | # contain strings" in Python 2.7 + Windows: 36 | python = str(info.python) 37 | print() # clear out messages from py.test 38 | print('Restoring previous PyCall.jl build with PYTHON =', python) 39 | build_pycall(julia=julia, python=python, quiet=True) 40 | else: 41 | yield 42 | 43 | 44 | @contextmanager 45 | def ignoring(sig): 46 | """ 47 | Context manager for ignoring signal `sig`. 48 | 49 | For example,:: 50 | 51 | with ignoring(signal.SIGINT): 52 | do_something() 53 | 54 | would ignore user's ctrl-c during ``do_something()``. This is 55 | useful when launching interactive program (in which ctrl-c is a 56 | valid keybinding) from Python. 57 | """ 58 | s = signal.signal(sig, signal.SIG_IGN) 59 | try: 60 | yield 61 | finally: 62 | signal.signal(sig, s) 63 | 64 | 65 | def with_rebuilt(rebuild, julia, command): 66 | with maybe_rebuild(rebuild, julia), ignoring(signal.SIGINT): 67 | print('Execute:', *command) 68 | return subprocess.call(command) 69 | 70 | 71 | def main(args=None): 72 | import argparse 73 | parser = argparse.ArgumentParser( 74 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 75 | description=__doc__) 76 | parser.add_argument( 77 | '--rebuild', default=os.getenv('PYJULIA_TEST_REBUILD', 'no'), 78 | choices=('yes', 'no'), 79 | help=""" 80 | *Be careful using this option!* When it is set to `yes`, your 81 | `PyCall.jl` installation will be rebuilt using the Python 82 | interpreter used for testing. The test suite tries to build 83 | back to the original configuration but the precompilation 84 | would be in the stale state after the test. Note also that it 85 | does not work if you unconditionally set `PYTHON` environment 86 | variable in your Julia startup file. 87 | """) 88 | parser.add_argument( 89 | '--julia', default=os.getenv('PYJULIA_TEST_RUNTIME', 'julia'), 90 | help=""" 91 | Julia executable to be used. 92 | Default to the value of environment variable PYJULIA_TEST_RUNTIME if set. 93 | """) 94 | parser.add_argument( 95 | 'command', nargs='+', 96 | help='Command and arguments to run.') 97 | ns = parser.parse_args(args) 98 | ns.rebuild = ns.rebuild == 'yes' 99 | sys.exit(with_rebuilt(**vars(ns))) 100 | 101 | 102 | if __name__ == '__main__': 103 | main() 104 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py3 3 | 4 | [testenv] 5 | deps = 6 | pytest-cov 7 | coverage < 6 8 | extras = 9 | test 10 | commands = 11 | python -m julia.find_libpython --list-all --verbose 12 | # Print libpython candidates found by `find_libpython`. It may be 13 | # useful for debugging. 14 | 15 | python -m julia.runtests -- \ 16 | --log-file {envlogdir}/pytest.log \ 17 | {posargs} 18 | 19 | commands_post = 20 | # Strip off ".tox/..." from the coverage 21 | # (see also [[coverage:paths]]): 22 | -coverage combine .coverage 23 | -coverage xml 24 | -coverage report 25 | 26 | setenv = 27 | PYJULIA_TEST_PYTHON_JL_IS_INSTALLED = yes 28 | 29 | passenv = 30 | # Allow a workaround for "error initializing LibGit2 module": 31 | # https://github.com/JuliaLang/julia/issues/18693 32 | # https://github.com/JuliaDiffEq/diffeqpy/pull/13/commits/850441ee63962a2417de2bce6f6223052ee9cceb 33 | SSL_CERT_FILE 34 | 35 | # See: julia/with_rebuilt.py 36 | PYJULIA_TEST_REBUILD 37 | PYJULIA_TEST_RUNTIME 38 | 39 | JULIA_DEBUG 40 | 41 | # See: test/test_compatible_exe.py 42 | PYJULIA_TEST_INCOMPATIBLE_PYTHONS 43 | 44 | # See: https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci 45 | TRAVIS 46 | TRAVIS_* 47 | 48 | # https://www.appveyor.com/docs/environment-variables/ 49 | APPVEYOR 50 | 51 | # https://docs.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables 52 | GITHUB_ACTIONS 53 | 54 | CI 55 | 56 | [pytest] 57 | log_file_level = DEBUG 58 | # Essential flags and configuration must be added to 59 | # src/julia/runtests.py 60 | 61 | markers = 62 | pyjulia__using_default_setup: mark tests to be skipped with non-default setup 63 | # https://docs.pytest.org/en/latest/mark.html#registering-marks 64 | 65 | [coverage:paths] 66 | source = 67 | src/julia 68 | .tox/*/lib/python*/site-packages/julia 69 | # https://coverage.readthedocs.io/en/coverage-4.5.3/config.html#paths 70 | 71 | [testenv:doc] 72 | deps = 73 | -r{toxinidir}/docs/requirements.txt 74 | commands = 75 | sphinx-build -b "html" -d build/doctrees {posargs} source "build/html" 76 | commands_post = 77 | changedir = {toxinidir}/docs 78 | 79 | [testenv:style] 80 | deps = 81 | isort == 4.3.17 82 | black == 22.3.0 83 | commands = 84 | isort --recursive --check-only . 85 | black . {posargs:--check --diff} 86 | commands_post = 87 | --------------------------------------------------------------------------------