├── .binder ├── jupyter_config.json ├── postBuild └── requirements.txt ├── .coveragerc ├── .eslintignore ├── .eslintrc.js ├── .github ├── dependabot.yml └── workflows │ ├── release.yml │ ├── test-docs.yml │ ├── test-ssh.yml │ ├── test.yml │ └── windows-ssh-image.yaml ├── .gitignore ├── .mailmap ├── .pre-commit-config.yaml ├── .prettierignore ├── .yarnrc.yml ├── CONTRIBUTING.md ├── COPYING.md ├── README.md ├── asv.conf.json ├── benchmarks ├── .gitignore ├── asv.conf.json ├── asv_normal.sh ├── asv_quick.sh ├── asv_runner.py ├── async_benchmark.ipynb ├── benchmark_result.py ├── benchmark_results.ipynb ├── benchmarks │ ├── __init__.py │ ├── constants.py │ ├── testing.py │ ├── throughput.py │ └── utils.py ├── cluster_start.py ├── depth_benchmark.ipynb ├── engines_test.py ├── explore │ ├── control_flow.py │ └── scheduler.py ├── gcloud_setup.py ├── instance_setup.py ├── logger.py ├── machine_configs │ ├── asv-testing-16.json │ ├── asv-testing-32.json │ ├── asv-testing-64.json │ └── asv-testing-96.json ├── profiling │ ├── __init__.py │ ├── profiling_code.py │ ├── profiling_code_runner.py │ └── view_profiling_results.py ├── profiling_initial_results.ipynb ├── profiling_latest_results.ipynb ├── push_benchmarks.ipynb ├── results │ ├── asv-testing-64-2020-04-28 │ │ ├── CoalescingBroadcast.json │ │ └── CoalescingPush.json │ ├── asv-testing-64-2020-05-12-19-52-58 │ │ └── results.json │ ├── asv-testing-64-2020-05-19-00-04-55 │ │ └── results.json │ ├── asv_testing-16-2020-04-29-20-02-25 │ │ └── results.json │ ├── asv_testing-16-2020-05-04-17-43 │ │ └── results.json │ └── benchmarks.json ├── scheduler_benchmarks.ipynb ├── throughtput_benchmarks.ipynb └── utils.py ├── ci ├── slurm │ ├── Dockerfile │ ├── docker-compose.yaml │ ├── entrypoint.sh │ └── slurm.conf └── ssh │ ├── Dockerfile │ ├── docker-compose.yaml │ ├── ipcluster_config.py │ ├── win_Dockerfile_template │ ├── win_base_Dockerfile │ └── win_docker-compose.yaml ├── docs ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── _static │ ├── IPyParallel-MPI-Example.png │ └── basic.mp4 │ ├── api │ └── ipyparallel.rst │ ├── changelog.md │ ├── conf.py │ ├── examples │ ├── Cluster API.ipynb │ ├── Data Publication API.ipynb │ ├── Futures.ipynb │ ├── Monitoring an MPI Simulation - 1.ipynb │ ├── Monitoring an MPI Simulation - 2.ipynb │ ├── Monte Carlo Options.ipynb │ ├── Parallel Decorator and map.ipynb │ ├── Parallel Magics.ipynb │ ├── Using Dill.ipynb │ ├── Using MPI with IPython Parallel.ipynb │ ├── broadcast │ │ ├── Broadcast view.ipynb │ │ ├── Dockerfile │ │ ├── MPI Broadcast.ipynb │ │ ├── docker-compose.yaml │ │ └── memmap Broadcast.ipynb │ ├── customresults.py │ ├── daVinci Word Count │ │ ├── pwordfreq.py │ │ └── wordfreq.py │ ├── dagdeps.py │ ├── dask.ipynb │ ├── dependencies.py │ ├── fetchparse.py │ ├── index.md │ ├── interengine │ │ ├── bintree.py │ │ ├── bintree_script.py │ │ ├── communicator.py │ │ └── interengine.py │ ├── iopubwatcher.py │ ├── itermapresult.py │ ├── joblib.ipynb │ ├── nwmerge.py │ ├── phistogram.py │ ├── pi │ │ ├── parallelpi.py │ │ └── pidigits.py │ ├── progress.ipynb │ ├── rmt │ │ ├── rmt.ipy │ │ ├── rmt.ipynb │ │ └── rmtkernel.py │ ├── task_mod.py │ ├── task_profiler.py │ ├── throughput.py │ ├── visualizing-tasks.ipynb │ └── wave2D │ │ ├── RectPartitioner.py │ │ ├── communicator.py │ │ ├── parallelwave-mpi.py │ │ ├── parallelwave.py │ │ └── wavesolver.py │ ├── index.md │ ├── links.txt │ ├── redirects.txt │ ├── reference │ ├── connections.md │ ├── dag_dependencies.md │ ├── db.md │ ├── details.md │ ├── figs │ │ ├── allconnections.png │ │ ├── allconnections.svg │ │ ├── dagdeps.pdf │ │ ├── dagdeps.png │ │ ├── frontend-kernel.png │ │ ├── frontend-kernel.svg │ │ ├── hbfade.png │ │ ├── iopubfade.png │ │ ├── ipy_kernel_and_terminal.png │ │ ├── ipy_kernel_and_terminal.svg │ │ ├── nbconvert.png │ │ ├── nbconvert.svg │ │ ├── notebook_components.png │ │ ├── notebook_components.svg │ │ ├── notiffade.png │ │ ├── other_kernels.png │ │ ├── other_kernels.svg │ │ ├── queryfade.png │ │ ├── queuefade.png │ │ ├── simpledag.pdf │ │ └── simpledag.png │ ├── index.md │ ├── launchers.md │ ├── messages.md │ ├── mpi.md │ └── security.md │ └── tutorial │ ├── asyncresult.md │ ├── demos.md │ ├── direct.md │ ├── figs │ ├── asian_call.pdf │ ├── asian_call.png │ ├── asian_put.pdf │ ├── asian_put.png │ ├── mec_simple.pdf │ ├── mec_simple.png │ ├── parallel_pi.pdf │ ├── parallel_pi.png │ ├── single_digits.pdf │ ├── single_digits.png │ ├── two_digit_counts.pdf │ ├── two_digit_counts.png │ └── wideView.png │ ├── index.md │ ├── intro.md │ ├── process.md │ └── task.md ├── examples ├── hatch_build.py ├── install.json ├── ipyparallel ├── __init__.py ├── _async.py ├── _version.py ├── apps │ ├── __init__.py │ ├── baseapp.py │ ├── ipclusterapp.py │ ├── ipcontrollerapp.py │ ├── ipengineapp.py │ ├── iploggerapp.py │ ├── launcher.py │ └── logwatcher.py ├── client │ ├── __init__.py │ ├── _joblib.py │ ├── asyncresult.py │ ├── client.py │ ├── futures.py │ ├── magics.py │ ├── map.py │ ├── remotefunction.py │ └── view.py ├── cluster │ ├── __init__.py │ ├── __main__.py │ ├── _winhpcjob.py │ ├── app.py │ ├── cluster.py │ ├── launcher.py │ ├── shellcmd.py │ └── shellcmd_receive.py ├── controller │ ├── __init__.py │ ├── __main__.py │ ├── app.py │ ├── broadcast_scheduler.py │ ├── dependency.py │ ├── dictdb.py │ ├── heartmonitor.py │ ├── hub.py │ ├── mongodb.py │ ├── scheduler.py │ ├── sqlitedb.py │ └── task_scheduler.py ├── datapub.py ├── engine │ ├── __init__.py │ ├── __main__.py │ ├── app.py │ ├── datapub.py │ ├── kernel.py │ ├── log.py │ └── nanny.py ├── error.py ├── joblib.py ├── logger.py ├── nbextension │ ├── __init__.py │ ├── base.py │ ├── handlers.py │ ├── install.py │ └── static │ │ ├── clusterlist.css │ │ ├── clusterlist.js │ │ └── main.js ├── serialize │ ├── __init__.py │ ├── canning.py │ ├── codeutil.py │ └── serialize.py ├── shellcmd.py ├── tests │ ├── __init__.py │ ├── _test_startup_crash.py │ ├── clienttest.py │ ├── conftest.py │ ├── test_apps.py │ ├── test_async.py │ ├── test_asyncresult.py │ ├── test_canning.py │ ├── test_client.py │ ├── test_cluster.py │ ├── test_db.py │ ├── test_dependency.py │ ├── test_executor.py │ ├── test_joblib.py │ ├── test_launcher.py │ ├── test_lbview.py │ ├── test_magics.py │ ├── test_mongodb.py │ ├── test_mpi.py │ ├── test_remotefunction.py │ ├── test_serialize.py │ ├── test_shellcmd.py │ ├── test_slurm.py │ ├── test_ssh.py │ ├── test_util.py │ ├── test_view.py │ └── test_view_broadcast.py ├── traitlets.py └── util.py ├── jupyter-config ├── jupyter_notebook_config.d │ └── ipyparallel.json ├── jupyter_server_config.d │ └── ipyparallel.json └── nbconfig │ └── tree.d │ └── ipyparallel.json ├── lab ├── schema │ └── plugin.json ├── src │ ├── clusters.tsx │ ├── commands.ts │ ├── dialog.tsx │ ├── index.ts │ ├── sidebar.ts │ └── svg.d.ts ├── style │ ├── code-dark.svg │ ├── code-light.svg │ ├── index.css │ └── logo.svg └── webpack.config.js ├── package.json ├── pyproject.toml ├── readthedocs.yml ├── tsconfig.eslint.json ├── tsconfig.json └── yarn.lock /.binder/jupyter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "LabApp": { 3 | "collaborative": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /.binder/postBuild: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | jlpm --prefer-offline --ignore-optional --ignore-scripts 3 | 4 | jlpm build 5 | 6 | IPP_DISABLE_JS=1 python -m pip install -e .[test,benchmark] 7 | 8 | jupyter serverextension enable --sys-prefix --py ipyparallel 9 | jupyter serverextension list 10 | 11 | jupyter server extension enable --sys-prefix --py ipyparallel 12 | jupyter server extension list 13 | 14 | jupyter nbextension install --symlink --sys-prefix --py ipyparallel 15 | jupyter nbextension enable --sys-prefix --py ipyparallel 16 | jupyter nbextension list 17 | 18 | jlpm install:extension 19 | jupyter labextension list 20 | -------------------------------------------------------------------------------- /.binder/requirements.txt: -------------------------------------------------------------------------------- 1 | ipywidgets 2 | jupyterlab >=3.2.4 3 | jupyterlab-link-share 4 | matplotlib 5 | networkx 6 | pidigits 7 | requests 8 | retrolab >=0.3.13 9 | wordfreq 10 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | parallel = True 3 | omit = 4 | ipyparallel/tests/* 5 | 6 | [report] 7 | exclude_lines = 8 | if self.debug: 9 | pragma: no cover 10 | raise NotImplementedError 11 | if __name__ == .__main__.: 12 | ignore_errors = True 13 | omit = 14 | ipyparallel/tests/* 15 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | coverage 4 | **/*.d.ts 5 | tests 6 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | es6: true, 5 | commonjs: true, 6 | node: true, 7 | }, 8 | root: true, 9 | extends: [ 10 | "eslint:recommended", 11 | "plugin:@typescript-eslint/eslint-recommended", 12 | "plugin:@typescript-eslint/recommended", 13 | "prettier", 14 | "plugin:react/recommended", 15 | ], 16 | parser: "@typescript-eslint/parser", 17 | parserOptions: { 18 | project: "tsconfig.eslint.json", 19 | }, 20 | plugins: ["@typescript-eslint"], 21 | rules: { 22 | "@typescript-eslint/no-floating-promises": ["error", { ignoreVoid: true }], 23 | "@typescript-eslint/naming-convention": [ 24 | "error", 25 | { 26 | selector: "interface", 27 | format: ["PascalCase"], 28 | custom: { 29 | regex: "^I[A-Z]", 30 | match: true, 31 | }, 32 | }, 33 | ], 34 | "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }], 35 | "@typescript-eslint/no-use-before-define": "off", 36 | "@typescript-eslint/camelcase": "off", 37 | "@typescript-eslint/no-explicit-any": "off", 38 | "@typescript-eslint/no-non-null-assertion": "off", 39 | "@typescript-eslint/no-namespace": "off", 40 | "@typescript-eslint/interface-name-prefix": "off", 41 | "@typescript-eslint/explicit-function-return-type": "off", 42 | "@typescript-eslint/ban-ts-comment": ["warn", { "ts-ignore": true }], 43 | "@typescript-eslint/ban-types": "warn", 44 | "@typescript-eslint/no-non-null-asserted-optional-chain": "warn", 45 | "@typescript-eslint/no-var-requires": "off", 46 | "@typescript-eslint/no-empty-interface": "off", 47 | "@typescript-eslint/triple-slash-reference": "warn", 48 | "@typescript-eslint/no-inferrable-types": "off", 49 | "no-inner-declarations": "off", 50 | "no-prototype-builtins": "off", 51 | "no-control-regex": "warn", 52 | "no-undef": "warn", 53 | "no-case-declarations": "warn", 54 | "no-useless-escape": "off", 55 | "prefer-const": "off", 56 | "react/prop-types": "warn", 57 | }, 58 | settings: { 59 | react: { 60 | version: "detect", 61 | }, 62 | }, 63 | }; 64 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # dependabot.yaml reference: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file 2 | version: 2 3 | updates: 4 | # Maintain dependencies in our GitHub Workflows 5 | - package-ecosystem: github-actions 6 | directory: "/" 7 | schedule: 8 | interval: monthly 9 | 10 | # jupyterlab extension 11 | - package-ecosystem: npm 12 | directory: "/" 13 | schedule: 14 | interval: monthly 15 | groups: 16 | # one big pull request 17 | lab-minor: 18 | patterns: 19 | - "*" 20 | update-types: 21 | - minor 22 | - patch 23 | # group major bumps 24 | lab-major: 25 | patterns: 26 | - "*" 27 | update-types: 28 | - major 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # Build releases and (on tags) publish to PyPI 2 | name: Release 3 | 4 | # always build releases (to make sure wheel-building works) 5 | # but only publish to PyPI on tags 6 | on: 7 | push: 8 | branches-ignore: 9 | - "pre-commit-ci*" 10 | tags: 11 | - "*" 12 | pull_request: 13 | 14 | concurrency: 15 | group: >- 16 | ${{ github.workflow }}- 17 | ${{ github.ref_type }}- 18 | ${{ github.event.pull_request.number || github.sha }} 19 | cancel-in-progress: true 20 | 21 | jobs: 22 | build-release: 23 | runs-on: ubuntu-22.04 24 | steps: 25 | - uses: actions/checkout@v4 26 | - uses: actions/setup-python@v5 27 | with: 28 | python-version: 3.11 29 | cache: pip 30 | 31 | - name: install build package 32 | run: | 33 | pip install --upgrade pip 34 | pip install build 35 | pip freeze 36 | 37 | - name: build release 38 | run: | 39 | python -m build --sdist --wheel . 40 | ls -l dist 41 | 42 | - name: verify wheel 43 | run: | 44 | cd dist 45 | pip install ./*.whl jupyterlab==4.* 46 | ipcluster --help-all 47 | ipcontroller --help-all 48 | ipengine --help-all 49 | jupyter labextension list 2>&1 | grep ipyparallel 50 | jupyter server extension list 2>&1 | grep ipyparallel 51 | 52 | # ref: https://github.com/actions/upload-artifact#readme 53 | - uses: actions/upload-artifact@v4 54 | with: 55 | name: ipyparallel-${{ github.sha }} 56 | path: "dist/*" 57 | if-no-files-found: error 58 | 59 | upload-pypi: 60 | permissions: 61 | id-token: write 62 | environment: release 63 | runs-on: ubuntu-22.04 64 | if: startsWith(github.ref, 'refs/tags/') 65 | needs: 66 | - build-release 67 | steps: 68 | - uses: actions/download-artifact@v4 69 | with: 70 | path: dist 71 | merge-multiple: true 72 | - name: Publish wheels to PyPI 73 | uses: pypa/gh-action-pypi-publish@release/v1 74 | -------------------------------------------------------------------------------- /.github/workflows/test-docs.yml: -------------------------------------------------------------------------------- 1 | name: Test docs 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches-ignore: 7 | - "dependabot/**" 8 | - "pre-commit-ci-update-config" 9 | tags: 10 | - "**" 11 | workflow_dispatch: 12 | 13 | env: 14 | # UTF-8 content may be interpreted as ascii and causes errors without this. 15 | LANG: C.UTF-8 16 | PYTEST_ADDOPTS: "--verbose --color=yes" 17 | 18 | jobs: 19 | test-docs: 20 | runs-on: ubuntu-22.04 21 | steps: 22 | - uses: actions/checkout@v4 23 | with: 24 | # make rediraffecheckdiff requires git history to compare current 25 | # commit with the main branch and previous releases. 26 | fetch-depth: 0 27 | 28 | - uses: actions/setup-python@v5 29 | with: 30 | python-version: "3.10" 31 | 32 | - name: Install requirements 33 | run: | 34 | pip install -r docs/requirements.txt . 35 | 36 | # readthedocs doesn't halt on warnings, 37 | # so raise any warnings here 38 | - name: build docs 39 | run: | 40 | cd docs 41 | make html 42 | 43 | - name: check links 44 | run: | 45 | cd docs 46 | make linkcheck 47 | 48 | # make rediraffecheckdiff compares files for different changesets 49 | # these diff targets aren't always available 50 | # - compare with base ref (usually 'main', always on 'origin') for pull requests 51 | # - only compare with tags when running against main repo 52 | # to avoid errors on forks, which often lack tags 53 | - name: check redirects for this PR 54 | if: github.event_name == 'pull_request' 55 | run: | 56 | cd docs 57 | export REDIRAFFE_BRANCH=origin/${{ github.base_ref }} 58 | make rediraffecheckdiff 59 | 60 | # this should check currently published 'stable' links for redirects 61 | - name: check redirects since last release 62 | if: github.repository == 'ipython/ipyparallel' 63 | run: | 64 | cd docs 65 | export REDIRAFFE_BRANCH=$(git describe --tags --abbrev=0) 66 | make rediraffecheckdiff 67 | 68 | # longer-term redirect check (fixed version) for older links 69 | - name: check redirects since 8.6.0 70 | if: github.repository == 'ipython/ipyparallel' 71 | run: | 72 | cd docs 73 | export REDIRAFFE_BRANCH=8.6.0 74 | make rediraffecheckdiff 75 | -------------------------------------------------------------------------------- /.github/workflows/windows-ssh-image.yaml: -------------------------------------------------------------------------------- 1 | # because it's too unbelievably slow to install ssh server on Windows 2 | name: Build image for Windows SSH CI 3 | 4 | on: 5 | workflow_dispatch: 6 | 7 | env: 8 | REGISTRY: ghcr.io 9 | IMAGE_NAME: ipyparallel-windows-ssh 10 | TAG: 3.12-2022 11 | 12 | jobs: 13 | build-and-push-image: 14 | runs-on: windows-2022 15 | 16 | permissions: 17 | contents: read 18 | packages: write 19 | 20 | steps: 21 | - name: Checkout repository 22 | uses: actions/checkout@v4 23 | 24 | - name: Log in to the Container registry 25 | uses: docker/login-action@v3 26 | with: 27 | registry: ${{ env.REGISTRY }} 28 | username: ${{ github.actor }} 29 | password: ${{ secrets.GITHUB_TOKEN }} 30 | 31 | # - name: Extract metadata (tags, labels) for Docker 32 | # id: meta 33 | # uses: docker/metadata-action@v5 34 | # with: 35 | # labels: | 36 | # org.opencontainers.image.title=${{ env.IMAGE_NAME }} 37 | # 38 | # images: ${{ github.repository_owner }}/${{ env.IMAGE_NAME }} 39 | # tags: | 40 | # type=raw,value=${{ env.TAG }} 41 | 42 | - name: Build image 43 | # can't use build-push-action on Windows 44 | # https://github.com/docker/build-push-action/issues/18 45 | run: | 46 | docker build -t ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.TAG }} -f ci/ssh/win_base_Dockerfile ci/ssh 47 | - name: Push image 48 | run: | 49 | docker push ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.TAG }} 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | MANIFEST 2 | build 3 | dist 4 | _build 5 | docs/man/*.gz 6 | docs/source/api/generated 7 | docs/source/config/options 8 | docs/source/interactive/magics-generated.txt 9 | examples/daVinci Word Count/*.txt 10 | examples/testdill.py 11 | *.py[co] 12 | __pycache__ 13 | *.egg-info 14 | *~ 15 | *.bak 16 | .ipynb_checkpoints 17 | .tox 18 | .DS_Store 19 | \#*# 20 | .#* 21 | .coverage 22 | *coverage.xml 23 | .coverage.* 24 | .idea 25 | htmlcov 26 | id_*sa 27 | .vscode 28 | 29 | node_modules 30 | lib 31 | 32 | ipyparallel/labextension 33 | tsconfig.tsbuildinfo 34 | dask-worker-space 35 | .yarn 36 | package-lock.json 37 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autoupdate_schedule: monthly 3 | 4 | repos: 5 | - repo: https://github.com/astral-sh/ruff-pre-commit 6 | rev: v0.11.4 7 | hooks: 8 | - id: ruff 9 | args: 10 | - "--fix" 11 | exclude_types: 12 | - jupyter 13 | - id: ruff 14 | name: ruff (notebooks) 15 | args: 16 | - "--fix" 17 | types: 18 | - jupyter 19 | - id: ruff-format 20 | # run via nbqa to handle cell magics 21 | exclude_types: 22 | - jupyter 23 | # run ruff via nbqa, which has better and configurable handling of cell magics 24 | # than ruff itself 25 | # but has trouble with isort rules 26 | - repo: https://github.com/nbQA-dev/nbQA 27 | rev: 1.9.1 28 | hooks: 29 | - id: nbqa-ruff-format 30 | - id: nbqa-ruff-check 31 | args: 32 | - --fix 33 | # isort rules get confused in the temporary files 34 | - --ignore=I 35 | - repo: https://github.com/pre-commit/mirrors-prettier 36 | rev: v4.0.0-alpha.8 37 | hooks: 38 | - id: prettier 39 | - repo: https://github.com/pre-commit/pre-commit-hooks 40 | rev: v5.0.0 41 | hooks: 42 | - id: end-of-file-fixer 43 | - id: check-case-conflict 44 | - id: check-executables-have-shebangs 45 | - id: requirements-txt-fixer 46 | - repo: https://github.com/pre-commit/mirrors-eslint 47 | rev: v9.24.0 48 | hooks: 49 | - id: eslint 50 | files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx 51 | exclude: ipyparallel/nbextension/.* 52 | types: [file] 53 | additional_dependencies: 54 | - "@typescript-eslint/eslint-plugin@2.27.0" 55 | - "@typescript-eslint/parser@2.27.0" 56 | - eslint@^6.0.0 57 | - eslint-config-prettier@6.10.1 58 | - eslint-plugin-prettier@3.1.4 59 | - eslint-plugin-react@7.21.5 60 | - typescript@4.1.3 61 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | docs/build 3 | htmlcov 4 | ipyparallel/labextension 5 | **/node_modules 6 | **/lib 7 | **/package.json 8 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | enableImmutableInstalls: false 2 | nodeLinker: node-modules 3 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We follow the [Jupyter Contributing Guide](https://jupyter.readthedocs.io/en/latest/contributing/content-contributor.html). 4 | 5 | Make sure to follow the [Jupyter Code of Conduct](https://jupyter.org/conduct/). 6 | 7 | ## Development install 8 | 9 | A basic development install of IPython parallel 10 | is the same as almost all Python packages: 11 | 12 | ```bash 13 | pip install -e . 14 | ``` 15 | 16 | To enable the server extension from a development install: 17 | 18 | ```bash 19 | # for jupyterlab 20 | jupyter server extension enable --sys-prefix ipyparallel 21 | # for classic notebook 22 | jupyter serverextension enable --sys-prefix ipyparallel 23 | ``` 24 | 25 | As described in the [JupyterLab documentation](https://jupyterlab.readthedocs.io/en/stable/extension/extension_dev.html#developing-a-prebuilt-extension) 26 | for a development install of the jupyterlab extension you can run the following in this directory: 27 | 28 | ```bash 29 | jlpm # Install npm package dependencies 30 | jlpm build # Compile the TypeScript sources to Javascript 31 | jupyter labextension develop . --overwrite # Install the current directory as an extension 32 | ``` 33 | 34 | If you are working on the lab extension, 35 | you can run jupyterlab in dev mode to always rebuild and reload your extensions. 36 | In two terminals, run: 37 | 38 | ```bash 39 | [term 1] $ jlpm watch 40 | [term 2] $ jupyter lab --extensions-in-dev-mode 41 | ``` 42 | 43 | You should then be able to refresh the JupyterLab page 44 | and it will pick up the changes to the extension as you work. 45 | -------------------------------------------------------------------------------- /COPYING.md: -------------------------------------------------------------------------------- 1 | # Licensing terms 2 | 3 | Traitlets is adapted from enthought.traits, Copyright (c) Enthought, Inc., 4 | under the terms of the Modified BSD License. 5 | 6 | This project is licensed under the terms of the Modified BSD License 7 | (also known as New or Revised or 3-Clause BSD), as follows: 8 | 9 | - Copyright (c) 2001-, IPython Development Team 10 | 11 | All rights reserved. 12 | 13 | Redistribution and use in source and binary forms, with or without 14 | modification, are permitted provided that the following conditions are met: 15 | 16 | Redistributions of source code must retain the above copyright notice, this 17 | list of conditions and the following disclaimer. 18 | 19 | Redistributions in binary form must reproduce the above copyright notice, this 20 | list of conditions and the following disclaimer in the documentation and/or 21 | other materials provided with the distribution. 22 | 23 | Neither the name of the IPython Development Team nor the names of its 24 | contributors may be used to endorse or promote products derived from this 25 | software without specific prior written permission. 26 | 27 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 28 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 29 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 30 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 31 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 33 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 34 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 35 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | 38 | ## About the IPython Development Team 39 | 40 | The IPython Development Team is the set of all contributors to the IPython project. 41 | This includes all of the IPython subprojects. 42 | 43 | The core team that coordinates development on GitHub can be found here: 44 | https://github.com/jupyter/. 45 | 46 | ## Our Copyright Policy 47 | 48 | IPython uses a shared copyright model. Each contributor maintains copyright 49 | over their contributions to IPython. But, it is important to note that these 50 | contributions are typically only changes to the repositories. Thus, the IPython 51 | source code, in its entirety is not the copyright of any single person or 52 | institution. Instead, it is the collective copyright of the entire IPython 53 | Development Team. If individual contributors want to maintain a record of what 54 | changes/contributions they have specific copyright on, they should indicate 55 | their copyright in the commit message of the change, when they commit the 56 | change to one of the IPython repositories. 57 | 58 | With this in mind, the following banner should be used in any source code file 59 | to indicate the copyright and license terms: 60 | 61 | # Copyright (c) IPython Development Team. 62 | # Distributed under the terms of the Modified BSD License. 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Interactive Parallel Computing with IPython 2 | 3 | IPython Parallel (`ipyparallel`) is a Python package and collection of CLI scripts for controlling clusters of IPython processes, built on the Jupyter protocol. 4 | 5 | IPython Parallel provides the following commands: 6 | 7 | - ipcluster - start/stop/list clusters 8 | - ipcontroller - start a controller 9 | - ipengine - start an engine 10 | 11 | ## Install 12 | 13 | Install IPython Parallel: 14 | 15 | pip install ipyparallel 16 | 17 | This will install and enable the IPython Parallel extensions 18 | for Jupyter Notebook and (as of 7.0) Jupyter Lab 3.0. 19 | 20 | ## Run 21 | 22 | Start a cluster: 23 | 24 | ipcluster start 25 | 26 | Use it from Python: 27 | 28 | ```python 29 | import os 30 | import ipyparallel as ipp 31 | 32 | cluster = ipp.Cluster(n=4) 33 | with cluster as rc: 34 | ar = rc[:].apply_async(os.getpid) 35 | pid_map = ar.get_dict() 36 | ``` 37 | 38 | See [the docs](https://ipyparallel.readthedocs.io) for more info. 39 | -------------------------------------------------------------------------------- /asv.conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "project": "ipyparallel", 4 | "project_url": "https://github.com/ipython/ipyparallel/", 5 | "repo": "https://github.com/ipython/ipyparallel.git", 6 | "environment_type": "conda", 7 | "matrix": { 8 | "numpy": [] 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /benchmarks/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | ipyparallel/ 4 | env/ 5 | html/ 6 | __pycache__ 7 | 8 | .ipynb_checkpoints 9 | -------------------------------------------------------------------------------- /benchmarks/asv.conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "project": "ipyparallel", 4 | "project_url": "https://github.com/ipython/ipyparallel/", 5 | "repo": "https://github.com/ipython/ipyparallel.git", 6 | "environment_type": "conda", 7 | "matrix": { 8 | "numpy": [] 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /benchmarks/asv_normal.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ipcluster start -n 200 --daemon --profile=asv 3 | asv run 4 | ipcluster stop --profile=asv 5 | -------------------------------------------------------------------------------- /benchmarks/asv_quick.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "starting engines" 3 | #ipcluster start -n 200 --daemon --profile=asv 4 | echo "starting benchmarks" 5 | asv run --quick --show-stderr 6 | echo "Benchmarks finished" 7 | #ipcluster stop --profile=asv 8 | -------------------------------------------------------------------------------- /benchmarks/asv_runner.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | import os 3 | import resource 4 | import socket 5 | import sys 6 | from subprocess import check_call 7 | 8 | import googleapiclient.discovery as gcd 9 | from cluster_start import start_cluster 10 | from google.cloud import storage 11 | 12 | os.environ['OPENBLAS_NUM_THREADS'] = '1' 13 | 14 | DEFAULT_MINICONDA_PATH = os.path.abspath(os.path.join('..', "miniconda3/bin/:")) 15 | env = os.environ.copy() 16 | env["PATH"] = DEFAULT_MINICONDA_PATH + env["PATH"] 17 | 18 | ZONE = "europe-west1-b" 19 | PROJECT_NAME = "jupyter-simula" 20 | BUCKET_NAME = 'ipyparallel_dev' 21 | 22 | instance_name = socket.gethostname() 23 | compute = gcd.build("compute", "v1") 24 | 25 | 26 | def cmd_run(*args, log_filename=None, error_filename=None): 27 | if len(args) == 1: 28 | args = args[0].split(" ") 29 | print(f'$ {" ".join(args)}') 30 | if not log_filename and not error_filename: 31 | check_call(args, env=env) 32 | else: 33 | check_call( 34 | args, 35 | env=env, 36 | stdout=open(log_filename, 'w'), 37 | stderr=open(error_filename, 'w'), 38 | ) 39 | 40 | 41 | def delete_self(): 42 | print(f'deleting: {instance_name}') 43 | compute.instances().delete( 44 | project=PROJECT_NAME, zone=ZONE, instance=instance_name 45 | ).execute() 46 | 47 | 48 | def upload_file(filename): 49 | blob_name = sys.argv[1] 50 | storage_client = storage.Client() 51 | bucket = storage_client.bucket(BUCKET_NAME) 52 | blob = bucket.blob(f'{instance_name}/{blob_name}') 53 | print(f'Uploading {filename}') 54 | blob.upload_from_filename(filename) 55 | 56 | 57 | if __name__ == '__main__': 58 | # atexit.register(delete_self) 59 | benchmark_name = sys.argv[1] 60 | template_name = sys.argv[2] 61 | 62 | soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) 63 | resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit)) 64 | 65 | ps = [] 66 | for i in [3, 4]: 67 | ps += start_cluster(i, 1030, '../miniconda3/bin/', log_output_to_file=True) 68 | # time.sleep(10) 69 | # ps += start_cluster( 70 | # 0, 'depth_0', 300, '../miniconda3/bin/', 71 | # log_output_to_file=True) 72 | # time.sleep(10) 73 | # ps += start_cluster( 74 | # 0, 'depth_1', 150, '../miniconda3/bin/', 75 | # log_output_to_file=True) 76 | 77 | log_filename = f'{instance_name}.log' 78 | error_log_filename = f'{instance_name}.error.log' 79 | 80 | def clean_up(): 81 | for p in ps: 82 | p.kill() 83 | 84 | atexit.register(clean_up) 85 | # cmd_run("ipcluster start -n 200 --daemon --profile=asv") # Starting 200 engines 86 | cmd_run( 87 | 'asv run -b DepthTestingSuite --verbose --show-stderr', 88 | # log_filename=log_filename, 89 | # error_filename=error_log_filename, 90 | ) 91 | clean_up() 92 | # cmd_run("ipcluster stop --profile=asv") 93 | print('uploading files') 94 | # upload_file(log_filename) 95 | # upload_file(error_log_filename) 96 | results_dir = f'results/{template_name}' 97 | for file_name in os.listdir(results_dir): 98 | if 'conda' in file_name: 99 | upload_file(f'{results_dir}/{file_name}') 100 | print('script finished') 101 | -------------------------------------------------------------------------------- /benchmarks/benchmarks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/benchmarks/benchmarks/__init__.py -------------------------------------------------------------------------------- /benchmarks/benchmarks/constants.py: -------------------------------------------------------------------------------- 1 | DEFAULT_NUMBER_OF_ENGINES = 16 2 | -------------------------------------------------------------------------------- /benchmarks/benchmarks/testing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/benchmarks/benchmarks/testing.py -------------------------------------------------------------------------------- /benchmarks/benchmarks/utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | from typing import Callable 4 | 5 | 6 | def wait_for(condition: Callable): 7 | for _ in range(750): 8 | if condition(): 9 | break 10 | else: 11 | time.sleep(0.1) 12 | if not condition(): 13 | raise TimeoutError('wait_for took to long to finish') 14 | 15 | 16 | def echo(delay=0): 17 | def inner_echo(x, **kwargs): 18 | import time 19 | 20 | if delay: 21 | time.sleep(delay) 22 | return x 23 | 24 | return inner_echo 25 | 26 | 27 | def get_time_stamp() -> str: 28 | return ( 29 | str(datetime.datetime.now()).split(".")[0].replace(" ", "-").replace(":", "-") 30 | ) 31 | -------------------------------------------------------------------------------- /benchmarks/cluster_start.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | import sys 3 | import time 4 | from subprocess import Popen 5 | 6 | import ipyparallel as ipp 7 | from benchmarks.throughput import wait_for 8 | 9 | 10 | def start_cluster(depth, number_of_engines, path='', log_output_to_file=False): 11 | ipcontroller_cmd = ( 12 | f'{path}ipcontroller --profile=asv --nodb ' 13 | f'--cluster-id=depth_{depth} ' 14 | f'--HubFactory.broadcast_scheduler_depth={depth} ' 15 | f'--HubFactory.db_class=NoDB' 16 | ) 17 | print(ipcontroller_cmd) 18 | ipengine_cmd = f'{path}ipengine --profile=asv --cluster-id=depth_{depth} ' 19 | ps = [ 20 | Popen( 21 | ipcontroller_cmd.split(), 22 | stdout=( 23 | open('ipcontroller_output.log', 'a+') 24 | if log_output_to_file 25 | else sys.stdout 26 | ), 27 | stderr=( 28 | open('ipcontroller_error_output.log', 'a+') 29 | if log_output_to_file 30 | else sys.stdout 31 | ), 32 | stdin=sys.stdin, 33 | ) 34 | ] 35 | time.sleep(2) 36 | client = ipp.Client(profile='asv', cluster_id=f'depth_{depth}') 37 | print(ipengine_cmd) 38 | for i in range(number_of_engines): 39 | ps.append( 40 | Popen( 41 | ipengine_cmd.split(), 42 | stdout=( 43 | open('ipengine_output.log', 'a+') 44 | if log_output_to_file 45 | else sys.stdout 46 | ), 47 | stderr=( 48 | open('ipengine_error_output.log', 'a+') 49 | if log_output_to_file 50 | else sys.stdout 51 | ), 52 | stdin=sys.stdin, 53 | ) 54 | ) 55 | if i % 10 == 0: 56 | wait_for(lambda: len(client) >= i - 10) 57 | if i % 20 == 0: 58 | time.sleep(2) 59 | print(f'{len(client)} engines started') 60 | 61 | return ps 62 | 63 | 64 | if __name__ == '__main__': 65 | if len(sys.argv) > 3: 66 | depth = sys.argv[1] 67 | number_of_engines = int(sys.argv[3]) 68 | else: 69 | depth = 3 70 | number_of_engines = 30 71 | 72 | ps = start_cluster(depth, number_of_engines) 73 | 74 | for p in ps: 75 | p.wait() 76 | 77 | def clean_up(): 78 | for p in ps: 79 | p.kill() 80 | 81 | atexit.register(clean_up) 82 | -------------------------------------------------------------------------------- /benchmarks/engines_test.py: -------------------------------------------------------------------------------- 1 | import time 2 | from subprocess import check_call 3 | 4 | import ipyparallel as ipp 5 | 6 | n = 20 7 | 8 | 9 | check_call(f'ipcluster start -n {n} --daemon --profile=asv --debug', shell=True) 10 | c = ipp.Client(profile='asv') 11 | seen = -1 12 | 13 | running_engines = len(c) 14 | 15 | while running_engines < n: 16 | if seen != running_engines: 17 | print(running_engines) 18 | seen = running_engines 19 | running_engines = len(c) 20 | time.sleep(0.1) 21 | 22 | check_call('ipcluster stop --profile=asv', shell=True) 23 | -------------------------------------------------------------------------------- /benchmarks/explore/control_flow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import ipyparallel as ipp 4 | 5 | 6 | def main(): 7 | client = ipp.Client(profile='asv') 8 | # client.debug = True 9 | direct_view = client[:] 10 | direct_view.targets = list(range(5)) 11 | direct_view_result = direct_view.apply(lambda x: x * 2, 13) 12 | print(direct_view_result.get()) 13 | # load_balanced_view = client.load_balanced_view() 14 | # result = load_balanced_view.apply(lambda x: x * 2, 13) 15 | # print(result.get()) 16 | broadcast_view = client.broadcast_view(is_coalescing=True) 17 | broadcast_result = broadcast_view.apply_sync( 18 | lambda x: x, np.array([0] * 8, dtype=np.int8) 19 | ) 20 | 21 | print(broadcast_result) 22 | print(len(broadcast_result)) 23 | # broadcast_view2 = client.broadcast_view(is_coalescing=False) 24 | # broadcast_result = broadcast_view2.apply_sync( 25 | # lambda x: x, np.array([0] * 8, dtype=np.int8) 26 | # ) 27 | # 28 | # print(broadcast_result) 29 | # print(len(broadcast_result)) 30 | 31 | 32 | if __name__ == '__main__': 33 | main() 34 | -------------------------------------------------------------------------------- /benchmarks/explore/scheduler.py: -------------------------------------------------------------------------------- 1 | from ipyparallel.apps import ipclusterapp as app 2 | 3 | 4 | def main(): 5 | app.launch_new_instance(['start', '-n', '16', '--debug', '--profile=asv']) 6 | 7 | 8 | if __name__ == '__main__': 9 | main() 10 | -------------------------------------------------------------------------------- /benchmarks/instance_setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | DEFAULT_MINICONDA_PATH = os.path.join(os.getcwd(), "miniconda3/bin/:") 4 | env = os.environ.copy() 5 | env["PATH"] = DEFAULT_MINICONDA_PATH + env["PATH"] 6 | 7 | from subprocess import check_call 8 | 9 | GITHUB_TOKEN = "" # Token for machine user 10 | ASV_TESTS_REPO = "github.com/tomoboy/ipyparallel_master_project.git" 11 | IPYPARALLEL_REPO = "github.com/tomoboy/ipyparallel.git" 12 | 13 | 14 | def cmd_run(*args, log_filename=None, error_filename=None): 15 | if len(args) == 1: 16 | args = args[0].split(" ") 17 | print(f'$ {" ".join(args)}') 18 | if not log_filename and not error_filename: 19 | check_call(args, env=env) 20 | else: 21 | check_call( 22 | args, 23 | env=env, 24 | stdout=open(log_filename, 'w'), 25 | stderr=open(error_filename, 'w'), 26 | ) 27 | 28 | 29 | if __name__ == "__main__": 30 | cmd_run( 31 | f"git clone -q https://{GITHUB_TOKEN}@{ASV_TESTS_REPO}" 32 | ) # Get benchmarks from repo 33 | print("Finished cloning benchmark repo") 34 | # Installing ipyparallel from the dev branch 35 | cmd_run(f"pip install -q git+https://{GITHUB_TOKEN}@{IPYPARALLEL_REPO}") 36 | print("Installed ipyparallel") 37 | # Create profile for ipyparallel, (should maybe be copied if we want some cusom values here) 38 | cmd_run("ipython profile create --parallel --profile=asv") 39 | cmd_run('echo 120000 > /proc/sys/kernel/threads-max') 40 | cmd_run('echo 600000 > /proc/sys/vm/max_map_count') 41 | cmd_run('echo 200000 > /proc/sys/kernel/piad_max') 42 | cmd_run('echo "* hard nproc 100000" > /etc/security/limits.d') 43 | cmd_run('echo "* soft nproc 100000" > /etc/security/limits.d') 44 | -------------------------------------------------------------------------------- /benchmarks/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import date 3 | 4 | from ipyparallel_master_project.benchmarks.utils import get_time_stamp 5 | 6 | LOGS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logs') 7 | GCLOUD_DIR = os.path.join(LOGS_DIR, 'gcloud_output') 8 | PROFILING_DIR = os.path.join(LOGS_DIR, 'profiling') 9 | TODAY = str(date.today()) 10 | 11 | 12 | def get_dir(main_dir): 13 | target_dir = f"{main_dir}/{TODAY}" 14 | if not os.path.exists(target_dir): 15 | os.makedirs(target_dir) 16 | return target_dir 17 | 18 | 19 | def get_profiling_log_file_name(): 20 | return os.path.join(get_dir(PROFILING_DIR), f'profiling_{get_time_stamp()}') 21 | 22 | 23 | def get_gcloud_log_file_name(instance_name): 24 | return os.path.join(get_dir(GCLOUD_DIR), instance_name) 25 | 26 | 27 | if __name__ == "__main__": 28 | print(LOGS_DIR) 29 | -------------------------------------------------------------------------------- /benchmarks/machine_configs/asv-testing-16.json: -------------------------------------------------------------------------------- 1 | { 2 | "asv-testing-16": { 3 | "arch": "x86_64", 4 | "cpu": "Intel(R) Xeon(R) CPU @ 2.30GHz", 5 | "machine": "asv-testing-16", 6 | "os": "Linux 4.15.0-1029-gcp", 7 | "ram": "16423396" 8 | }, 9 | "version": 1 10 | } 11 | -------------------------------------------------------------------------------- /benchmarks/machine_configs/asv-testing-32.json: -------------------------------------------------------------------------------- 1 | { 2 | "asv-testing-16": { 3 | "arch": "x86_64", 4 | "cpu": "Intel(R) Xeon(R) CPU @ 2.30GHz", 5 | "machine": "asv-testing-32", 6 | "os": "Linux 4.15.0-1029-gcp", 7 | "ram": "32423396" 8 | }, 9 | "version": 1 10 | } 11 | -------------------------------------------------------------------------------- /benchmarks/machine_configs/asv-testing-64.json: -------------------------------------------------------------------------------- 1 | { 2 | "asv-testing-64": { 3 | "arch": "x86_64", 4 | "cpu": "Intel(R) Xeon(R) CPU @ 2.30GHz", 5 | "machine": "asv-testing-64", 6 | "os": "Linux 4.15.0-1029-gcp", 7 | "ram": "64423396" 8 | }, 9 | "version": 1 10 | } 11 | -------------------------------------------------------------------------------- /benchmarks/machine_configs/asv-testing-96.json: -------------------------------------------------------------------------------- 1 | { 2 | "asv-testing-16-16": { 3 | "arch": "x86_64", 4 | "cpu": "Intel(R) Xeon(R) CPU @ 2.30GHz", 5 | "machine": "asv-testing-96-96", 6 | "os": "Linux 4.15.0-1029-gcp", 7 | "ram": "96423396" 8 | }, 9 | "version": 1 10 | } 11 | -------------------------------------------------------------------------------- /benchmarks/profiling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/benchmarks/profiling/__init__.py -------------------------------------------------------------------------------- /benchmarks/profiling/profiling_code.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import numpy as np 4 | 5 | import ipyparallel as ipp 6 | 7 | 8 | def echo(delay=0): 9 | def inner_echo(x, **kwargs): 10 | import time 11 | 12 | if delay: 13 | time.sleep(delay) 14 | return x 15 | 16 | return inner_echo 17 | 18 | 19 | def profile_many_empty_tasks(lview, n, block=True): 20 | lview.map(echo(0), [None] * n, block=block) 21 | 22 | 23 | # def profile_echo_many_arguments(lview, number_of_arguments): 24 | # lview.map( 25 | # lambda x: echo_many_arguments(*x), 26 | # [ 27 | # tuple(np.empty(1, dtype=np.int8) for n in range(number_of_arguments)) 28 | # for x in range(16) 29 | # ], 30 | # block=False, 31 | # ) 32 | 33 | 34 | def profile_tasks_with_large_data(lview, num_bytes): 35 | for _ in range(10): 36 | for i in range(10): 37 | lview.apply_sync(echo(0), np.array([0] * num_bytes, dtype=np.int8)) 38 | 39 | 40 | def run_profiling(selected_profiling_task, selected_view): 41 | client = ipp.Client(profile='asv') 42 | # add it to path on the engines 43 | # client[:].apply_sync(add_to_path, master_project_parent) 44 | print('profiling task: ', selected_profiling_task) 45 | print('profiling view: ', selected_view) 46 | if selected_view == 'direct': 47 | view = client[:] 48 | elif selected_view == 'spanning_tree': 49 | view = client.spanning_tree_view() 50 | else: 51 | view = client.load_balanced_view() 52 | 53 | # view = client[:] if selected_view == 'direct' else client.load_balanced_view() 54 | 55 | if selected_profiling_task == 'many_empty_tasks': 56 | for x in range(1, 5): 57 | profile_many_empty_tasks(view, 10**x) 58 | elif selected_profiling_task == 'many_empty_tasks_non_blocking': 59 | for x in range(1, 5): 60 | profile_many_empty_tasks(view, 10**x, block=False) 61 | elif selected_profiling_task == 'tasks_with_large_data': 62 | for x in range(1, 8): 63 | profile_tasks_with_large_data(view, 10**x) 64 | # elif selected_profiling_task == 'echo_many_arguments': 65 | # for i in range(100): 66 | # for number_of_arguments in ((2 ** x) - 1 for x in range(1, 9)): 67 | # profile_echo_many_arguments(view, number_of_arguments) 68 | 69 | 70 | if __name__ == "__main__": 71 | run_profiling(sys.argv[1], sys.argv[2]) 72 | -------------------------------------------------------------------------------- /benchmarks/profiling/profiling_code_runner.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import sys 4 | import time 5 | from subprocess import Popen, check_call, check_output 6 | 7 | import ipyparallel as ipp 8 | 9 | 10 | def wait_for(condition): 11 | for _ in range(750): 12 | if condition(): 13 | break 14 | else: 15 | time.sleep(0.1) 16 | if not condition(): 17 | raise TimeoutError('wait_for took to long to finish') 18 | 19 | 20 | def get_time_stamp() -> str: 21 | return ( 22 | str(datetime.datetime.now()).split(".")[0].replace(" ", "-").replace(":", "-") 23 | ) 24 | 25 | 26 | def start_cmd(cmd, blocking=True): 27 | print(cmd) 28 | return ( 29 | check_call( 30 | cmd, 31 | stdout=sys.__stdout__, 32 | stderr=open('spanning_tree_error.out', 'a+'), 33 | shell=True, 34 | ) 35 | if blocking 36 | else Popen( 37 | cmd, 38 | stdout=sys.__stdout__, 39 | stderr=open('spanning_tree_error.out', 'a+'), 40 | shell=True, 41 | ) 42 | ) 43 | 44 | 45 | def stop_cluster(): 46 | if '-s' not in sys.argv: 47 | start_cmd('ipcluster stop --profile=asv') 48 | 49 | 50 | # atexit.register(stop_cluster) 51 | 52 | PROFILING_TASKS = [ 53 | 'many_empty_tasks', 54 | 'many_empty_tasks_non_blocking', 55 | 'tasks_with_large_data', 56 | 'echo_many_arguments', 57 | ] 58 | 59 | VIEW_TYPES = ['direct', 'load_balanced'] 60 | 61 | 62 | def get_tasks_to_execute(program_arguments): 63 | return ( 64 | [f'{program_arguments[2]} {view}' for view in VIEW_TYPES] 65 | if len(program_arguments) >= 2 66 | else [f'{task} {view}' for task in PROFILING_TASKS for view in VIEW_TYPES] 67 | ) 68 | 69 | 70 | if __name__ == "__main__": 71 | print('profiling_code_runner_started') 72 | if '-s' not in sys.argv: 73 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 16 74 | client = ipp.Client(profile='asv') 75 | print(f'Waiting for {n} engines to get available') 76 | try: 77 | wait_for(lambda: len(client) >= n) 78 | except TimeoutError as e: 79 | print(e) 80 | exit(1) 81 | print('Starting the profiling') 82 | 83 | controller_pid = check_output('pgrep -f ipyparallel.controller', shell=True) 84 | number_of_schedulers = 15 85 | scheduler_pids = sorted(int(x) for x in controller_pid.decode('utf-8').split())[ 86 | -number_of_schedulers: 87 | ] 88 | 89 | client_output_path = os.path.join(os.getcwd(), 'spanning_tree_client.svg') 90 | 91 | files_to_upload = [client_output_path] 92 | ps = [] 93 | for i, scheduler_pid in enumerate(scheduler_pids): 94 | scheduler_output_path = os.path.join( 95 | os.getcwd(), f'spanning_tree_{i}_scheduler.svg' 96 | ) 97 | files_to_upload.append(scheduler_output_path) 98 | ps.append( 99 | start_cmd( 100 | f'sudo py-spy --function -d 60 --flame {scheduler_output_path} --pid {scheduler_pid}', 101 | blocking=False, 102 | ) 103 | ) 104 | 105 | start_cmd( 106 | f'sudo py-spy --function -d 60 --flame {client_output_path} -- python profiling_code.py tasks_with_large_data spanning_tree' 107 | ) 108 | print('client ended') 109 | for p in ps: 110 | p.wait() 111 | -------------------------------------------------------------------------------- /benchmarks/profiling/view_profiling_results.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | master_project_path = os.path.abspath( 4 | os.path.join(os.path.dirname(__file__), os.pardir) 5 | ) 6 | ALL_RESULTS_DIRECTORY = os.path.join(master_project_path, 'results', 'profiling') 7 | 8 | 9 | def get_latest_results_dir(): 10 | return os.path.join( 11 | 'results', 12 | 'profiling', 13 | max( 14 | dirname 15 | for dirname in os.listdir(ALL_RESULTS_DIRECTORY) 16 | if 'initial_results' not in dirname 17 | ), 18 | ) 19 | 20 | 21 | def get_initial_results_dir(): 22 | return os.path.join( 23 | 'results', 24 | 'profiling', 25 | next( 26 | ( 27 | dirname 28 | for dirname in os.listdir(ALL_RESULTS_DIRECTORY) 29 | if 'initial_results' in dirname 30 | ), 31 | max(dirname for dirname in os.listdir(ALL_RESULTS_DIRECTORY)), 32 | ), 33 | ) 34 | 35 | 36 | if __name__ == '__main__': 37 | print(get_latest_results_dir()) 38 | -------------------------------------------------------------------------------- /benchmarks/results/asv_testing-16-2020-05-04-17-43/results.json: -------------------------------------------------------------------------------- 1 | { 2 | "results": { 3 | "throughput.CoalescingBroadcast.time_broadcast": { 4 | "result": null, 5 | "params": [ 6 | ["0", "0.1"], 7 | ["1", "10", "50", "100", "200"], 8 | ["10", "100", "1000", "10000", "100000", "1000000"] 9 | ] 10 | }, 11 | "throughput.DirectViewBroadCast.time_broadcast": { 12 | "result": null, 13 | "params": [ 14 | ["0", "0.1"], 15 | ["1", "10", "50", "100", "200"], 16 | ["10", "100", "1000", "10000", "100000", "1000000"] 17 | ] 18 | }, 19 | "throughput.NonCoalescingBroadcast.time_broadcast": { 20 | "result": null, 21 | "params": [ 22 | ["0", "0.1"], 23 | ["1", "10", "50", "100", "200"], 24 | ["10", "100", "1000", "10000", "100000", "1000000"] 25 | ] 26 | }, 27 | "throughput.SpanningTreeBroadCast.time_broadcast": { 28 | "result": null, 29 | "params": [ 30 | ["0", "0.1"], 31 | ["1", "10", "50", "100", "200"], 32 | ["10", "100", "1000", "10000", "100000", "1000000"] 33 | ] 34 | } 35 | }, 36 | "params": { 37 | "arch": "x86_64", 38 | "cpu": "Intel(R) Xeon(R) CPU @ 2.30GHz", 39 | "machine": "asv-testing-16", 40 | "os": "Linux 4.15.0-1029-gcp", 41 | "ram": "16423396", 42 | "python": "3.7", 43 | "numpy": "" 44 | }, 45 | "requirements": { 46 | "numpy": "" 47 | }, 48 | "commit_hash": "766bdabf611694c8bf1807677ed3a501bf0b87f2", 49 | "date": 1588108958000, 50 | "env_name": "conda-py3.7-numpy", 51 | "python": "3.7", 52 | "profiles": {}, 53 | "started_at": { 54 | "throughput.CoalescingBroadcast.time_broadcast": 1588607341576, 55 | "throughput.DirectViewBroadCast.time_broadcast": 1588607371588, 56 | "throughput.NonCoalescingBroadcast.time_broadcast": 1588607401077, 57 | "throughput.SpanningTreeBroadCast.time_broadcast": 1588607436141 58 | }, 59 | "ended_at": { 60 | "throughput.CoalescingBroadcast.time_broadcast": 1588607371588, 61 | "throughput.DirectViewBroadCast.time_broadcast": 1588607401077, 62 | "throughput.NonCoalescingBroadcast.time_broadcast": 1588607436140, 63 | "throughput.SpanningTreeBroadCast.time_broadcast": 1588607459984 64 | }, 65 | "benchmark_version": { 66 | "throughput.CoalescingBroadcast.time_broadcast": "fa34b3be29f11e8e92ff3f69ee3db3cef435c17dc271602df2b926f774e93a8e", 67 | "throughput.DirectViewBroadCast.time_broadcast": "fa34b3be29f11e8e92ff3f69ee3db3cef435c17dc271602df2b926f774e93a8e", 68 | "throughput.NonCoalescingBroadcast.time_broadcast": "fa34b3be29f11e8e92ff3f69ee3db3cef435c17dc271602df2b926f774e93a8e", 69 | "throughput.SpanningTreeBroadCast.time_broadcast": "fa34b3be29f11e8e92ff3f69ee3db3cef435c17dc271602df2b926f774e93a8e" 70 | }, 71 | "version": 1 72 | } 73 | -------------------------------------------------------------------------------- /benchmarks/utils.py: -------------------------------------------------------------------------------- 1 | def seconds_to_ms(seconds): 2 | return round(seconds * 1000, 2) 3 | 4 | 5 | def ms_to_seconds(ms): 6 | return ms / 1000 7 | -------------------------------------------------------------------------------- /ci/slurm/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1.2.1 2 | FROM ubuntu:20.04 3 | 4 | ENV DEBIAN_FRONTEND=noninteractive 5 | RUN --mount=type=cache,target=/var/cache/apt \ 6 | rm -f /etc/apt/apt.conf.d/docker-clean \ 7 | && apt-get update && apt-get -y install python3-pip slurm-wlm 8 | ENV PIP_CACHE_DIR=/tmp/pip-cache 9 | RUN --mount=type=cache,target=${PIP_CACHE_DIR} python3 -m pip install ipyparallel pytest-asyncio pytest-cov 10 | RUN mkdir /var/spool/slurmctl \ 11 | && mkdir /var/spool/slurmd 12 | COPY slurm.conf /etc/slurm-llnl/slurm.conf 13 | COPY entrypoint.sh /entrypoint 14 | ENV IPP_DISABLE_JS=1 15 | ENTRYPOINT ["/entrypoint"] 16 | 17 | # the mounted directory 18 | RUN mkdir /io 19 | ENV PYTHONPATH=/io 20 | WORKDIR "/io" 21 | -------------------------------------------------------------------------------- /ci/slurm/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "2.2" 2 | 3 | services: 4 | slurmctld: 5 | image: ipp-cluster:slurm 6 | build: . 7 | container_name: slurmctld 8 | hostname: slurmctld 9 | command: 10 | - tail 11 | - "-f" 12 | - /var/log/slurm-llnl/slurmctld.log 13 | volumes: 14 | - etc_munge:/etc/munge 15 | - etc_slurm:/etc/slurm 16 | - slurm_jobdir:/data 17 | - var_log_slurm:/var/log/slurm 18 | - ../..:/io 19 | expose: 20 | - "6817" 21 | networks: 22 | common-network: 23 | ipv4_address: 10.1.1.10 24 | 25 | c1: 26 | image: ipp-cluster:slurm 27 | build: . 28 | hostname: c1 29 | command: 30 | - tail 31 | - "-f" 32 | - /var/log/slurm-llnl/slurmd.log 33 | container_name: c1 34 | 35 | volumes: 36 | - etc_munge:/etc/munge 37 | - etc_slurm:/etc/slurm 38 | - slurm_jobdir:/data 39 | - var_log_slurm:/var/log/slurm 40 | - ../..:/io 41 | expose: 42 | - "6818" 43 | depends_on: 44 | - "slurmctld" 45 | networks: 46 | common-network: 47 | ipv4_address: 10.1.1.11 48 | 49 | c2: 50 | image: ipp-cluster:slurm 51 | build: . 52 | command: 53 | - tail 54 | - "-f" 55 | - /var/log/slurm-llnl/slurmd.log 56 | hostname: c2 57 | container_name: c2 58 | volumes: 59 | - etc_munge:/etc/munge 60 | - etc_slurm:/etc/slurm 61 | - slurm_jobdir:/data 62 | - var_log_slurm:/var/log/slurm 63 | - ../..:/io 64 | expose: 65 | - "6818" 66 | depends_on: 67 | - "slurmctld" 68 | networks: 69 | common-network: 70 | ipv4_address: 10.1.1.12 71 | 72 | volumes: 73 | etc_munge: 74 | etc_slurm: 75 | slurm_jobdir: 76 | var_log_slurm: 77 | 78 | networks: 79 | common-network: 80 | driver: bridge 81 | ipam: 82 | driver: default 83 | config: 84 | - subnet: 10.1.1.0/24 85 | -------------------------------------------------------------------------------- /ci/slurm/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | # set permissions on munge dir, may be mounted 4 | chown -R munge:munge /etc/munge 5 | 6 | echo "starting munge" 7 | service munge start 8 | 9 | echo "hostname=$(hostname)" 10 | if [[ "$(hostname)" == *"slurmctl"* ]]; then 11 | echo "starting slurmctld" 12 | service slurmctld start 13 | else 14 | echo "starting slurmd" 15 | service slurmd start 16 | fi 17 | 18 | exec "$@" 19 | -------------------------------------------------------------------------------- /ci/slurm/slurm.conf: -------------------------------------------------------------------------------- 1 | # slurm.conf file generated by configurator easy.html. 2 | # Put this file on all nodes of your cluster. 3 | # See the slurm.conf man page for more information. 4 | # 5 | SlurmctldHost=slurmctld 6 | # 7 | #MailProg=/bin/mail 8 | MpiDefault=none 9 | #MpiParams=ports=#-# 10 | ProctrackType=proctrack/linuxproc 11 | ReturnToService=1 12 | SlurmctldPidFile=/var/run/slurmctld.pid 13 | #SlurmctldPort=6817 14 | SlurmdPidFile=/var/run/slurmd.pid 15 | #SlurmdPort=6818 16 | SlurmdSpoolDir=/var/spool/slurmd 17 | SlurmUser=root 18 | #SlurmdUser=root 19 | StateSaveLocation=/var/spool/slurmctl 20 | SwitchType=switch/none 21 | #TaskPlugin=task/affinity 22 | #CoreSpecPlugin=core_spec/none 23 | 24 | # 25 | # TIMERS 26 | #KillWait=30 27 | #MinJobAge=300 28 | #SlurmctldTimeout=120 29 | #SlurmdTimeout=300 30 | # 31 | # 32 | # SCHEDULING 33 | SchedulerType=sched/backfill 34 | SelectType=select/cons_tres 35 | SelectTypeParameters=CR_Core 36 | # 37 | # 38 | # LOGGING AND ACCOUNTING 39 | AccountingStorageType=accounting_storage/none 40 | ClusterName=cluster 41 | #JobAcctGatherFrequency=30 42 | JobAcctGatherType=jobacct_gather/none 43 | SlurmctldDebug=debug5 44 | SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log 45 | SlurmdDebug=debug5 46 | SlurmdLogFile=/var/log/slurm-llnl/slurmd.log 47 | # 48 | # 49 | # COMPUTE NODES 50 | # Note: CPUs apparently cannot be oversubscribed 51 | # this can only run where at least 2 CPUs are available 52 | #NodeName=localhost NodeHostName=localhost NodeAddr=127.0.0.1 CPUs=2 State=UNKNOWN 53 | #PartitionName=part Nodes=localhost Default=YES MaxTime=INFINITE State=UP OverSubscribe=YES 54 | 55 | # COMPUTE NODES 56 | NodeName=c[1-2] RealMemory=4096 CPUs=2 State=UNKNOWN 57 | # 58 | # PARTITIONS 59 | PartitionName=normal Default=yes Nodes=c[1-2] Priority=50 DefMemPerCPU=2048 Shared=NO MaxNodes=2 MaxTime=5-00:00:00 DefaultTime=5-00:00:00 State=UP 60 | -------------------------------------------------------------------------------- /ci/ssh/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1.2.1 2 | FROM ubuntu:20.04 3 | RUN --mount=type=cache,target=/var/cache/apt \ 4 | rm -f /etc/apt/apt.conf.d/docker-clean \ 5 | && apt-get update \ 6 | && apt-get -y install \ 7 | iputils-ping \ 8 | bind9-utils \ 9 | wget \ 10 | openssh-server 11 | 12 | ENV MAMBA_ROOT_PREFIX=/opt/conda 13 | ENV PATH=$MAMBA_ROOT_PREFIX/bin:$PATH 14 | ENV IPP_DISABLE_JS=1 15 | # x86_64 -> 64, aarch64 unmodified 16 | RUN ARCH=$(uname -m | sed s@x86_@@) \ 17 | && wget -qO- https://github.com/mamba-org/micromamba-releases/releases/latest/download/micromamba-linux-${ARCH} > /usr/local/bin/micromamba \ 18 | && chmod +x /usr/local/bin/micromamba 19 | 20 | RUN --mount=type=cache,target=${MAMBA_ROOT_PREFIX}/pkgs \ 21 | micromamba install -y -p $MAMBA_ROOT_PREFIX -c conda-forge \ 22 | python=3.8 \ 23 | pip \ 24 | ipyparallel 25 | 26 | # generate a user with home directory and trusted ssh keypair 27 | RUN useradd -m -s /bin/bash -N ciuser 28 | USER ciuser 29 | RUN mkdir ~/.ssh \ 30 | && chmod 0700 ~/.ssh \ 31 | && ssh-keygen -q -t rsa -N '' -f /home/ciuser/.ssh/id_rsa \ 32 | && cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys \ 33 | && chmod 0600 ~/.ssh/* 34 | USER root 35 | 36 | 37 | ENV PIP_CACHE_DIR=/tmp/pip-cache 38 | COPY . /src/ipyparallel 39 | RUN --mount=type=cache,target=${PIP_CACHE_DIR} python3 -m pip install -e 'file:///src/ipyparallel#egg=ipyparallel[test]' 40 | 41 | # needed for sshd to start 42 | RUN mkdir /run/sshd 43 | # run sshd in the foreground 44 | CMD /usr/sbin/sshd -D -e 45 | -------------------------------------------------------------------------------- /ci/ssh/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | sshd: 3 | image: ipyparallel-sshd 4 | build: 5 | context: ../.. 6 | dockerfile: ci/ssh/Dockerfile 7 | ports: 8 | - "2222:22" 9 | -------------------------------------------------------------------------------- /ci/ssh/ipcluster_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | c = get_config() # noqa 4 | 5 | c.Cluster.controller_ip = '0.0.0.0' 6 | c.Cluster.engine_launcher_class = 'SSH' 7 | 8 | ssh_key = os.path.join(os.path.dirname(__file__), "id_rsa") 9 | c.Cluster.controller_ip = '0.0.0.0' 10 | c.Cluster.engine_launcher_class = 'SSH' 11 | c.SSHEngineSetLauncher.scp_args = c.SSHLauncher.ssh_args = [ 12 | "-o", 13 | "UserKnownHostsFile=/dev/null", 14 | "-o", 15 | "StrictHostKeyChecking=no", 16 | "-i", 17 | ssh_key, 18 | ] 19 | c.SSHEngineSetLauncher.engines = {"ciuser@127.0.0.1:2222": 4} 20 | c.SSHEngineSetLauncher.remote_python = "/opt/conda/bin/python3" 21 | c.SSHEngineSetLauncher.remote_profile_dir = "/home/ciuser/.ipython/profile_default" 22 | c.SSHEngineSetLauncher.engine_args = ['--debug'] 23 | -------------------------------------------------------------------------------- /ci/ssh/win_Dockerfile_template: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1.2.1 2 | FROM python:3.12-windowsservercore-ltsc2022 3 | SHELL ["powershell"] 4 | 5 | # using docker env doesn't seem to work fully correctly. The environment variable is set in the powershell 6 | # but somehow it is not present during pip install process, which will cause the ipyparallel installation 7 | # to fail. Therefore, we use the SetEnvironmentVariable command (for the machine) to make this 'permanent'. 8 | # See run command below. 9 | ENV IPP_DISABLE_JS=1 10 | 11 | # the following env variable values will be 'statically' replaced by the corresponding github workflow script 12 | # if the values aren't replaced the container hosts file isn't changed which is typically no problem in a local 13 | # environment. but it's a necessity for github runners, since the host name resolution inside the container doesn't 14 | # work correctly while trying to register with the ipyparallel controller (for linux runners this isn't an issue) 15 | ENV docker_host_ip ${docker_host_ip} 16 | ENV docker_host_name ${docker_host_name} 17 | 18 | # set IPP_DISABLE_JS=1 and install latest node js version 19 | RUN [System.Environment]::SetEnvironmentVariable('IPP_DISABLE_JS','1', [EnvironmentVariableTarget]::Machine); \ 20 | #add the docker host name and ip to the container hosts file (needed for the github runners since the docker host name resolution doesn't work there) 21 | $hostsfile='C:\Windows\System32\drivers\etc\hosts'; \ 22 | $line=\"$env:docker_host_ip $env:docker_host_name\"; \ 23 | if ($line.Trim().Length -eq 0) { \ 24 | Write-Host 'Environment variables docker_host_[name|ip] not set. Hosts file unchanged!'; \ 25 | } else { \ 26 | Write-Host 'Adapting hosts file '; \ 27 | $h=(Get-Content $hostsfile)+$line; \ 28 | echo $h | out-file -encoding ASCII $hostsfile; \ 29 | type $hostsfile; \ 30 | } 31 | 32 | RUN Get-WindowsCapability -Online | Where-Object Name -like 'OpenSSH*'; \ 33 | Write-Host 'Install OpenSSH Server...'; \ 34 | Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0; \ 35 | Write-Host 'Initializing OpenSSH Server...'; \ 36 | Start-Service sshd; \ 37 | Stop-Service sshd 38 | 39 | 40 | # create ciuser including key pair 41 | RUN Write-Host 'Create user ciuser...';\ 42 | NET USER ciuser /add 43 | USER ciuser 44 | RUN Write-Host 'Create key pair and copy public key...';\ 45 | ssh-keygen -t rsa -N '\"\"' -f $env:USERPROFILE/.ssh/id_rsa; \ 46 | cp $env:USERPROFILE/.ssh/id_rsa.pub $env:USERPROFILE/.ssh/authorized_keys 47 | 48 | # switch back to the admin user 49 | USER containeradministrator 50 | 51 | # This is apparently the only way to keep the sshd service running. 52 | # Running sshd in the foreground in the context of a user (as it is done for linux), doesn't work under Windows. 53 | # Even if it is started as admin user, errors occur during logon (lack of some system rights) 54 | CMD powershell -NoExit -Command "Start-Service sshd" 55 | -------------------------------------------------------------------------------- /ci/ssh/win_base_Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1.2.1 2 | FROM python:3.12-windowsservercore-ltsc2022 3 | SHELL ["powershell"] 4 | 5 | 6 | RUN Get-WindowsCapability -Online | Where-Object Name -like 'OpenSSH*'; \ 7 | Write-Host 'Install OpenSSH Server...'; \ 8 | Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0; \ 9 | Write-Host 'Initializing OpenSSH Server...'; \ 10 | Start-Service sshd; \ 11 | Stop-Service sshd 12 | 13 | # This is apparently the only way to keep the sshd service running. 14 | # Running sshd in the foreground in the context of a user (as it is done for linux), doesn't work under Windows. 15 | # Even if it is started as admin user, errors occur during logon (lack of some system rights) 16 | CMD powershell -NoExit -Command "Start-Service sshd" 17 | -------------------------------------------------------------------------------- /ci/ssh/win_docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | sshd: 3 | image: ipyparallel-sshd 4 | tty: true 5 | build: 6 | context: ../.. 7 | dockerfile: ci/ssh/win_Dockerfile 8 | ports: 9 | - "2222:22" 10 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation generated by sphinx-quickstart 2 | # ---------------------------------------------------------------------------- 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= --color -W --keep-going 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. 19 | # 20 | # Several sphinx-build commands can be used through this, for example: 21 | # 22 | # - make clean 23 | # - make linkcheck 24 | # - make spelling 25 | # 26 | %: Makefile 27 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) 28 | 29 | 30 | # Manually added targets - related to code generation 31 | # ---------------------------------------------------------------------------- 32 | 33 | # For local development: 34 | # - builds the html 35 | # - NOTE: If the pre-requisites for the html target is updated, also update the 36 | # Read The Docs section in docs/source/conf.py. 37 | # 38 | html: 39 | $(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) 40 | @echo 41 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 42 | 43 | 44 | # Manually added targets - related to development 45 | # ---------------------------------------------------------------------------- 46 | 47 | # For local development: 48 | # - requires sphinx-autobuild, see 49 | # https://sphinxcontrib-spelling.readthedocs.io/en/latest/ 50 | # - builds and rebuilds html on changes to source, but does not re-generate 51 | # metrics/scopes files 52 | # - starts a livereload enabled webserver and opens up a browser 53 | devenv: html 54 | sphinx-autobuild -b html --open-browser "$(SOURCEDIR)" "$(BUILDDIR)/html" 55 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=--color -W --keep-going 9 | ) 10 | if "%SPHINXBUILD%" == "" ( 11 | set SPHINXBUILD=sphinx-build 12 | ) 13 | set SOURCEDIR=source 14 | set BUILDDIR=_build 15 | 16 | if "%1" == "" goto help 17 | if "%1" == "devenv" goto devenv 18 | goto default 19 | 20 | 21 | :default 22 | %SPHINXBUILD% >NUL 2>NUL 23 | if errorlevel 9009 ( 24 | echo. 25 | echo.The 'sphinx-build' command was not found. Open and read README.md! 26 | exit /b 1 27 | ) 28 | %SPHINXBUILD% -M %1 "%SOURCEDIR%" "%BUILDDIR%" %SPHINXOPTS% 29 | goto end 30 | 31 | 32 | :help 33 | %SPHINXBUILD% -M help "%SOURCEDIR%" "%BUILDDIR%" %SPHINXOPTS% 34 | goto end 35 | 36 | 37 | :devenv 38 | sphinx-autobuild >NUL 2>NUL 39 | if errorlevel 9009 ( 40 | echo. 41 | echo.The 'sphinx-autobuild' command was not found. Open and read README.md! 42 | exit /b 1 43 | ) 44 | sphinx-autobuild -b html --open-browser "%SOURCEDIR%" "%BUILDDIR%/html" 45 | goto end 46 | 47 | 48 | :end 49 | popd 50 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # pin jinja2 due to broken deprecation used in nbconvert 2 | autodoc-traits 3 | intersphinx-registry 4 | jinja2==3.0.* 5 | matplotlib 6 | myst-nb 7 | myst-parser 8 | numpy 9 | pydata-sphinx-theme 10 | sphinx 11 | sphinx-copybutton 12 | sphinxext-rediraffe 13 | -------------------------------------------------------------------------------- /docs/source/_static/IPyParallel-MPI-Example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/_static/IPyParallel-MPI-Example.png -------------------------------------------------------------------------------- /docs/source/_static/basic.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/_static/basic.mp4 -------------------------------------------------------------------------------- /docs/source/api/ipyparallel.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | .. module:: ipyparallel 5 | 6 | .. autodata:: version_info 7 | 8 | The IPython parallel version as a tuple of integers. 9 | There will always be 3 integers. Development releases will have 'dev' as a fourth element. 10 | 11 | Classes 12 | ------- 13 | 14 | .. autoconfigurable:: Cluster 15 | :inherited-members: 16 | 17 | .. autoclass:: Client 18 | :inherited-members: 19 | 20 | .. autoclass:: DirectView 21 | :inherited-members: 22 | 23 | .. autoclass:: LoadBalancedView 24 | :inherited-members: 25 | 26 | .. autoclass:: BroadcastView 27 | :inherited-members: 28 | 29 | .. autoclass:: AsyncResult 30 | :inherited-members: 31 | 32 | .. autoclass:: ViewExecutor 33 | :inherited-members: 34 | 35 | Decorators 36 | ---------- 37 | 38 | IPython parallel provides some decorators to assist in using your functions as tasks. 39 | 40 | .. autodecorator:: interactive 41 | .. autodecorator:: require 42 | .. autodecorator:: depend 43 | .. autodecorator:: remote 44 | .. autodecorator:: parallel 45 | 46 | 47 | Exceptions 48 | ---------- 49 | 50 | .. autoexception:: RemoteError 51 | :no-members: 52 | .. autoexception:: CompositeError 53 | :no-members: 54 | .. autoexception:: NoEnginesRegistered 55 | .. autoexception:: ImpossibleDependency 56 | .. autoexception:: InvalidDependency 57 | -------------------------------------------------------------------------------- /docs/source/examples/Parallel Decorator and map.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Load balanced map and parallel function decorator" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": { 14 | "collapsed": true 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import ipyparallel as ipp" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "rc = ipp.Client()\n", 28 | "v = rc.load_balanced_view()" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 3, 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "name": "stdout", 38 | "output_type": "stream", 39 | "text": [ 40 | "Simple, default map: [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]\n" 41 | ] 42 | } 43 | ], 44 | "source": [ 45 | "result = v.map(lambda x: 2 * x, range(10))\n", 46 | "print(\"Simple, default map: \", list(result))" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 4, 52 | "metadata": {}, 53 | "outputs": [ 54 | { 55 | "name": "stdout", 56 | "output_type": "stream", 57 | "text": [ 58 | "Submitted tasks, got ids: ['b21595ef-61f4-4ec3-ac7a-7888c025e492', '5e29335b-526d-4516-b22f-0a4b358fa242', '18cd0bd2-7aad-4340-8c81-9b2e384d73d9', '1ef7ccbc-6790-4479-aa90-c4acb5fc8cc4', '8d2c6d43-6e59-4dcf-9511-70707871aeb1', '58042f85-a7c1-492e-a698-d2655c095424', 'd629bf13-4d8b-4a54-996e-d531306293ea', '79039685-1b02-4aa5-a259-4eb9b8d8a65a', '16ffe6f3-fe82-4610-9ec9-a0a3138313a9', 'a3d9050b-faf2-4fa4-873a-65c81cab4c56']\n", 59 | "Using a mapper: [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]\n" 60 | ] 61 | } 62 | ], 63 | "source": [ 64 | "ar = v.map_async(lambda x: 2 * x, range(10))\n", 65 | "print(\"Submitted tasks, got ids: \", ar.msg_ids)\n", 66 | "result = ar.get()\n", 67 | "print(\"Using a mapper: \", result)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 5, 73 | "metadata": {}, 74 | "outputs": [ 75 | { 76 | "name": "stdout", 77 | "output_type": "stream", 78 | "text": [ 79 | "Using a parallel function: [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]\n" 80 | ] 81 | } 82 | ], 83 | "source": [ 84 | "@v.parallel(block=True)\n", 85 | "def f(x):\n", 86 | " return 2 * x\n", 87 | "\n", 88 | "\n", 89 | "result = f.map(range(10))\n", 90 | "print(\"Using a parallel function: \", result)" 91 | ] 92 | } 93 | ], 94 | "metadata": { 95 | "kernelspec": { 96 | "display_name": "Python 3", 97 | "language": "python", 98 | "name": "python3" 99 | }, 100 | "language_info": { 101 | "codemirror_mode": { 102 | "name": "ipython", 103 | "version": 3 104 | }, 105 | "file_extension": ".py", 106 | "mimetype": "text/x-python", 107 | "name": "python", 108 | "nbconvert_exporter": "python", 109 | "pygments_lexer": "ipython3", 110 | "version": "3.5.1" 111 | } 112 | }, 113 | "nbformat": 4, 114 | "nbformat_minor": 0 115 | } 116 | -------------------------------------------------------------------------------- /docs/source/examples/broadcast/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/scipy-notebook:4a6f5b7e5db1 2 | RUN mamba install -yq openmpi mpi4py 3 | RUN pip install --upgrade https://github.com/ipython/ipyparallel/archive/HEAD.tar.gz 4 | -------------------------------------------------------------------------------- /docs/source/examples/broadcast/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | # need /ipython to be writable by the container user (1000) 3 | # can't seem to start volumes with the right permissions 4 | init-permissions: 5 | image: alpine 6 | volumes: 7 | - ipython:/ipython 8 | command: chown -Rv 1000:1000 /ipython 9 | 10 | controller: 11 | depends_on: 12 | - init-permissions 13 | # condition: service_completed_successfully 14 | 15 | build: . 16 | volumes: 17 | - ipython:/ipython 18 | environment: 19 | IPYTHONDIR: /ipython 20 | command: | 21 | ipcontroller --location controller --ip 0.0.0.0 --nodb --HubFactory.broadcast_scheduler_depth=2 22 | deploy: 23 | restart_policy: 24 | condition: on-failure 25 | engines: 26 | build: . 27 | depends_on: 28 | - controller 29 | volumes_from: 30 | - controller 31 | environment: 32 | IPYTHONDIR: /ipython 33 | links: 34 | - controller 35 | command: | 36 | ipcluster engines -n 16 37 | deploy: 38 | mode: replicated 39 | replicas: 4 40 | restart_policy: 41 | condition: on-failure 42 | 43 | client: 44 | build: . 45 | ports: 46 | - "9999:8888" 47 | links: 48 | - controller 49 | volumes_from: 50 | - controller 51 | volumes: 52 | - .:/home/jovyan/example 53 | environment: 54 | IPYTHONDIR: /ipython 55 | JUPYTER_CONFIG_DIR: /ipython/jupyter 56 | 57 | volumes: 58 | ipython: {} 59 | -------------------------------------------------------------------------------- /docs/source/examples/customresults.py: -------------------------------------------------------------------------------- 1 | """An example for handling results in a way that AsyncMapResult doesn't provide 2 | 3 | Specifically, out-of-order results with some special handing of metadata. 4 | 5 | This just submits a bunch of jobs, waits on the results, and prints the stdout 6 | and results of each as they finish. 7 | 8 | Authors 9 | ------- 10 | * MinRK 11 | """ 12 | 13 | import random 14 | 15 | import ipyparallel as ipp 16 | 17 | # create client & views 18 | rc = ipp.Client() 19 | dv = rc[:] 20 | v = rc.load_balanced_view() 21 | 22 | 23 | # scatter 'id', so id=0,1,2 on engines 0,1,2 24 | dv.scatter('id', rc.ids, flatten=True) 25 | print(dv['id']) 26 | 27 | 28 | def sleep_here(count, t): 29 | """simple function that takes args, prints a short message, sleeps for a time, and returns the same args""" 30 | import sys 31 | import time 32 | 33 | print(f"hi from engine {id}") 34 | sys.stdout.flush() 35 | time.sleep(t) 36 | return count, t 37 | 38 | 39 | amr = v.map(sleep_here, range(100), [random.random() for i in range(100)], chunksize=2) 40 | 41 | pending = set(amr.msg_ids) 42 | while pending: 43 | try: 44 | rc.wait(pending, 1e-3) 45 | except TimeoutError: 46 | # ignore timeouterrors, since they only mean that at least one isn't done 47 | pass 48 | # finished is the set of msg_ids that are complete 49 | finished = pending.difference(rc.outstanding) 50 | # update pending to exclude those that just finished 51 | pending = pending.difference(finished) 52 | for msg_id in finished: 53 | # we know these are done, so don't worry about blocking 54 | ar = rc.get_result(msg_id) 55 | print(f"job id {msg_id} finished on engine {ar.engine_id}") 56 | print("with stdout:") 57 | print(' ' + ar.stdout.replace('\n', '\n ').rstrip()) 58 | print("and results:") 59 | 60 | # note that each job in a map always returns a list of length chunksize 61 | # even if chunksize == 1 62 | for count, t in ar.get(): 63 | print(f" item {count}: slept for {t:.2f}s") 64 | -------------------------------------------------------------------------------- /docs/source/examples/daVinci Word Count/pwordfreq.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Parallel word frequency counter. 3 | 4 | This only works for a local cluster, because the filenames are local paths. 5 | """ 6 | 7 | import os 8 | import time 9 | from itertools import repeat 10 | 11 | import requests 12 | from wordfreq import print_wordfreq, wordfreq 13 | 14 | import ipyparallel as ipp 15 | 16 | davinci_url = "https://www.gutenberg.org/files/5000/5000-8.txt" 17 | 18 | 19 | def pwordfreq(view, fnames): 20 | """Parallel word frequency counter. 21 | 22 | view - An IPython DirectView 23 | fnames - The filenames containing the split data. 24 | """ 25 | assert len(fnames) == len(view.targets) 26 | view.scatter('fname', fnames, flatten=True) 27 | ar = view.apply(wordfreq, ipp.Reference('fname')) 28 | freqs_list = ar.get() 29 | word_set = set() 30 | for f in freqs_list: 31 | word_set.update(f.keys()) 32 | freqs = dict(zip(word_set, repeat(0))) 33 | for f in freqs_list: 34 | for word, count in f.items(): 35 | freqs[word] += count 36 | return freqs 37 | 38 | 39 | if __name__ == '__main__': 40 | # Create a Client and View 41 | rc = ipp.Client() 42 | 43 | view = rc[:] 44 | view.apply_sync(os.chdir, os.getcwd()) 45 | 46 | if not os.path.exists('davinci.txt'): 47 | # download from project gutenberg 48 | print("Downloading Da Vinci's notebooks from Project Gutenberg") 49 | r = requests.get(davinci_url) 50 | with open('davinci.txt', 'w', encoding='utf8') as f: 51 | f.write(r.text) 52 | 53 | # Run the serial version 54 | print("Serial word frequency count:") 55 | text = open('davinci.txt', encoding='latin1').read() 56 | tic = time.time() 57 | freqs = wordfreq(text) 58 | toc = time.time() 59 | print_wordfreq(freqs, 10) 60 | print(f"Took {toc - tic:.3f}s to calculate") 61 | 62 | # The parallel version 63 | print("\nParallel word frequency count:") 64 | # split the davinci.txt into one file per engine: 65 | lines = text.splitlines() 66 | nlines = len(lines) 67 | n = len(rc) 68 | block = nlines // n 69 | for i in range(n): 70 | chunk = lines[i * block : i * (block + 1)] 71 | with open(f'davinci{i}.txt', 'w', encoding='utf8') as f: 72 | f.write('\n'.join(chunk)) 73 | 74 | try: # python2 75 | cwd = os.path.abspath(os.getcwdu()) 76 | except AttributeError: # python3 77 | cwd = os.path.abspath(os.getcwd()) 78 | fnames = [os.path.join(cwd, f'davinci{i}.txt') for i in range(n)] 79 | tic = time.time() 80 | pfreqs = pwordfreq(view, fnames) 81 | toc = time.time() 82 | print_wordfreq(freqs) 83 | print(f"Took {toc - tic:.3f} s to calculate on {len(view.targets)} engines") 84 | # cleanup split files 85 | map(os.remove, fnames) 86 | -------------------------------------------------------------------------------- /docs/source/examples/daVinci Word Count/wordfreq.py: -------------------------------------------------------------------------------- 1 | """Count the frequencies of words in a string""" 2 | 3 | 4 | def wordfreq(text, is_filename=False): 5 | """Return a dictionary of words and word counts in a string.""" 6 | if is_filename: 7 | with open(text) as f: 8 | text = f.read() 9 | freqs = {} 10 | for word in text.split(): 11 | lword = word.lower() 12 | freqs[lword] = freqs.get(lword, 0) + 1 13 | return freqs 14 | 15 | 16 | def print_wordfreq(freqs, n=10): 17 | """Print the n most common words and counts in the freqs dict.""" 18 | 19 | words, counts = freqs.keys(), freqs.values() 20 | items = zip(counts, words) 21 | items = sorted(items, reverse=True) 22 | for count, word in items[:n]: 23 | print(word, count) 24 | 25 | 26 | def wordfreq_to_weightsize( 27 | worddict, minsize=25, maxsize=50, minalpha=0.5, maxalpha=1.0 28 | ): 29 | mincount = min(worddict.itervalues()) 30 | maxcount = max(worddict.itervalues()) 31 | weights = {} 32 | for k, v in worddict.iteritems(): 33 | w = (v - mincount) / (maxcount - mincount) 34 | alpha = minalpha + (maxalpha - minalpha) * w 35 | size = minsize + (maxsize - minsize) * w 36 | weights[k] = (alpha, size) 37 | return weights 38 | 39 | 40 | def tagcloud(worddict, n=10, minsize=25, maxsize=50, minalpha=0.5, maxalpha=1.0): 41 | import random 42 | 43 | from matplotlib import pyplot as plt 44 | 45 | worddict = wordfreq_to_weightsize(worddict, minsize, maxsize, minalpha, maxalpha) 46 | 47 | fig = plt.figure() 48 | ax = fig.add_subplot(111) 49 | ax.set_position([0.0, 0.0, 1.0, 1.0]) 50 | plt.xticks([]) 51 | plt.yticks([]) 52 | 53 | words = worddict.keys() 54 | alphas = [v[0] for v in worddict.values()] 55 | sizes = [v[1] for v in worddict.values()] 56 | items = zip(alphas, sizes, words) 57 | items.sort(reverse=True) 58 | for alpha, size, word in items[:n]: 59 | # xpos = random.normalvariate(0.5, 0.3) 60 | # ypos = random.normalvariate(0.5, 0.3) 61 | xpos = random.uniform(0.0, 1.0) 62 | ypos = random.uniform(0.0, 1.0) 63 | ax.text(xpos, ypos, word.lower(), alpha=alpha, fontsize=size) 64 | ax.autoscale_view() 65 | return ax 66 | -------------------------------------------------------------------------------- /docs/source/examples/dagdeps.py: -------------------------------------------------------------------------------- 1 | """Example for generating an arbitrary DAG as a dependency map. 2 | 3 | This demo uses networkx to generate the graph. 4 | 5 | Authors 6 | ------- 7 | * MinRK 8 | """ 9 | 10 | from random import randint 11 | 12 | import networkx as nx 13 | 14 | import ipyparallel as parallel 15 | 16 | 17 | def randomwait(): 18 | import time 19 | from random import random 20 | 21 | time.sleep(random()) 22 | return time.time() 23 | 24 | 25 | def random_dag(nodes, edges): 26 | """Generate a random Directed Acyclic Graph (DAG) with a given number of nodes and edges.""" 27 | G = nx.DiGraph() 28 | for i in range(nodes): 29 | G.add_node(i) 30 | while edges > 0: 31 | a = randint(0, nodes - 1) 32 | b = a 33 | while b == a: 34 | b = randint(0, nodes - 1) 35 | G.add_edge(a, b) 36 | if nx.is_directed_acyclic_graph(G): 37 | edges -= 1 38 | else: 39 | # we closed a loop! 40 | G.remove_edge(a, b) 41 | return G 42 | 43 | 44 | def add_children(G, parent, level, n=2): 45 | """Add children recursively to a binary tree.""" 46 | if level == 0: 47 | return 48 | for i in range(n): 49 | child = parent + str(i) 50 | G.add_node(child) 51 | G.add_edge(parent, child) 52 | add_children(G, child, level - 1, n) 53 | 54 | 55 | def make_bintree(levels): 56 | """Make a symmetrical binary tree with @levels""" 57 | G = nx.DiGraph() 58 | root = '0' 59 | G.add_node(root) 60 | add_children(G, root, levels, 2) 61 | return G 62 | 63 | 64 | def submit_jobs(view, G, jobs): 65 | """Submit jobs via client where G describes the time dependencies.""" 66 | results = {} 67 | for node in nx.topological_sort(G): 68 | with view.temp_flags(after=[results[n] for n in G.predecessors(node)]): 69 | results[node] = view.apply(jobs[node]) 70 | return results 71 | 72 | 73 | def validate_tree(G, results): 74 | """Validate that jobs executed after their dependencies.""" 75 | for node in G: 76 | started = results[node].metadata.started 77 | for parent in G.predecessors(node): 78 | finished = results[parent].metadata.completed 79 | assert started > finished, f"{node} should have happened after {parent}" 80 | 81 | 82 | def main(nodes, edges): 83 | """Generate a random graph, submit jobs, then validate that the 84 | dependency order was enforced. 85 | Finally, plot the graph, with time on the x-axis, and 86 | in-degree on the y (just for spread). All arrows must 87 | point at least slightly to the right if the graph is valid. 88 | """ 89 | from matplotlib import pyplot as plt 90 | from matplotlib.cm import gist_rainbow 91 | from matplotlib.dates import date2num 92 | 93 | print("building DAG") 94 | G = random_dag(nodes, edges) 95 | jobs = {} 96 | pos = {} 97 | colors = {} 98 | for node in G: 99 | jobs[node] = randomwait 100 | 101 | client = parallel.Client() 102 | view = client.load_balanced_view() 103 | print(f"submitting {nodes} tasks with {edges} dependencies") 104 | results = submit_jobs(view, G, jobs) 105 | print("waiting for results") 106 | client.wait_interactive() 107 | for node in G: 108 | md = results[node].metadata 109 | start = date2num(md.started) 110 | runtime = date2num(md.completed) - start 111 | pos[node] = (start, runtime) 112 | colors[node] = md.engine_id 113 | validate_tree(G, results) 114 | nx.draw( 115 | G, 116 | pos, 117 | node_list=list(colors.keys()), 118 | node_color=list(colors.values()), 119 | cmap=gist_rainbow, 120 | with_labels=False, 121 | ) 122 | x, y = zip(*pos.values()) 123 | xmin, ymin = map(min, (x, y)) 124 | xmax, ymax = map(max, (x, y)) 125 | xscale = xmax - xmin 126 | yscale = ymax - ymin 127 | plt.xlim(xmin - xscale * 0.1, xmax + xscale * 0.1) 128 | plt.ylim(ymin - yscale * 0.1, ymax + yscale * 0.1) 129 | return G, results 130 | 131 | 132 | if __name__ == '__main__': 133 | from matplotlib import pyplot as plt 134 | 135 | # main(5,10) 136 | main(32, 96) 137 | plt.show() 138 | -------------------------------------------------------------------------------- /docs/source/examples/dependencies.py: -------------------------------------------------------------------------------- 1 | import ipyparallel as ipp 2 | from ipyparallel import Dependency 3 | 4 | client = ipp.Client() 5 | 6 | 7 | # this will only run on machines that can import numpy: 8 | @ipp.require('numpy') 9 | def norm(A): 10 | from numpy.linalg import norm 11 | 12 | return norm(A, 2) 13 | 14 | 15 | def checkpid(pid): 16 | """return the pid of the engine""" 17 | import os 18 | 19 | return os.getpid() == pid 20 | 21 | 22 | def checkhostname(host): 23 | import socket 24 | 25 | return socket.gethostname() == host 26 | 27 | 28 | def getpid(): 29 | import os 30 | 31 | return os.getpid() 32 | 33 | 34 | pid0 = client[0].apply_sync(getpid) 35 | 36 | 37 | # this will depend on the pid being that of target 0: 38 | @ipp.depend(checkpid, pid0) 39 | def getpid2(): 40 | import os 41 | 42 | return os.getpid() 43 | 44 | 45 | view = client.load_balanced_view() 46 | view.block = True 47 | 48 | # will run on anything: 49 | pids1 = [view.apply(getpid) for i in range(len(client.ids))] 50 | print(pids1) 51 | # will only run on e0: 52 | pids2 = [view.apply(getpid2) for i in range(len(client.ids))] 53 | print(pids2) 54 | 55 | print("now test some dependency behaviors") 56 | 57 | 58 | def wait(t): 59 | import time 60 | 61 | time.sleep(t) 62 | return t 63 | 64 | 65 | # fail after some time: 66 | def wait_and_fail(t): 67 | import time 68 | 69 | time.sleep(t) 70 | return 1 / 0 71 | 72 | 73 | successes = [view.apply_async(wait, 1).msg_ids[0] for i in range(len(client.ids))] 74 | failures = [ 75 | view.apply_async(wait_and_fail, 1).msg_ids[0] for i in range(len(client.ids)) 76 | ] 77 | 78 | mixed = [failures[0], successes[0]] 79 | d1a = Dependency(mixed, all=False, failure=True) # yes 80 | d1b = Dependency(mixed, all=False) # yes 81 | d2a = Dependency(mixed, all=True, failure=True) # yes after / no follow 82 | d2b = Dependency(mixed, all=True) # no 83 | d3 = Dependency(failures, all=False) # no 84 | d4 = Dependency(failures, all=False, failure=True) # yes 85 | d5 = Dependency(failures, all=True, failure=True) # yes after / no follow 86 | d6 = Dependency(successes, all=True, failure=True) # yes after / no follow 87 | 88 | view.block = False 89 | flags = view.temp_flags 90 | with flags(after=d1a): 91 | r1a = view.apply(getpid) 92 | with flags(follow=d1b): 93 | r1b = view.apply(getpid) 94 | with flags(after=d2b, follow=d2a): 95 | r2a = view.apply(getpid) 96 | with flags(after=d2a, follow=d2b): 97 | r2b = view.apply(getpid) 98 | with flags(after=d3): 99 | r3 = view.apply(getpid) 100 | with flags(after=d4): 101 | r4a = view.apply(getpid) 102 | with flags(follow=d4): 103 | r4b = view.apply(getpid) 104 | with flags(after=d3, follow=d4): 105 | r4c = view.apply(getpid) 106 | with flags(after=d5): 107 | r5 = view.apply(getpid) 108 | with flags(follow=d5, after=d3): 109 | r5b = view.apply(getpid) 110 | with flags(follow=d6): 111 | r6 = view.apply(getpid) 112 | with flags(after=d6, follow=d2b): 113 | r6b = view.apply(getpid) 114 | 115 | 116 | def should_fail(f): 117 | try: 118 | f() 119 | except ipp.error.KernelError: 120 | pass 121 | else: 122 | print('should have raised') 123 | # raise Exception("should have raised") 124 | 125 | 126 | # print(r1a.msg_ids) 127 | r1a.get() 128 | # print(r1b.msg_ids) 129 | r1b.get() 130 | # print(r2a.msg_ids) 131 | should_fail(r2a.get) 132 | # print(r2b.msg_ids) 133 | should_fail(r2b.get) 134 | # print(r3.msg_ids) 135 | should_fail(r3.get) 136 | # print(r4a.msg_ids) 137 | r4a.get() 138 | # print(r4b.msg_ids) 139 | r4b.get() 140 | # print(r4c.msg_ids) 141 | should_fail(r4c.get) 142 | # print(r5.msg_ids) 143 | r5.get() 144 | # print(r5b.msg_ids) 145 | should_fail(r5b.get) 146 | # print(r6.msg_ids) 147 | should_fail(r6.get) # assuming > 1 engine 148 | # print(r6b.msg_ids) 149 | should_fail(r6b.get) 150 | print('done') 151 | -------------------------------------------------------------------------------- /docs/source/examples/fetchparse.py: -------------------------------------------------------------------------------- 1 | """ 2 | An exceptionally lousy site spider 3 | Ken Kinder 4 | 5 | Updated for newparallel by Min Ragan-Kelley 6 | 7 | This module gives an example of how the task interface to the 8 | IPython controller works. Before running this script start the IPython controller 9 | and some engines using something like:: 10 | 11 | ipcluster start -n 4 12 | """ 13 | 14 | import sys 15 | import time 16 | 17 | import bs4 # noqa this isn't necessary, but it helps throw the dependency error earlier 18 | 19 | import ipyparallel as ipp 20 | 21 | 22 | def fetchAndParse(url, data=None): 23 | from urllib.parse import urljoin 24 | 25 | import bs4 # noqa 26 | import requests 27 | 28 | links = [] 29 | r = requests.get(url, data=data) 30 | r.raise_for_status() 31 | if 'text/html' in r.headers.get('content-type'): 32 | doc = bs4.BeautifulSoup(r.text, "html.parser") 33 | for node in doc.findAll('a'): 34 | href = node.get('href', None) 35 | if href: 36 | links.append(urljoin(url, href)) 37 | return links 38 | 39 | 40 | class DistributedSpider: 41 | # Time to wait between polling for task results. 42 | pollingDelay = 0.5 43 | 44 | def __init__(self, site): 45 | self.client = ipp.Client() 46 | self.view = self.client.load_balanced_view() 47 | self.mux = self.client[:] 48 | 49 | self.allLinks = [] 50 | self.linksWorking = {} 51 | self.linksDone = {} 52 | 53 | self.site = site 54 | 55 | def visitLink(self, url): 56 | if url not in self.allLinks: 57 | self.allLinks.append(url) 58 | if url.startswith(self.site): 59 | print(' ', url) 60 | self.linksWorking[url] = self.view.apply(fetchAndParse, url) 61 | 62 | def onVisitDone(self, links, url): 63 | print(url + ':') 64 | self.linksDone[url] = None 65 | del self.linksWorking[url] 66 | for link in links: 67 | self.visitLink(link) 68 | 69 | def run(self): 70 | self.visitLink(self.site) 71 | while self.linksWorking: 72 | print(len(self.linksWorking), 'pending...') 73 | self.synchronize() 74 | time.sleep(self.pollingDelay) 75 | 76 | def synchronize(self): 77 | for url, ar in list(self.linksWorking.items()): 78 | # Calling get_task_result with block=False will return None if the 79 | # task is not done yet. This provides a simple way of polling. 80 | try: 81 | links = ar.get(0) 82 | except ipp.error.TimeoutError: 83 | continue 84 | except Exception as e: 85 | self.linksDone[url] = None 86 | del self.linksWorking[url] 87 | print(f'{url}: {e}') 88 | else: 89 | self.onVisitDone(links, url) 90 | 91 | 92 | def main(): 93 | if len(sys.argv) > 1: 94 | site = sys.argv[1] 95 | else: 96 | site = input('Enter site to crawl: ') 97 | distributedSpider = DistributedSpider(site) 98 | distributedSpider.run() 99 | 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /docs/source/examples/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | file_format: mystnb 3 | kernelspec: 4 | name: python3 5 | mystnb: 6 | execution_mode: "force" 7 | remove_code_source: true 8 | --- 9 | 10 | # examples 11 | 12 | Here you will find example notebooks and scripts for working with IPython Parallel. 13 | 14 | (example-tutorials)= 15 | 16 | ## Tutorials 17 | 18 | ```{toctree} 19 | Cluster API 20 | Parallel Magics 21 | progress 22 | Data Publication API 23 | visualizing-tasks 24 | broadcast/Broadcast view 25 | broadcast/memmap Broadcast 26 | broadcast/MPI Broadcast 27 | ``` 28 | 29 | ## Examples 30 | 31 | ```{toctree} 32 | Monitoring an MPI Simulation - 1 33 | Monitoring an MPI Simulation - 2 34 | Parallel Decorator and map 35 | Using MPI with IPython Parallel 36 | Monte Carlo Options 37 | rmt/rmt 38 | ``` 39 | 40 | ## Integrating IPython Parallel with other tools 41 | 42 | There are lots of cool tools for working with asynchronous and parallel execution. IPython Parallel aims to be fairly compatible with these, both by implementing explicit support via methods such as `Client.become_dask`, and by using standards such as the `concurrent.futures.Future` API. 43 | 44 | ```{toctree} 45 | Futures 46 | joblib 47 | dask 48 | Using Dill 49 | ``` 50 | 51 | ## Non-notebook examples 52 | 53 | This directory also contains some examples that are scripts instead of notebooks. 54 | 55 | ```{code-cell} 56 | import glob 57 | import os 58 | 59 | from IPython.display import FileLink, FileLinks, display 60 | 61 | FileLinks(".", included_suffixes=[".py"], recursive=True) 62 | ``` 63 | -------------------------------------------------------------------------------- /docs/source/examples/interengine/bintree_script.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Script for setting up and using [all]reduce with a binary-tree engine interconnect. 4 | 5 | usage: `python bintree_script.py` 6 | 7 | This spanning tree strategy ensures that a single node node mailbox will never 8 | receive more that 2 messages at once. This is very important to scale to large 9 | clusters (e.g. 1000 nodes) since if you have many incoming messages of a couple 10 | of megabytes you might saturate the network interface of a single node and 11 | potentially its memory buffers if the messages are not consumed in a streamed 12 | manner. 13 | 14 | Note that the AllReduce scheme implemented with the spanning tree strategy 15 | impose the aggregation function to be commutative and distributive. It might 16 | not be the case if you implement the naive gather / reduce / broadcast strategy 17 | where you can reorder the partial data before performing the reduce. 18 | """ 19 | 20 | import ipyparallel as ipp 21 | 22 | # connect client and create views 23 | rc = ipp.Client() 24 | rc.block = True 25 | ids = rc.ids 26 | 27 | root_id = ids[0] 28 | root = rc[root_id] 29 | 30 | view = rc[:] 31 | 32 | from bintree import bintree, print_bintree 33 | 34 | # generate binary tree of parents 35 | btree = bintree(ids) 36 | 37 | print("setting up binary tree interconnect:") 38 | print_bintree(btree) 39 | 40 | view.run('bintree.py') 41 | view.scatter('id', ids, flatten=True) 42 | view['root_id'] = root_id 43 | 44 | # create the Communicator objects on the engines 45 | view.execute('com = BinaryTreeCommunicator(id, root = id==root_id )') 46 | pub_url = root.apply_sync(lambda: com.pub_url) # noqa: F821 47 | 48 | # gather the connection information into a dict 49 | ar = view.apply_async(lambda: com.info) # noqa: F821 50 | peers = ar.get_dict() 51 | # this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators 52 | 53 | 54 | # connect the engines to each other: 55 | def connect(com, peers, tree, pub_url, root_id): 56 | """this function will be called on the engines""" 57 | com.connect(peers, tree, pub_url, root_id) 58 | 59 | 60 | view.apply_sync(connect, ipp.Reference('com'), peers, btree, pub_url, root_id) 61 | 62 | 63 | # functions that can be used for reductions 64 | # max and min builtins can be used as well 65 | def add(a, b): 66 | """cumulative sum reduction""" 67 | return a + b 68 | 69 | 70 | def mul(a, b): 71 | """cumulative product reduction""" 72 | return a * b 73 | 74 | 75 | view['add'] = add 76 | view['mul'] = mul 77 | 78 | # scatter some data 79 | data = list(range(1000)) 80 | view.scatter('data', data) 81 | 82 | # perform cumulative sum via allreduce 83 | print("reducing") 84 | ar = view.execute("data_sum = com.allreduce(add, data, flat=False)", block=False) 85 | ar.wait_interactive() 86 | print("allreduce sum of data on all engines:", view['data_sum']) 87 | 88 | # perform cumulative sum *without* final broadcast 89 | # when not broadcasting with allreduce, the final result resides on the root node: 90 | view.execute("ids_sum = com.reduce(add, id, flat=True)") 91 | print("reduce sum of engine ids (not broadcast):", root['ids_sum']) 92 | print("partial result on each engine:", view['ids_sum']) 93 | -------------------------------------------------------------------------------- /docs/source/examples/interengine/communicator.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import uuid 3 | 4 | import zmq 5 | 6 | from ipyparallel.util import disambiguate_url 7 | 8 | 9 | class EngineCommunicator: 10 | def __init__(self, interface='tcp://*', identity=None): 11 | self._ctx = zmq.Context() 12 | self.socket = self._ctx.socket(zmq.XREP) 13 | self.pub = self._ctx.socket(zmq.PUB) 14 | self.sub = self._ctx.socket(zmq.SUB) 15 | 16 | # configure sockets 17 | self.identity = identity or bytes(uuid.uuid4()) 18 | self.socket.IDENTITY = self.identity 19 | self.sub.SUBSCRIBE = b'' 20 | 21 | # bind to ports 22 | port = self.socket.bind_to_random_port(interface) 23 | pub_port = self.pub.bind_to_random_port(interface) 24 | self.url = f"{interface}:{port}" 25 | self.pub_url = f"{interface}:{pub_port}" 26 | # guess first public IP from socket 27 | self.location = socket.gethostbyname_ex(socket.gethostname())[-1][0] 28 | self.peers = {} 29 | 30 | def __del__(self): 31 | self.socket.close() 32 | self.pub.close() 33 | self.sub.close() 34 | self._ctx.term() 35 | 36 | @property 37 | def info(self): 38 | """return the connection info for this object's sockets.""" 39 | return (self.identity, self.url, self.pub_url, self.location) 40 | 41 | def connect(self, peers): 42 | """connect to peers. `peers` will be a dict of 4-tuples, keyed by name. 43 | {peer : (ident, addr, pub_addr, location)} 44 | where peer is the name, ident is the XREP identity, addr,pub_addr are the 45 | """ 46 | for peer, (ident, url, pub_url, location) in peers.items(): 47 | self.peers[peer] = ident 48 | if ident != self.identity: 49 | self.sub.connect(disambiguate_url(pub_url, location)) 50 | if ident > self.identity: 51 | # prevent duplicate xrep, by only connecting 52 | # engines to engines with higher IDENTITY 53 | # a doubly-connected pair will crash 54 | self.socket.connect(disambiguate_url(url, location)) 55 | 56 | def send(self, peers, msg, flags=0, copy=True): 57 | if not isinstance(peers, list): 58 | peers = [peers] 59 | if not isinstance(msg, list): 60 | msg = [msg] 61 | for p in peers: 62 | ident = self.peers[p] 63 | self.socket.send_multipart([ident] + msg, flags=flags, copy=copy) 64 | 65 | def recv(self, flags=0, copy=True): 66 | return self.socket.recv_multipart(flags=flags, copy=copy)[1:] 67 | 68 | def publish(self, msg, flags=0, copy=True): 69 | if not isinstance(msg, list): 70 | msg = [msg] 71 | self.pub.send_multipart(msg, copy=copy) 72 | 73 | def consume(self, flags=0, copy=True): 74 | return self.sub.recv_multipart(flags=flags, copy=copy) 75 | -------------------------------------------------------------------------------- /docs/source/examples/interengine/interengine.py: -------------------------------------------------------------------------------- 1 | import ipyparallel as ipp 2 | 3 | rc = ipp.Client() 4 | rc.block = True 5 | view = rc[:] 6 | view.run('communicator.py') 7 | view.execute('com = EngineCommunicator()') 8 | 9 | # gather the connection information into a dict 10 | ar = view.apply_async(lambda: com.info) # noqa: F821 11 | peers = ar.get_dict() 12 | # this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators 13 | 14 | # connect the engines to each other: 15 | view.apply_sync(lambda pdict: com.connect(pdict), peers) # noqa: F821 16 | 17 | # now all the engines are connected, and we can communicate between them: 18 | 19 | 20 | def broadcast(client, sender, msg_name, dest_name=None, block=None): 21 | """broadcast a message from one engine to all others.""" 22 | dest_name = msg_name if dest_name is None else dest_name 23 | client[sender].execute(f'com.publish({msg_name})', block=None) 24 | targets = client.ids 25 | targets.remove(sender) 26 | return client[targets].execute(f'{dest_name}=com.consume()', block=None) 27 | 28 | 29 | def send(client, sender, targets, msg_name, dest_name=None, block=None): 30 | """send a message from one to one-or-more engines.""" 31 | dest_name = msg_name if dest_name is None else dest_name 32 | 33 | def _send(targets, m_name): 34 | msg = globals()[m_name] 35 | return com.send(targets, msg) # noqa: F821 36 | 37 | client[sender].apply_async(_send, targets, msg_name) 38 | 39 | return client[targets].execute(f'{dest_name}=com.recv()', block=None) 40 | -------------------------------------------------------------------------------- /docs/source/examples/iopubwatcher.py: -------------------------------------------------------------------------------- 1 | """A script for watching all traffic on the IOPub channel (stdout/stderr/pyerr) of engines. 2 | 3 | This connects to the default cluster, or you can pass the path to your ipcontroller-client.json 4 | 5 | Try running this script, and then running a few jobs that print (and call sys.stdout.flush), 6 | and you will see the print statements as they arrive, notably not waiting for the results 7 | to finish. 8 | 9 | You can use the zeromq SUBSCRIBE mechanism to only receive information from specific engines, 10 | and easily filter by message type. 11 | 12 | Authors 13 | ------- 14 | * MinRK 15 | """ 16 | 17 | import json 18 | import sys 19 | 20 | import zmq 21 | from ipykernel.connect import find_connection_file 22 | from jupyter_client.session import Session 23 | 24 | 25 | def main(connection_file): 26 | """watch iopub channel, and print messages""" 27 | 28 | ctx = zmq.Context.instance() 29 | 30 | with open(connection_file) as f: 31 | cfg = json.loads(f.read()) 32 | 33 | reg_url = cfg['interface'] 34 | iopub_port = cfg['iopub'] 35 | iopub_url = f"{reg_url}:{iopub_port}" 36 | 37 | session = Session(key=cfg['key'].encode('ascii')) 38 | sub = ctx.socket(zmq.SUB) 39 | 40 | # This will subscribe to all messages: 41 | sub.SUBSCRIBE = b'' 42 | # replace with b'' with b'engine.1.stdout' to subscribe only to engine 1's stdout 43 | # 0MQ subscriptions are simple 'foo*' matches, so 'engine.1.' subscribes 44 | # to everything from engine 1, but there is no way to subscribe to 45 | # just stdout from everyone. 46 | # multiple calls to subscribe will add subscriptions, e.g. to subscribe to 47 | # engine 1's stderr and engine 2's stdout: 48 | # sub.SUBSCRIBE = b'engine.1.stderr' 49 | # sub.SUBSCRIBE = b'engine.2.stdout' 50 | sub.connect(iopub_url) 51 | while True: 52 | try: 53 | idents, msg = session.recv(sub, mode=0) 54 | except KeyboardInterrupt: 55 | return 56 | # ident always length 1 here 57 | topic = idents[0].decode('utf8', 'replace') 58 | if msg['msg_type'] == 'stream': 59 | # stdout/stderr 60 | # stream names are in msg['content']['name'], if you want to handle 61 | # them differently 62 | print("{}: {}".format(topic, msg['content']['text'])) 63 | elif msg['msg_type'] == 'error': 64 | # Python traceback 65 | c = msg['content'] 66 | print(topic + ':') 67 | for line in c['traceback']: 68 | # indent lines 69 | print(' ' + line) 70 | elif msg['msg_type'] == 'error': 71 | # Python traceback 72 | c = msg['content'] 73 | print(topic + ':') 74 | for line in c['traceback']: 75 | # indent lines 76 | print(' ' + line) 77 | 78 | 79 | if __name__ == '__main__': 80 | if len(sys.argv) > 1: 81 | pattern = sys.argv[1] 82 | else: 83 | # This gets the security file for the default profile: 84 | pattern = 'ipcontroller-client.json' 85 | cf = find_connection_file(pattern) 86 | print(f"Using connection file {cf}") 87 | main(cf) 88 | -------------------------------------------------------------------------------- /docs/source/examples/itermapresult.py: -------------------------------------------------------------------------------- 1 | """Example of iteration through AsyncMapResults, without waiting for all results 2 | 3 | When you call view.map(func, sequence), you will receive a special AsyncMapResult 4 | object. These objects are used to reconstruct the results of the split call. 5 | One feature AsyncResults provide is that they are iterable *immediately*, so 6 | you can iterate through the actual results as they complete. 7 | 8 | This is useful if you submit a large number of tasks that may take some time, 9 | but want to perform logic on elements in the result, or even abort subsequent 10 | tasks in cases where you are searching for the first affirmative result. 11 | 12 | By default, the results will match the ordering of the submitted sequence, but 13 | if you call `map(...ordered=False)`, then results will be provided to the iterator 14 | on a first come first serve basis. 15 | 16 | Authors 17 | ------- 18 | * MinRK 19 | """ 20 | 21 | import time 22 | 23 | import ipyparallel as ipp 24 | 25 | # create client & view 26 | rc = ipp.Client() 27 | dv = rc[:] 28 | v = rc.load_balanced_view() 29 | 30 | # scatter 'id', so id=0,1,2 on engines 0,1,2 31 | dv.scatter('id', rc.ids, flatten=True) 32 | print("Engine IDs: ", dv['id']) 33 | 34 | # create a Reference to `id`. This will be a different value on each engine 35 | ref = ipp.Reference('id') 36 | print("sleeping for `id` seconds on each engine") 37 | tic = time.perf_counter() 38 | ar = dv.apply(time.sleep, ref) 39 | for i, r in enumerate(ar): 40 | print(f"{i}: {time.perf_counter() - tic:.3f}") 41 | 42 | 43 | def sleep_here(t): 44 | import time 45 | 46 | time.sleep(t) 47 | return id, t 48 | 49 | 50 | # one call per task 51 | print("running with one call per task") 52 | amr = v.map(sleep_here, [0.01 * t for t in range(100)]) 53 | tic = time.perf_counter() 54 | for i, r in enumerate(amr): 55 | print(f"task {i} on engine {r[0]}: {time.perf_counter() - tic:.3f}") 56 | 57 | print("running with four calls per task") 58 | # with chunksize, we can have four calls per task 59 | amr = v.map(sleep_here, [0.01 * t for t in range(100)], chunksize=4) 60 | tic = time.perf_counter() 61 | for i, r in enumerate(amr): 62 | print(f"task {i} on engine {r[0]}: {time.perf_counter() - tic:.3f}") 63 | 64 | print("running with two calls per task, with unordered results") 65 | # We can even iterate through faster results first, with ordered=False 66 | amr = v.map( 67 | sleep_here, [0.01 * t for t in range(100, 0, -1)], ordered=False, chunksize=2 68 | ) 69 | tic = time.perf_counter() 70 | for i, r in enumerate(amr): 71 | print(f"slept {r[1]:.2f}s on engine {r[0]}: {time.perf_counter() - tic:.3f}") 72 | -------------------------------------------------------------------------------- /docs/source/examples/phistogram.py: -------------------------------------------------------------------------------- 1 | """Parallel histogram function""" 2 | 3 | from ipyparallel import Reference 4 | 5 | 6 | def phistogram(view, a, bins=10, rng=None, normed=False): 7 | """Compute the histogram of a remote array a. 8 | 9 | Parameters 10 | ---------- 11 | view 12 | IPython DirectView instance 13 | a : str 14 | String name of the remote array 15 | bins : int 16 | Number of histogram bins 17 | rng : (float, float) 18 | Tuple of min, max of the range to histogram 19 | normed : boolean 20 | Should the histogram counts be normalized to 1 21 | """ 22 | nengines = len(view.targets) 23 | 24 | # view.push(dict(bins=bins, rng=rng)) 25 | with view.sync_imports(): 26 | import numpy 27 | rets = view.apply_sync( 28 | lambda a, b, rng: numpy.histogram(a, b, rng), Reference(a), bins, rng 29 | ) 30 | hists = [r[0] for r in rets] 31 | lower_edges = [r[1] for r in rets] 32 | # view.execute('hist, lower_edges = numpy.histogram(%s, bins, rng)' % a) 33 | lower_edges = view.pull('lower_edges', targets=0) 34 | hist_array = numpy.array(hists).reshape(nengines, -1) 35 | # hist_array.shape = (nengines,-1) 36 | total_hist = numpy.sum(hist_array, 0) 37 | if normed: 38 | total_hist = total_hist / numpy.sum(total_hist, dtype=float) 39 | return total_hist, lower_edges 40 | -------------------------------------------------------------------------------- /docs/source/examples/pi/parallelpi.py: -------------------------------------------------------------------------------- 1 | """Calculate statistics on the digits of pi in parallel. 2 | 3 | This program uses the functions in :file:`pidigits.py` to calculate 4 | the frequencies of 2 digit sequences in the digits of pi. The 5 | results are plotted using matplotlib. 6 | 7 | To run, text files from https://www.super-computing.org/ 8 | must be installed in the working directory of the IPython engines. 9 | The actual filenames to be used can be set with the ``filestring`` 10 | variable below. 11 | 12 | The dataset we have been using for this is the 200 million digit one here: 13 | ftp://pi.super-computing.org/.2/pi200m/ 14 | 15 | and the files used will be downloaded if they are not in the working directory 16 | of the IPython engines. 17 | """ 18 | 19 | from timeit import default_timer as clock 20 | 21 | from matplotlib import pyplot as plt 22 | from pidigits import ( 23 | compute_two_digit_freqs, 24 | fetch_pi_file, 25 | plot_two_digit_freqs, 26 | reduce_freqs, 27 | ) 28 | 29 | import ipyparallel as ipp 30 | 31 | # Files with digits of pi (10m digits each) 32 | filestring = 'pi200m.ascii.%(i)02dof20' 33 | files = [filestring % {'i': i} for i in range(1, 21)] 34 | 35 | # Connect to the IPython cluster 36 | c = ipp.Client() 37 | c[:].run('pidigits.py') 38 | 39 | # the number of engines 40 | n = len(c) 41 | id0 = c.ids[0] 42 | v = c[:] 43 | v.block = True 44 | # fetch the pi-files 45 | print(f"downloading {n} files of pi") 46 | v.map(fetch_pi_file, files[:n]) # noqa: F821 47 | print("done") 48 | 49 | # Run 10m digits on 1 engine 50 | t1 = clock() 51 | freqs10m = c[id0].apply_sync(compute_two_digit_freqs, files[0]) 52 | t2 = clock() 53 | digits_per_second1 = 10.0e6 / (t2 - t1) 54 | print("Digits per second (1 core, 10m digits): ", digits_per_second1) 55 | 56 | 57 | # Run n*10m digits on all engines 58 | t1 = clock() 59 | freqs_all = v.map(compute_two_digit_freqs, files[:n]) 60 | freqs150m = reduce_freqs(freqs_all) 61 | t2 = clock() 62 | digits_per_second8 = n * 10.0e6 / (t2 - t1) 63 | print(f"Digits per second ({n} engines, {n}0m digits): ", digits_per_second8) 64 | 65 | print("Speedup: ", digits_per_second8 / digits_per_second1) 66 | 67 | plot_two_digit_freqs(freqs150m) 68 | plt.title(f"2 digit sequences in {n}0m digits of pi") 69 | plt.show() 70 | -------------------------------------------------------------------------------- /docs/source/examples/rmt/rmt.ipy: -------------------------------------------------------------------------------- 1 | # 2 2 | 3 | # 4 | 5 | # # Eigenvalue distribution of Gaussian orthogonal random matrices 6 | 7 | # 8 | 9 | # The eigenvalues of random matrices obey certain statistical laws. Here we construct random matrices 10 | # from the Gaussian Orthogonal Ensemble (GOE), find their eigenvalues and then investigate the nearest 11 | # neighbor eigenvalue distribution $\rho(s)$. 12 | 13 | # 14 | 15 | from rmtkernel import ensemble_diffs, normalize_diffs, GOE 16 | import numpy as np 17 | import ipyparallel as ipp 18 | 19 | # 20 | 21 | # ## Wigner's nearest neighbor eigenvalue distribution 22 | 23 | # 24 | 25 | # The Wigner distribution gives the theoretical result for the nearest neighbor eigenvalue distribution 26 | # for the GOE: 27 | # 28 | # $$\rho(s) = \frac{\pi s}{2} \exp(-\pi s^2/4)$$ 29 | 30 | # 31 | 32 | def wigner_dist(s): 33 | """Returns (s, rho(s)) for the Wigner GOE distribution.""" 34 | return (np.pi*s/2.0) * np.exp(-np.pi*s**2/4.) 35 | 36 | # 37 | 38 | def generate_wigner_data(): 39 | s = np.linspace(0.0,4.0,400) 40 | rhos = wigner_dist(s) 41 | return s, rhos 42 | 43 | # 44 | 45 | s, rhos = generate_wigner_data() 46 | 47 | # 48 | 49 | plot(s, rhos) 50 | xlabel('Normalized level spacing s') 51 | ylabel('Probability $\rho(s)$') 52 | 53 | # 54 | 55 | # ## Serial calculation of nearest neighbor eigenvalue distribution 56 | 57 | # 58 | 59 | # In this section we numerically construct and diagonalize a large number of GOE random matrices 60 | # and compute the nerest neighbor eigenvalue distribution. This comptation is done on a single core. 61 | 62 | # 63 | 64 | def serial_diffs(num, N): 65 | """Compute the nearest neighbor distribution for num NxX matrices.""" 66 | diffs = ensemble_diffs(num, N) 67 | normalized_diffs = normalize_diffs(diffs) 68 | return normalized_diffs 69 | 70 | # 71 | 72 | serial_nmats = 1000 73 | serial_matsize = 50 74 | 75 | # 76 | 77 | %timeit -r1 -n1 serial_diffs(serial_nmats, serial_matsize) 78 | 79 | # 80 | 81 | serial_diffs = serial_diffs(serial_nmats, serial_matsize) 82 | 83 | # 84 | 85 | # The numerical computation agrees with the predictions of Wigner, but it would be nice to get more 86 | # statistics. For that we will do a parallel computation. 87 | 88 | # 89 | 90 | hist_data = hist(serial_diffs, bins=30, normed=True) 91 | plot(s, rhos) 92 | xlabel('Normalized level spacing s') 93 | ylabel('Probability $P(s)$') 94 | 95 | # 96 | 97 | # ## Parallel calculation of nearest neighbor eigenvalue distribution 98 | 99 | # 100 | 101 | # Here we perform a parallel computation, where each process constructs and diagonalizes a subset of 102 | # the overall set of random matrices. 103 | 104 | # 105 | 106 | def parallel-diffs(rc, num, N): 107 | nengines = len(rc.targets) 108 | num_per_engine = num/nengines 109 | print "Running with", num_per_engine, "per engine." 110 | ar = rc.apply_async(ensemble_diffs, num_per_engine, N) 111 | diffs = np.array(ar.get()).flatten() 112 | normalized_diffs = normalize_diffs(diffs) 113 | return normalized_diffs 114 | 115 | # 116 | 117 | client = ipp.Client() 118 | view = client[:] 119 | view.run('rmtkernel.py') 120 | view.block = False 121 | 122 | # 123 | 124 | parallel-nmats = 40*serial_nmats 125 | parallel-matsize = 50 126 | 127 | # 128 | 129 | %timeit -r1 -n1 parallel-diffs(view, parallel-nmats, parallel-matsize) 130 | 131 | # 132 | 133 | pdiffs = parallel-diffs(view, parallel-nmats, parallel-matsize) 134 | 135 | # 136 | 137 | # Again, the agreement with the Wigner distribution is excellent, but now we have better 138 | # statistics. 139 | 140 | # 141 | 142 | hist_data = hist(pdiffs, bins=30, normed=True) 143 | plot(s, rhos) 144 | xlabel('Normalized level spacing s') 145 | ylabel('Probability $P(s)$') 146 | -------------------------------------------------------------------------------- /docs/source/examples/rmt/rmtkernel.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------- 2 | # Core routines for computing properties of symmetric random matrices. 3 | # ------------------------------------------------------------------------------- 4 | import numpy as np 5 | 6 | ra = np.random 7 | la = np.linalg 8 | 9 | 10 | def GOE(N): 11 | """Creates an NxN element of the Gaussian Orthogonal Ensemble""" 12 | m = ra.standard_normal((N, N)) 13 | m += m.T 14 | return m / 2 15 | 16 | 17 | def center_eigenvalue_diff(mat): 18 | """Compute the eigvals of mat and then find the center eigval difference.""" 19 | N = len(mat) 20 | evals = np.sort(la.eigvals(mat)) 21 | diff = np.abs(evals[N / 2] - evals[N / 2 - 1]) 22 | return diff 23 | 24 | 25 | def ensemble_diffs(num, N): 26 | """Return num eigenvalue diffs for the NxN GOE ensemble.""" 27 | diffs = np.empty(num) 28 | for i in range(num): 29 | mat = GOE(N) 30 | diffs[i] = center_eigenvalue_diff(mat) 31 | return diffs 32 | 33 | 34 | def normalize_diffs(diffs): 35 | """Normalize an array of eigenvalue diffs.""" 36 | return diffs / diffs.mean() 37 | 38 | 39 | def normalized_ensemble_diffs(num, N): 40 | """Return num *normalized* eigenvalue diffs for the NxN GOE ensemble.""" 41 | diffs = ensemble_diffs(num, N) 42 | return normalize_diffs(diffs) 43 | -------------------------------------------------------------------------------- /docs/source/examples/task_mod.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from numpy.linalg import norm 3 | 4 | 5 | def task(n): 6 | """Generates a 1xN array and computes its 2-norm""" 7 | A = numpy.ones(n) 8 | return norm(A, 2) 9 | -------------------------------------------------------------------------------- /docs/source/examples/task_profiler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Test the performance of the task farming system. 3 | 4 | This script submits a set of tasks via a LoadBalancedView. The tasks 5 | are basically just a time.sleep(t), where t is a random number between 6 | two limits that can be configured at the command line. To run 7 | the script there must first be an IPython controller and engines running:: 8 | 9 | ipcluster start -n 16 10 | 11 | A good test to run with 16 engines is:: 12 | 13 | python task_profiler.py -n 128 -t 0.01 -T 1.0 14 | 15 | This should show a speedup of 13-14x. The limitation here is that the 16 | overhead of a single task is about 0.001-0.01 seconds. 17 | """ 18 | 19 | import random 20 | import time 21 | from optparse import OptionParser 22 | 23 | import ipyparallel as ipp 24 | 25 | 26 | def main(): 27 | parser = OptionParser() 28 | parser.set_defaults(n=100) 29 | parser.set_defaults(tmin=1e-3) 30 | parser.set_defaults(tmax=1) 31 | parser.set_defaults(profile='default') 32 | 33 | parser.add_option("-n", type='int', dest='n', help='the number of tasks to run') 34 | parser.add_option( 35 | "-t", type='float', dest='tmin', help='the minimum task length in seconds' 36 | ) 37 | parser.add_option( 38 | "-T", type='float', dest='tmax', help='the maximum task length in seconds' 39 | ) 40 | parser.add_option( 41 | "-p", 42 | '--profile', 43 | type='str', 44 | dest='profile', 45 | help="the cluster profile [default: 'default']", 46 | ) 47 | 48 | (opts, args) = parser.parse_args() 49 | assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin" 50 | 51 | rc = ipp.Client() 52 | view = rc.load_balanced_view() 53 | print(view) 54 | rc.block = True 55 | nengines = len(rc.ids) 56 | 57 | # the jobs should take a random time within a range 58 | times = [ 59 | random.random() * (opts.tmax - opts.tmin) + opts.tmin for i in range(opts.n) 60 | ] 61 | stime = sum(times) 62 | 63 | print(f"executing {opts.n} tasks, totalling {stime:.1f} secs on {nengines} engines") 64 | time.sleep(1) 65 | start = time.perf_counter() 66 | amr = view.map(time.sleep, times) 67 | amr.get() 68 | stop = time.perf_counter() 69 | 70 | ptime = stop - start 71 | scale = stime / ptime 72 | 73 | print(f"executed {stime:.1f} secs in {ptime:.1f} secs") 74 | print(f"{scale:3f}x parallel performance on {nengines} engines") 75 | print(f"{scale / nengines:.0%} of theoretical max") 76 | 77 | 78 | if __name__ == '__main__': 79 | main() 80 | -------------------------------------------------------------------------------- /docs/source/examples/throughput.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import numpy as np 4 | 5 | import ipyparallel as parallel 6 | 7 | nlist = map(int, np.logspace(2, 9, 16, base=2)) 8 | nlist2 = map(int, np.logspace(2, 8, 15, base=2)) 9 | tlist = map(int, np.logspace(7, 22, 16, base=2)) 10 | nt = 16 11 | 12 | 13 | def wait(t=0): 14 | import time 15 | 16 | time.sleep(t) 17 | 18 | 19 | def echo(s=''): 20 | return s 21 | 22 | 23 | def time_throughput(nmessages, t=0, f=wait): 24 | client = parallel.Client() 25 | view = client.load_balanced_view() 26 | # do one ping before starting timing 27 | if f is echo: 28 | t = np.random.random(t / 8) 29 | view.apply_sync(echo, '') 30 | client.spin() 31 | tic = time.time() 32 | for i in range(nmessages): 33 | view.apply(f, t) 34 | lap = time.time() 35 | client.wait() 36 | toc = time.time() 37 | return lap - tic, toc - tic 38 | 39 | 40 | def do_runs(nlist, t=0, f=wait, trials=2, runner=time_throughput): 41 | A = np.zeros((len(nlist), 2)) 42 | for i, n in enumerate(nlist): 43 | t1 = t2 = 0 44 | for _ in range(trials): 45 | time.sleep(0.25) 46 | ts = runner(n, t, f) 47 | t1 += ts[0] 48 | t2 += ts[1] 49 | t1 /= trials 50 | t2 /= trials 51 | A[i] = (t1, t2) 52 | A[i] = n / A[i] 53 | print(n, A[i]) 54 | return A 55 | 56 | 57 | def do_echo(n, tlist=[0], f=echo, trials=2, runner=time_throughput): 58 | A = np.zeros((len(tlist), 2)) 59 | for i, t in enumerate(tlist): 60 | t1 = t2 = 0 61 | for _ in range(trials): 62 | time.sleep(0.25) 63 | ts = runner(n, t, f) 64 | t1 += ts[0] 65 | t2 += ts[1] 66 | t1 /= trials 67 | t2 /= trials 68 | A[i] = (t1, t2) 69 | A[i] = n / A[i] 70 | print(t, A[i]) 71 | return A 72 | -------------------------------------------------------------------------------- /docs/source/examples/wave2D/communicator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """A simple Communicator class that has N,E,S,W neighbors connected via 0MQ PEER sockets""" 3 | 4 | import socket 5 | 6 | import zmq 7 | 8 | from ipyparallel.util import disambiguate_url 9 | 10 | 11 | class EngineCommunicator: 12 | """An object that connects Engines to each other. 13 | north and east sockets listen, while south and west sockets connect. 14 | 15 | This class is useful in cases where there is a set of nodes that 16 | must communicate only with their nearest neighbors. 17 | """ 18 | 19 | def __init__(self, interface='tcp://*', identity=None): 20 | self._ctx = zmq.Context() 21 | self.north = self._ctx.socket(zmq.PAIR) 22 | self.west = self._ctx.socket(zmq.PAIR) 23 | self.south = self._ctx.socket(zmq.PAIR) 24 | self.east = self._ctx.socket(zmq.PAIR) 25 | 26 | # bind to ports 27 | northport = self.north.bind_to_random_port(interface) 28 | eastport = self.east.bind_to_random_port(interface) 29 | 30 | self.north_url = f"{interface}:{northport}" 31 | self.east_url = f"{interface}:{eastport}" 32 | 33 | # guess first public IP from socket 34 | self.location = socket.gethostbyname_ex(socket.gethostname())[-1][0] 35 | 36 | def __del__(self): 37 | self.north.close() 38 | self.south.close() 39 | self.east.close() 40 | self.west.close() 41 | self._ctx.term() 42 | 43 | @property 44 | def info(self): 45 | """return the connection info for this object's sockets.""" 46 | return (self.location, self.north_url, self.east_url) 47 | 48 | def connect(self, south_peer=None, west_peer=None): 49 | """connect to peers. `peers` will be a 3-tuples, of the form: 50 | (location, north_addr, east_addr) 51 | as produced by 52 | """ 53 | if south_peer is not None: 54 | location, url, _ = south_peer 55 | self.south.connect(disambiguate_url(url, location)) 56 | if west_peer is not None: 57 | location, _, url = west_peer 58 | self.west.connect(disambiguate_url(url, location)) 59 | -------------------------------------------------------------------------------- /docs/source/index.md: -------------------------------------------------------------------------------- 1 | # Using IPython for parallel computing 2 | 3 | > ```{eval-rst} 4 | > 5 | > :Release: |release| 6 | > :Date: |today| 7 | > ``` 8 | 9 | (install)= 10 | 11 | ## Installing IPython Parallel 12 | 13 | As of 4.0, IPython parallel is now a standalone package called {mod}`ipyparallel`. 14 | You can install it with: 15 | 16 | ``` 17 | pip install ipyparallel 18 | ``` 19 | 20 | or: 21 | 22 | ``` 23 | conda install ipyparallel 24 | ``` 25 | 26 | As of IPython Parallel 7, 27 | this will include installing/enabling an extension for both the classic Jupyter Notebook 28 | and JupyterLab ≥ 3.0. 29 | 30 | ## Quickstart 31 | 32 | IPython Parallel 33 | 34 | A quick example to: 35 | 36 | 1. allocate a cluster (collection of IPython engines for use in parallel) 37 | 2. run a collection of tasks on the cluster 38 | 3. wait interactively for results 39 | 4. cleanup resources after the task is done 40 | 41 | ```python 42 | import time 43 | import ipyparallel as ipp 44 | 45 | task_durations = [1] * 25 46 | # request a cluster 47 | with ipp.Cluster() as rc: 48 | # get a view on the cluster 49 | view = rc.load_balanced_view() 50 | # submit the tasks 51 | asyncresult = view.map_async(time.sleep, task_durations) 52 | # wait interactively for results 53 | asyncresult.wait_interactive() 54 | # retrieve actual results 55 | result = asyncresult.get() 56 | # at this point, the cluster processes have been shutdown 57 | ``` 58 | 59 | 62 | 63 | You can similarly run MPI code using IPyParallel (requires [mpi4py](https://mpi4py.readthedocs.io/en/stable/install.html)): 64 | 65 | ```python 66 | import ipyparallel as ipp 67 | 68 | def mpi_example(): 69 | from mpi4py import MPI 70 | comm = MPI.COMM_WORLD 71 | return f"Hello World from rank {comm.Get_rank()}. total ranks={comm.Get_size()}" 72 | 73 | # request an MPI cluster with 4 engines 74 | with ipp.Cluster(engines='mpi', n=4) as rc: 75 | # get a broadcast_view on the cluster which is best 76 | # suited for MPI style computation 77 | view = rc.broadcast_view() 78 | # run the mpi_example function on all engines in parallel 79 | r = view.apply_sync(mpi_example) 80 | # Retrieve and print the result from the engines 81 | print("\n".join(r)) 82 | # at this point, the cluster processes have been shutdown 83 | ``` 84 | 85 | ![IPyParallel-MPI-Example](./_static/IPyParallel-MPI-Example.png) 86 | 87 | Follow the [tutorial][] to learn more. 88 | 89 | [tutorial]: ./tutorial/index 90 | 91 | ## Contents 92 | 93 | ```{toctree} 94 | :maxdepth: 2 95 | 96 | tutorial/index 97 | reference/index 98 | ``` 99 | 100 | ```{toctree} 101 | :maxdepth: 2 102 | 103 | examples/index 104 | ``` 105 | 106 | ```{toctree} 107 | :maxdepth: 1 108 | 109 | changelog 110 | ``` 111 | 112 | # IPython Parallel API 113 | 114 | ```{toctree} 115 | :maxdepth: 2 116 | 117 | api/ipyparallel.rst 118 | ``` 119 | 120 | # Indices and tables 121 | 122 | - {ref}`genindex` 123 | - {ref}`modindex` 124 | - {ref}`search` 125 | -------------------------------------------------------------------------------- /docs/source/redirects.txt: -------------------------------------------------------------------------------- 1 | "asyncresult.rst" "tutorial/asyncresult.md" 2 | "dag_dependencies.rst" "reference/dag_dependencies.md" 3 | "db.rst" "reference/db.md" 4 | "demos.rst" "tutorial/demos.md" 5 | "details.rst" "reference/details.md" 6 | "development/connections.rst" "reference/connections.md" 7 | "development/messages.rst" "reference/messages.md" 8 | "intro.rst" "tutorial/intro.md" 9 | "magics.rst" "examples/Parallel Magics.ipynb" 10 | "tutorial/magics.md" "examples/Parallel Magics.ipynb" 11 | "mpi.rst" "reference/mpi.md" 12 | "multiengine.rst" "tutorial/direct.md" 13 | "process.rst" "tutorial/process.md" 14 | "security.rst" "reference/security.md" 15 | "task.rst" "tutorial/task.md" 16 | "transition.rst" "tutorial/index.md" 17 | "winhpc.rst" "reference/launchers.md" 18 | 19 | "examples/Index.ipynb" "examples/index.md" 20 | -------------------------------------------------------------------------------- /docs/source/reference/figs/allconnections.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/allconnections.png -------------------------------------------------------------------------------- /docs/source/reference/figs/dagdeps.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/dagdeps.pdf -------------------------------------------------------------------------------- /docs/source/reference/figs/dagdeps.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/dagdeps.png -------------------------------------------------------------------------------- /docs/source/reference/figs/frontend-kernel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/frontend-kernel.png -------------------------------------------------------------------------------- /docs/source/reference/figs/hbfade.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/hbfade.png -------------------------------------------------------------------------------- /docs/source/reference/figs/iopubfade.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/iopubfade.png -------------------------------------------------------------------------------- /docs/source/reference/figs/ipy_kernel_and_terminal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/ipy_kernel_and_terminal.png -------------------------------------------------------------------------------- /docs/source/reference/figs/nbconvert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/nbconvert.png -------------------------------------------------------------------------------- /docs/source/reference/figs/notebook_components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/notebook_components.png -------------------------------------------------------------------------------- /docs/source/reference/figs/notiffade.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/notiffade.png -------------------------------------------------------------------------------- /docs/source/reference/figs/other_kernels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/other_kernels.png -------------------------------------------------------------------------------- /docs/source/reference/figs/queryfade.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/queryfade.png -------------------------------------------------------------------------------- /docs/source/reference/figs/queuefade.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/queuefade.png -------------------------------------------------------------------------------- /docs/source/reference/figs/simpledag.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/simpledag.pdf -------------------------------------------------------------------------------- /docs/source/reference/figs/simpledag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/reference/figs/simpledag.png -------------------------------------------------------------------------------- /docs/source/reference/index.md: -------------------------------------------------------------------------------- 1 | # Reference 2 | 3 | ```{toctree} 4 | --- 5 | maxdepth: 2 6 | --- 7 | mpi 8 | db 9 | security 10 | dag_dependencies 11 | details 12 | messages 13 | connections 14 | launchers 15 | ``` 16 | -------------------------------------------------------------------------------- /docs/source/tutorial/figs/asian_call.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/asian_call.pdf -------------------------------------------------------------------------------- /docs/source/tutorial/figs/asian_call.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/asian_call.png -------------------------------------------------------------------------------- /docs/source/tutorial/figs/asian_put.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/asian_put.pdf -------------------------------------------------------------------------------- /docs/source/tutorial/figs/asian_put.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/asian_put.png -------------------------------------------------------------------------------- /docs/source/tutorial/figs/mec_simple.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/mec_simple.pdf -------------------------------------------------------------------------------- /docs/source/tutorial/figs/mec_simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/mec_simple.png -------------------------------------------------------------------------------- /docs/source/tutorial/figs/parallel_pi.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/parallel_pi.pdf -------------------------------------------------------------------------------- /docs/source/tutorial/figs/parallel_pi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/parallel_pi.png -------------------------------------------------------------------------------- /docs/source/tutorial/figs/single_digits.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/single_digits.pdf -------------------------------------------------------------------------------- /docs/source/tutorial/figs/single_digits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/single_digits.png -------------------------------------------------------------------------------- /docs/source/tutorial/figs/two_digit_counts.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/two_digit_counts.pdf -------------------------------------------------------------------------------- /docs/source/tutorial/figs/two_digit_counts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/two_digit_counts.png -------------------------------------------------------------------------------- /docs/source/tutorial/figs/wideView.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/docs/source/tutorial/figs/wideView.png -------------------------------------------------------------------------------- /docs/source/tutorial/index.md: -------------------------------------------------------------------------------- 1 | # Tutorial 2 | 3 | ```{toctree} 4 | --- 5 | maxdepth: 2 6 | --- 7 | intro 8 | process 9 | direct 10 | task 11 | asyncresult 12 | demos 13 | ``` 14 | 15 | :::{seealso} 16 | [Tutorial example notebooks](example-tutorials) 17 | 18 | ::: 19 | -------------------------------------------------------------------------------- /examples: -------------------------------------------------------------------------------- 1 | docs/source/examples -------------------------------------------------------------------------------- /hatch_build.py: -------------------------------------------------------------------------------- 1 | """Custom build script for hatch backend""" 2 | 3 | import glob 4 | import os 5 | import subprocess 6 | 7 | from hatchling.builders.hooks.plugin.interface import BuildHookInterface 8 | 9 | 10 | class CustomHook(BuildHookInterface): 11 | def initialize(self, version, build_data): 12 | if self.target_name not in ["wheel", "sdist"]: 13 | return 14 | cmd = "build:prod" if version == "standard" else "build" 15 | osp = os.path 16 | here = osp.abspath(osp.dirname(__file__)) 17 | lab_path = osp.join(here, 'ipyparallel', 'labextension') 18 | 19 | if os.environ.get("IPP_DISABLE_JS") == "1": 20 | print("Skipping js installation") 21 | return 22 | 23 | # this tells us if labextension is built at all, not if it's up-to-date 24 | labextension_built = glob.glob(osp.join(lab_path, "*")) 25 | needs_js = True 26 | if not osp.isdir(osp.join(here, ".git")): 27 | print("Installing from a dist, not a repo") 28 | # not in a repo, probably installing from sdist 29 | # could be git-archive, though! 30 | # skip rebuilding js if it's already present 31 | if labextension_built: 32 | print(f"Not regenerating labextension in {lab_path}") 33 | needs_js = False 34 | 35 | if needs_js: 36 | subprocess.check_call(['jlpm'], cwd=here) 37 | subprocess.check_call(['jlpm', 'run', cmd], cwd=here) 38 | -------------------------------------------------------------------------------- /install.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageManager": "python", 3 | "packageName": "ipyparallel", 4 | "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package ipyparallel" 5 | } 6 | -------------------------------------------------------------------------------- /ipyparallel/__init__.py: -------------------------------------------------------------------------------- 1 | """The IPython ZMQ-based parallel computing interface.""" 2 | 3 | # Copyright (c) IPython Development Team. 4 | # Distributed under the terms of the Modified BSD License. 5 | # export return_when constants 6 | import os 7 | from concurrent.futures import ALL_COMPLETED # noqa 8 | from concurrent.futures import FIRST_COMPLETED # noqa 9 | from concurrent.futures import FIRST_EXCEPTION # noqa 10 | 11 | from traitlets.config.configurable import MultipleInstanceError 12 | 13 | from ._version import __version__ # noqa 14 | from ._version import version_info # noqa 15 | from .client.asyncresult import * # noqa 16 | from .client.client import Client # noqa 17 | from .client.remotefunction import * # noqa 18 | from .client.view import * # noqa 19 | from .cluster import Cluster # noqa 20 | from .cluster import ClusterManager # noqa 21 | from .controller.dependency import * # noqa 22 | from .error import * # noqa 23 | from .serialize import * # noqa 24 | from .util import interactive # noqa 25 | 26 | # ----------------------------------------------------------------------------- 27 | # Functions 28 | # ----------------------------------------------------------------------------- 29 | 30 | 31 | def bind_kernel(**kwargs): 32 | """Bind an Engine's Kernel to be used as a full IPython kernel. 33 | 34 | This allows a running Engine to be used simultaneously as a full IPython kernel 35 | with the QtConsole or other frontends. 36 | 37 | This function returns immediately. 38 | """ 39 | from ipykernel.kernelapp import IPKernelApp 40 | 41 | from ipyparallel.engine.app import IPEngine 42 | 43 | if IPEngine.initialized(): 44 | try: 45 | app = IPEngine.instance() 46 | except MultipleInstanceError: 47 | pass 48 | else: 49 | return app.bind_kernel(**kwargs) 50 | 51 | raise RuntimeError("bind_kernel must be called from an IPEngine instance") 52 | 53 | 54 | def register_joblib_backend(name='ipyparallel', make_default=False): 55 | """Register the default ipyparallel backend for joblib.""" 56 | from .joblib import register 57 | 58 | return register(name=name, make_default=make_default) 59 | 60 | 61 | # nbextension installation (requires notebook ≥ 4.2) 62 | def _jupyter_server_extension_paths(): 63 | return [{'module': 'ipyparallel'}] 64 | 65 | 66 | def _jupyter_nbextension_paths(): 67 | return [ 68 | { 69 | 'section': 'tree', 70 | 'src': 'nbextension/static', 71 | 'dest': 'ipyparallel', 72 | 'require': 'ipyparallel/main', 73 | } 74 | ] 75 | 76 | 77 | def _jupyter_labextension_paths(): 78 | return [ 79 | { 80 | "src": "labextension", 81 | "dest": "ipyparallel-labextension", 82 | } 83 | ] 84 | 85 | 86 | def _load_jupyter_server_extension(app): 87 | """Load the server extension""" 88 | # localte the appropriate APIHandler base class before importing our handler classes 89 | from .nbextension.base import get_api_handler 90 | 91 | get_api_handler(app) 92 | 93 | from .nbextension.handlers import load_jupyter_server_extension 94 | 95 | return load_jupyter_server_extension(app) 96 | 97 | 98 | # backward-compat 99 | load_jupyter_server_extension = _load_jupyter_server_extension 100 | 101 | _NONINTERACTIVE = os.getenv("IPP_NONINTERACTIVE", "") not in {"", "0"} 102 | -------------------------------------------------------------------------------- /ipyparallel/_async.py: -------------------------------------------------------------------------------- 1 | """Async utilities""" 2 | 3 | import asyncio 4 | import concurrent.futures 5 | import inspect 6 | import threading 7 | from functools import partial 8 | 9 | from tornado.ioloop import IOLoop 10 | 11 | from ipyparallel.util import _OutputProducingThread as Thread 12 | 13 | 14 | def _asyncio_run(coro): 15 | """Like asyncio.run, but works when there's no event loop""" 16 | # for now: using tornado for broader compatibility with FDs, 17 | # e.g. when using the only partially functional default 18 | # Proactor on windows 19 | loop = IOLoop(make_current=False) 20 | try: 21 | return loop.run_sync(lambda: asyncio.ensure_future(coro)) 22 | finally: 23 | loop.close() 24 | 25 | 26 | class AsyncFirst: 27 | """Wrapper class that defines synchronous `_sync` method wrappers 28 | 29 | around async-native methods. 30 | 31 | Every coroutine method automatically gets an `_sync` alias 32 | that runs it synchronously. 33 | """ 34 | 35 | _async_thread = None 36 | 37 | def _thread_main(self): 38 | loop = self._thread_loop = IOLoop(make_current=False) 39 | loop.add_callback(self._loop_started.set) 40 | loop.start() 41 | 42 | def _in_thread(self, async_f, *args, **kwargs): 43 | """Run an async function in a background thread""" 44 | if self._async_thread is None: 45 | self._loop_started = threading.Event() 46 | self._async_thread = Thread(target=self._thread_main, daemon=True) 47 | self._async_thread.start() 48 | self._loop_started.wait(timeout=5) 49 | 50 | future = concurrent.futures.Future() 51 | 52 | async def thread_callback(): 53 | try: 54 | future.set_result(await async_f(*args, **kwargs)) 55 | except Exception as e: 56 | future.set_exception(e) 57 | 58 | self._thread_loop.add_callback(thread_callback) 59 | return future.result() 60 | 61 | def _synchronize(self, async_f, *args, **kwargs): 62 | """Run a method synchronously 63 | 64 | Uses asyncio.run if asyncio is not running, 65 | otherwise puts it in a background thread 66 | """ 67 | try: 68 | loop = asyncio.get_running_loop() 69 | except RuntimeError: 70 | # not in a running loop 71 | loop = None 72 | if loop: 73 | return self._in_thread(async_f, *args, **kwargs) 74 | else: 75 | return _asyncio_run(async_f(*args, **kwargs)) 76 | 77 | def __getattr__(self, name): 78 | if name.endswith("_sync"): 79 | # lazily define `_sync` method wrappers for coroutine methods 80 | async_name = name[:-5] 81 | async_method = super().__getattribute__(async_name) 82 | if not inspect.iscoroutinefunction(async_method): 83 | raise AttributeError(async_name) 84 | return partial(self._synchronize, async_method) 85 | return super().__getattribute__(name) 86 | 87 | def __dir__(self): 88 | attrs = super().__dir__() 89 | seen = set() 90 | for cls in self.__class__.mro(): 91 | for name, value in cls.__dict__.items(): 92 | if name in seen: 93 | continue 94 | seen.add(name) 95 | if inspect.iscoroutinefunction(value): 96 | async_name = name + "_sync" 97 | attrs.append(async_name) 98 | return attrs 99 | -------------------------------------------------------------------------------- /ipyparallel/_version.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | __version__ = "9.1.0.dev" 4 | 5 | # matches tbump regex in pyproject.toml 6 | _version_regex = re.compile( 7 | r''' 8 | (?P\d+) 9 | \. 10 | (?P\d+) 11 | \. 12 | (?P\d+) 13 | (?P
((a|b|rc)\d+))?
14 |   (\.
15 |     (?Pdev\d*)
16 |   )?
17 |   ''',
18 |     re.VERBOSE,
19 | )
20 | 
21 | _version_fields = _version_regex.match(__version__).groupdict()
22 | version_info = tuple(
23 |     field
24 |     for field in (
25 |         int(_version_fields["major"]),
26 |         int(_version_fields["minor"]),
27 |         int(_version_fields["patch"]),
28 |         _version_fields["pre"],
29 |         _version_fields["dev"],
30 |     )
31 |     if field is not None
32 | )
33 | 
34 | __all__ = ["__version__", "version_info"]
35 | 


--------------------------------------------------------------------------------
/ipyparallel/apps/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/ipyparallel/apps/__init__.py


--------------------------------------------------------------------------------
/ipyparallel/apps/ipclusterapp.py:
--------------------------------------------------------------------------------
 1 | import warnings
 2 | 
 3 | warnings.warn(f"{__name__} is deprecated in ipyparallel 7. Use ipyparallel.cluster")
 4 | 
 5 | from ipyparallel.cluster.app import IPCluster, IPClusterStart, main  # noqa
 6 | 
 7 | IPClusterApp = IPCluster
 8 | launch_new_instance = main
 9 | 
10 | if __name__ == "__main__":
11 |     main()
12 | 


--------------------------------------------------------------------------------
/ipyparallel/apps/ipcontrollerapp.py:
--------------------------------------------------------------------------------
 1 | import warnings
 2 | 
 3 | warnings.warn(f"{__name__} is deprecated in ipyparallel 7. Use ipyparallel.controller")
 4 | 
 5 | from ipyparallel.controller.app import IPController, main
 6 | 
 7 | IPControllerApp = IPController
 8 | launch_new_instance = main
 9 | 
10 | if __name__ == "__main__":
11 |     main()
12 | 


--------------------------------------------------------------------------------
/ipyparallel/apps/ipengineapp.py:
--------------------------------------------------------------------------------
 1 | import warnings
 2 | 
 3 | warnings.warn(f"{__name__} is deprecated in ipyparallel 7. Use ipyparallel.engine")
 4 | 
 5 | from ipyparallel.engine.app import IPEngine, main
 6 | 
 7 | IPEngineApp = IPEngine
 8 | launch_new_instance = main
 9 | 
10 | if __name__ == "__main__":
11 |     main()
12 | 


--------------------------------------------------------------------------------
/ipyparallel/apps/iploggerapp.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | """
 3 | A simple IPython logger application
 4 | """
 5 | 
 6 | from IPython.core.profiledir import ProfileDir
 7 | from traitlets import Dict
 8 | 
 9 | from ipyparallel.apps.baseapp import (
10 |     BaseParallelApplication,
11 |     base_aliases,
12 |     catch_config_error,
13 | )
14 | from ipyparallel.apps.logwatcher import LogWatcher
15 | 
16 | # -----------------------------------------------------------------------------
17 | # Module level variables
18 | # -----------------------------------------------------------------------------
19 | 
20 | #: The default config file name for this application
21 | _description = """Start an IPython logger for parallel computing.
22 | 
23 | IPython controllers and engines (and your own processes) can broadcast log messages
24 | by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
25 | logger can be configured using command line options or using a cluster
26 | directory. Cluster directories contain config, log and security files and are
27 | usually located in your ipython directory and named as "profile_name".
28 | See the `profile` and `profile-dir` options for details.
29 | """
30 | 
31 | 
32 | # -----------------------------------------------------------------------------
33 | # Main application
34 | # -----------------------------------------------------------------------------
35 | aliases = {}
36 | aliases.update(base_aliases)
37 | aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics'))
38 | 
39 | 
40 | class IPLoggerApp(BaseParallelApplication):
41 |     name = 'iplogger'
42 |     description = _description
43 |     classes = [LogWatcher, ProfileDir]
44 |     aliases = Dict(aliases)
45 | 
46 |     @catch_config_error
47 |     def initialize(self, argv=None):
48 |         super().initialize(argv)
49 |         self.init_watcher()
50 | 
51 |     def init_watcher(self):
52 |         try:
53 |             self.watcher = LogWatcher(parent=self, log=self.log)
54 |         except BaseException:
55 |             self.log.error("Couldn't start the LogWatcher", exc_info=True)
56 |             self.exit(1)
57 |         self.log.info(f"Listening for log messages on {self.watcher.url!r}")
58 | 
59 |     def start(self):
60 |         self.watcher.start()
61 |         try:
62 |             self.watcher.loop.start()
63 |         except KeyboardInterrupt:
64 |             self.log.critical("Logging Interrupted, shutting down...\n")
65 | 
66 | 
67 | launch_new_instance = IPLoggerApp.launch_instance
68 | 
69 | 
70 | if __name__ == '__main__':
71 |     launch_new_instance()
72 | 


--------------------------------------------------------------------------------
/ipyparallel/apps/launcher.py:
--------------------------------------------------------------------------------
 1 | """Deprecated import for ipyparallel.cluster.launcher"""
 2 | 
 3 | import warnings
 4 | 
 5 | from ipyparallel.cluster.launcher import *  # noqa
 6 | 
 7 | warnings.warn(
 8 |     f"{__name__} is deprecated in ipyparallel 7. Use ipyparallel.cluster.launcher.",
 9 |     DeprecationWarning,
10 | )
11 | 


--------------------------------------------------------------------------------
/ipyparallel/apps/logwatcher.py:
--------------------------------------------------------------------------------
 1 | """
 2 | A logger object that consolidates messages incoming from ipcluster processes.
 3 | """
 4 | 
 5 | import logging
 6 | 
 7 | import zmq
 8 | from jupyter_client.localinterfaces import localhost
 9 | from tornado import ioloop
10 | from traitlets import Instance, List, Unicode
11 | from traitlets.config.configurable import LoggingConfigurable
12 | from zmq.eventloop import zmqstream
13 | 
14 | 
15 | class LogWatcher(LoggingConfigurable):
16 |     """A simple class that receives messages on a SUB socket, as published
17 |     by subclasses of `zmq.log.handlers.PUBHandler`, and logs them itself.
18 | 
19 |     This can subscribe to multiple topics, but defaults to all topics.
20 |     """
21 | 
22 |     # configurables
23 |     topics = List(
24 |         [''],
25 |         config=True,
26 |         help="The ZMQ topics to subscribe to. Default is to subscribe to all messages",
27 |     )
28 |     url = Unicode(config=True, help="ZMQ url on which to listen for log messages")
29 | 
30 |     def _url_default(self):
31 |         return f'tcp://{localhost()}:20202'
32 | 
33 |     # internals
34 |     stream = Instance('zmq.eventloop.zmqstream.ZMQStream', allow_none=True)
35 | 
36 |     context = Instance(zmq.Context)
37 | 
38 |     def _context_default(self):
39 |         return zmq.Context.instance()
40 | 
41 |     loop = Instance('tornado.ioloop.IOLoop')
42 | 
43 |     def _loop_default(self):
44 |         return ioloop.IOLoop.current()
45 | 
46 |     def __init__(self, **kwargs):
47 |         super().__init__(**kwargs)
48 |         s = self.context.socket(zmq.SUB)
49 |         s.bind(self.url)
50 |         self.stream = zmqstream.ZMQStream(s, self.loop)
51 |         self.subscribe()
52 |         self.on_trait_change(self.subscribe, 'topics')
53 | 
54 |     def start(self):
55 |         self.stream.on_recv(self.log_message)
56 | 
57 |     def stop(self):
58 |         self.stream.stop_on_recv()
59 | 
60 |     def subscribe(self):
61 |         """Update our SUB socket's subscriptions."""
62 |         self.stream.setsockopt(zmq.UNSUBSCRIBE, '')
63 |         if '' in self.topics:
64 |             self.log.debug("Subscribing to: everything")
65 |             self.stream.setsockopt(zmq.SUBSCRIBE, '')
66 |         else:
67 |             for topic in self.topics:
68 |                 self.log.debug(f"Subscribing to: {topic!r}")
69 |                 self.stream.setsockopt(zmq.SUBSCRIBE, topic)
70 | 
71 |     def _extract_level(self, topic_str):
72 |         """Turn 'engine.0.INFO.extra' into (logging.INFO, 'engine.0.extra')"""
73 |         topics = topic_str.split('.')
74 |         for idx, t in enumerate(topics):
75 |             level = getattr(logging, t, None)
76 |             if level is not None:
77 |                 break
78 | 
79 |         if level is None:
80 |             level = logging.INFO
81 |         else:
82 |             topics.pop(idx)
83 | 
84 |         return level, '.'.join(topics)
85 | 
86 |     def log_message(self, raw):
87 |         """receive and parse a message, then log it."""
88 |         if len(raw) != 2 or '.' not in raw[0]:
89 |             self.log.error(f"Invalid log message: {raw}")
90 |             return
91 |         else:
92 |             topic, msg = raw
93 |             # don't newline, since log messages always newline:
94 |             topic, level_name = topic.rsplit('.', 1)
95 |             level, topic = self._extract_level(topic)
96 |             if msg[-1] == '\n':
97 |                 msg = msg[:-1]
98 |             self.log.log(level, f"[{topic}] {msg}")
99 | 


--------------------------------------------------------------------------------
/ipyparallel/client/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/ipyparallel/client/__init__.py


--------------------------------------------------------------------------------
/ipyparallel/client/_joblib.py:
--------------------------------------------------------------------------------
 1 | """joblib parallel backend for IPython Parallel"""
 2 | 
 3 | # Copyright (c) IPython Development Team.
 4 | # Distributed under the terms of the Modified BSD License.
 5 | from joblib.parallel import AutoBatchingMixin, ParallelBackendBase
 6 | 
 7 | import ipyparallel as ipp
 8 | 
 9 | 
10 | class IPythonParallelBackend(AutoBatchingMixin, ParallelBackendBase):
11 |     def __init__(self, view=None, **kwargs):
12 |         super().__init__(**kwargs)
13 |         self._cluster_owner = False
14 |         self._client_owner = False
15 |         if view is None:
16 |             self._client_owner = True
17 |             try:
18 |                 # load the default cluster
19 |                 cluster = ipp.Cluster.from_file()
20 |             except FileNotFoundError:
21 |                 # other load errors?
22 |                 cluster = self._cluster = ipp.Cluster()
23 |                 self._cluster_owner = True
24 |                 cluster.start_cluster_sync()
25 |             else:
26 |                 # cluster running, ensure some engines are, too
27 |                 if not cluster.engines:
28 |                     cluster.start_engines_sync()
29 |             rc = cluster.connect_client_sync()
30 |             rc.wait_for_engines(cluster.n or 1)
31 |             view = rc.load_balanced_view()
32 | 
33 |             # use cloudpickle or dill for closures, if available.
34 |             # joblib tends to create closures default pickle can't handle.
35 |             try:
36 |                 import cloudpickle  # noqa
37 |             except ImportError:
38 |                 try:
39 |                     import dill  # noqa
40 |                 except ImportError:
41 |                     pass
42 |                 else:
43 |                     view.client[:].use_dill()
44 |             else:
45 |                 view.client[:].use_cloudpickle()
46 |         self._view = view
47 | 
48 |     def effective_n_jobs(self, n_jobs):
49 |         """A View can run len(view) jobs at a time"""
50 |         return len(self._view)
51 | 
52 |     def terminate(self):
53 |         """Close the client if we created it"""
54 |         if self._client_owner:
55 |             self._view.client.close()
56 |         if self._cluster_owner:
57 |             self._cluster.stop_cluster_sync()
58 | 
59 |     def apply_async(self, func, callback=None):
60 |         """Schedule a func to be run"""
61 |         future = self._view.apply_async(func)
62 |         if callback:
63 |             future.add_done_callback(lambda f: callback(f.result()))
64 |         return future
65 | 


--------------------------------------------------------------------------------
/ipyparallel/client/futures.py:
--------------------------------------------------------------------------------
  1 | """Future-related utils"""
  2 | 
  3 | # Copyright (c) IPython Development Team.
  4 | # Distributed under the terms of the Modified BSD License.
  5 | import sys
  6 | from concurrent.futures import Future
  7 | from threading import Event
  8 | 
  9 | from tornado.log import app_log
 10 | 
 11 | 
 12 | class MessageFuture(Future):
 13 |     """Future class to wrap async messages"""
 14 | 
 15 |     def __init__(self, msg_id, header=None, *, track=False):
 16 |         super().__init__()
 17 |         self.msg_id = msg_id
 18 |         self.header = header or {"msg_type": "unknown_request"}
 19 |         self._evt = Event()
 20 |         self.track = track
 21 |         self._tracker = None
 22 |         self.tracker = Future()
 23 |         self.iopub_callbacks = []
 24 |         if not track:
 25 |             self.tracker.set_result(None)
 26 |         self.add_done_callback(lambda f: self._evt.set())
 27 | 
 28 |     def wait(self, timeout=None):
 29 |         if not self.done():
 30 |             return self._evt.wait(timeout)
 31 |         return True
 32 | 
 33 | 
 34 | # The following are from tornado 5.0b1
 35 | # avoids hang using gen.multi_future on asyncio,
 36 | # because Futures cannot be created in another thread
 37 | 
 38 | 
 39 | def future_set_result_unless_cancelled(future, value):
 40 |     """Set the given ``value`` as the `Future`'s result, if not cancelled.
 41 | 
 42 |     Avoids asyncio.InvalidStateError when calling set_result() on
 43 |     a cancelled `asyncio.Future`.
 44 | 
 45 |     .. versionadded:: 5.0
 46 |     """
 47 |     if not future.cancelled():
 48 |         future.set_result(value)
 49 | 
 50 | 
 51 | def future_set_exc_info(future, exc_info):
 52 |     """Set the given ``exc_info`` as the `Future`'s exception.
 53 | 
 54 |     Understands both `asyncio.Future` and Tornado's extensions to
 55 |     enable better tracebacks on Python 2.
 56 | 
 57 |     .. versionadded:: 5.0
 58 |     """
 59 |     if hasattr(future, 'set_exc_info'):
 60 |         # Tornado's Future
 61 |         future.set_exc_info(exc_info)
 62 |     else:
 63 |         # asyncio.Future
 64 |         future.set_exception(exc_info[1])
 65 | 
 66 | 
 67 | def future_add_done_callback(future, callback):
 68 |     """Arrange to call ``callback`` when ``future`` is complete.
 69 | 
 70 |     ``callback`` is invoked with one argument, the ``future``.
 71 | 
 72 |     If ``future`` is already done, ``callback`` is invoked immediately.
 73 |     This may differ from the behavior of ``Future.add_done_callback``,
 74 |     which makes no such guarantee.
 75 | 
 76 |     .. versionadded:: 5.0
 77 |     """
 78 |     if future.done():
 79 |         callback(future)
 80 |     else:
 81 |         future.add_done_callback(callback)
 82 | 
 83 | 
 84 | def multi_future(children):
 85 |     """Wait for multiple asynchronous futures in parallel.
 86 | 
 87 |     This function is similar to `multi`, but does not support
 88 |     `YieldPoints `.
 89 | 
 90 |     .. versionadded:: 4.0
 91 |     """
 92 |     unfinished_children = set(children)
 93 | 
 94 |     future = Future()
 95 |     if not children:
 96 |         future_set_result_unless_cancelled(future, [])
 97 | 
 98 |     def callback(f):
 99 |         unfinished_children.remove(f)
100 |         if not unfinished_children:
101 |             result_list = []
102 |             for f in children:
103 |                 try:
104 |                     result_list.append(f.result())
105 |                 except Exception as e:
106 |                     if future.done():
107 |                         app_log.error(
108 |                             "Multiple exceptions in yield list", exc_info=True
109 |                         )
110 |                     else:
111 |                         future_set_exc_info(future, sys.exc_info())
112 |             if not future.done():
113 |                 future_set_result_unless_cancelled(future, result_list)
114 | 
115 |     listening = set()
116 |     for f in children:
117 |         if f not in listening:
118 |             listening.add(f)
119 |             future_add_done_callback(f, callback)
120 |     return future
121 | 


--------------------------------------------------------------------------------
/ipyparallel/client/map.py:
--------------------------------------------------------------------------------
  1 | """Classes used in scattering and gathering sequences.
  2 | 
  3 | Scattering consists of partitioning a sequence and sending the various
  4 | pieces to individual nodes in a cluster.
  5 | """
  6 | 
  7 | # Copyright (c) IPython Development Team.
  8 | # Distributed under the terms of the Modified BSD License.
  9 | import sys
 10 | from itertools import chain, islice
 11 | 
 12 | numpy = None
 13 | 
 14 | 
 15 | def is_array(obj):
 16 |     """Is an object a numpy array?
 17 | 
 18 |     Avoids importing numpy until it is requested
 19 |     """
 20 |     global numpy
 21 |     if 'numpy' not in sys.modules:
 22 |         return False
 23 | 
 24 |     if numpy is None:
 25 |         import numpy
 26 |     return isinstance(obj, numpy.ndarray)
 27 | 
 28 | 
 29 | class Map:
 30 |     """A class for partitioning a sequence using a map."""
 31 | 
 32 |     def getPartition(self, seq, p, q, n=None):
 33 |         """Returns the pth partition of q partitions of seq.
 34 | 
 35 |         The length can be specified as `n`,
 36 |         otherwise it is the value of `len(seq)`
 37 |         """
 38 |         n = len(seq) if n is None else n
 39 |         # Test for error conditions here
 40 |         if p < 0 or p >= q:
 41 |             raise ValueError(f"must have 0 <= p <= q, but have p={p},q={q}")
 42 | 
 43 |         remainder = n % q
 44 |         basesize = n // q
 45 | 
 46 |         if p < remainder:
 47 |             low = p * (basesize + 1)
 48 |             high = low + basesize + 1
 49 |         else:
 50 |             low = p * basesize + remainder
 51 |             high = low + basesize
 52 | 
 53 |         try:
 54 |             result = seq[low:high]
 55 |         except TypeError:
 56 |             # some objects (iterators) can't be sliced,
 57 |             # use islice:
 58 |             result = list(islice(seq, low, high))
 59 | 
 60 |         return result
 61 | 
 62 |     def joinPartitions(self, listOfPartitions):
 63 |         return self.concatenate(listOfPartitions)
 64 | 
 65 |     def concatenate(self, listOfPartitions):
 66 |         testObject = listOfPartitions[0]
 67 |         # First see if we have a known array type
 68 |         if is_array(testObject):
 69 |             return numpy.concatenate(listOfPartitions)
 70 |         # Next try for Python sequence types
 71 |         if isinstance(testObject, (list, tuple)):
 72 |             return list(chain.from_iterable(listOfPartitions))
 73 |         # If we have scalars, just return listOfPartitions
 74 |         return listOfPartitions
 75 | 
 76 | 
 77 | class RoundRobinMap(Map):
 78 |     """Partitions a sequence in a round robin fashion.
 79 | 
 80 |     This currently does not work!
 81 |     """
 82 | 
 83 |     def getPartition(self, seq, p, q, n=None):
 84 |         n = len(seq) if n is None else n
 85 |         return seq[p:n:q]
 86 | 
 87 |     def joinPartitions(self, listOfPartitions):
 88 |         testObject = listOfPartitions[0]
 89 |         # First see if we have a known array type
 90 |         if is_array(testObject):
 91 |             return self.flatten_array(listOfPartitions)
 92 |         if isinstance(testObject, (list, tuple)):
 93 |             return self.flatten_list(listOfPartitions)
 94 |         return listOfPartitions
 95 | 
 96 |     def flatten_array(self, listOfPartitions):
 97 |         test = listOfPartitions[0]
 98 |         shape = list(test.shape)
 99 |         shape[0] = sum(p.shape[0] for p in listOfPartitions)
100 |         A = numpy.ndarray(shape)
101 |         N = shape[0]
102 |         q = len(listOfPartitions)
103 |         for p, part in enumerate(listOfPartitions):
104 |             A[p:N:q] = part
105 |         return A
106 | 
107 |     def flatten_list(self, listOfPartitions):
108 |         flat = []
109 |         for i in range(len(listOfPartitions[0])):
110 |             flat.extend([part[i] for part in listOfPartitions if len(part) > i])
111 |         return flat
112 | 
113 | 
114 | def mappable(obj):
115 |     """return whether an object is mappable or not."""
116 |     if isinstance(obj, (tuple, list)):
117 |         return True
118 |     if is_array(obj):
119 |         return True
120 |     return False
121 | 
122 | 
123 | dists = {'b': Map, 'r': RoundRobinMap}
124 | 


--------------------------------------------------------------------------------
/ipyparallel/cluster/__init__.py:
--------------------------------------------------------------------------------
1 | from .cluster import *  # noqa
2 | 


--------------------------------------------------------------------------------
/ipyparallel/cluster/__main__.py:
--------------------------------------------------------------------------------
1 | if __name__ == '__main__':
2 |     from .app import main
3 | 
4 |     main()
5 | 


--------------------------------------------------------------------------------
/ipyparallel/controller/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/ipyparallel/controller/__init__.py


--------------------------------------------------------------------------------
/ipyparallel/controller/__main__.py:
--------------------------------------------------------------------------------
1 | if __name__ == '__main__':
2 |     from .app import main
3 | 
4 |     main()
5 | 


--------------------------------------------------------------------------------
/ipyparallel/datapub.py:
--------------------------------------------------------------------------------
1 | from .engine.datapub import publish_data  # noqa: F401
2 | 


--------------------------------------------------------------------------------
/ipyparallel/engine/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ipython/ipyparallel/40de02a317c5822d9eb68f9d8b792c958570ee9f/ipyparallel/engine/__init__.py


--------------------------------------------------------------------------------
/ipyparallel/engine/__main__.py:
--------------------------------------------------------------------------------
1 | if __name__ == '__main__':
2 |     from .app import main
3 | 
4 |     main()
5 | 


--------------------------------------------------------------------------------
/ipyparallel/engine/datapub.py:
--------------------------------------------------------------------------------
 1 | """Publishing native (typically pickled) objects."""
 2 | 
 3 | # Copyright (c) IPython Development Team.
 4 | # Distributed under the terms of the Modified BSD License.
 5 | from ipykernel.jsonutil import json_clean
 6 | from jupyter_client.session import Session, extract_header
 7 | from traitlets import Any, CBytes, Dict, Instance
 8 | from traitlets.config import Configurable
 9 | 
10 | from ipyparallel.serialize import serialize_object
11 | 
12 | 
13 | class ZMQDataPublisher(Configurable):
14 |     topic = topic = CBytes(b'datapub')
15 |     session = Instance(Session, allow_none=True)
16 |     pub_socket = Any(allow_none=True)
17 |     parent_header = Dict({})
18 | 
19 |     def set_parent(self, parent):
20 |         """Set the parent for outbound messages."""
21 |         self.parent_header = extract_header(parent)
22 | 
23 |     def publish_data(self, data):
24 |         """publish a data_message on the IOPub channel
25 | 
26 |         Parameters
27 |         ----------
28 |         data : dict
29 |             The data to be published. Think of it as a namespace.
30 |         """
31 |         session = self.session
32 |         buffers = serialize_object(
33 |             data,
34 |             buffer_threshold=session.buffer_threshold,
35 |             item_threshold=session.item_threshold,
36 |         )
37 |         content = json_clean(dict(keys=list(data.keys())))
38 |         session.send(
39 |             self.pub_socket,
40 |             'data_message',
41 |             content=content,
42 |             parent=self.parent_header,
43 |             buffers=buffers,
44 |             ident=self.topic,
45 |         )
46 | 
47 | 
48 | def publish_data(data):
49 |     """publish a data_message on the IOPub channel
50 | 
51 |     Parameters
52 |     ----------
53 |     data : dict
54 |         The data to be published. Think of it as a namespace.
55 |     """
56 |     from ipykernel.zmqshell import ZMQInteractiveShell
57 | 
58 |     ZMQInteractiveShell.instance().data_pub.publish_data(data)
59 | 


--------------------------------------------------------------------------------
/ipyparallel/engine/log.py:
--------------------------------------------------------------------------------
 1 | from zmq.log.handlers import PUBHandler
 2 | 
 3 | 
 4 | class EnginePUBHandler(PUBHandler):
 5 |     """A simple PUBHandler subclass that sets root_topic"""
 6 | 
 7 |     engine = None
 8 | 
 9 |     def __init__(self, engine, *args, **kwargs):
10 |         PUBHandler.__init__(self, *args, **kwargs)
11 |         self.engine = engine
12 | 
13 |     @property
14 |     def root_topic(self):
15 |         """this is a property, in case the handler is created
16 |         before the engine gets registered with an id"""
17 |         if isinstance(getattr(self.engine, 'id', None), int):
18 |             return f"engine.{self.engine.id}"
19 |         else:
20 |             return "engine"
21 | 


--------------------------------------------------------------------------------
/ipyparallel/joblib.py:
--------------------------------------------------------------------------------
 1 | """IPython parallel backend for joblib
 2 | 
 3 | To enable the default view as a backend for joblib::
 4 | 
 5 |     import ipyparallel as ipp
 6 |     ipp.register_joblib_backend()
 7 | 
 8 | Or to enable a particular View you have already set up::
 9 | 
10 |     view.register_joblib_backend()
11 | 
12 | At this point, you can use it with::
13 | 
14 |     with parallel_backend('ipyparallel'):
15 |         Parallel(n_jobs=2)(delayed(some_function)(i) for i in range(10))
16 | 
17 | .. versionadded:: 5.1
18 | """
19 | 
20 | # Copyright (c) IPython Development Team.
21 | # Distributed under the terms of the Modified BSD License.
22 | from joblib.parallel import register_parallel_backend
23 | 
24 | from .client._joblib import IPythonParallelBackend
25 | 
26 | 
27 | def register(name='ipyparallel', make_default=False):
28 |     """Register the default ipyparallel Client as a joblib backend
29 | 
30 |     See joblib.parallel.register_parallel_backend for details.
31 |     """
32 |     return register_parallel_backend(
33 |         name, IPythonParallelBackend, make_default=make_default
34 |     )
35 | 


--------------------------------------------------------------------------------
/ipyparallel/logger.py:
--------------------------------------------------------------------------------
1 | if __name__ == '__main__':
2 |     from ipyparallel.apps import iploggerapp as app
3 | 
4 |     app.launch_new_instance()
5 | 


--------------------------------------------------------------------------------
/ipyparallel/nbextension/__init__.py:
--------------------------------------------------------------------------------
 1 | """Jupyter server extension(s)"""
 2 | 
 3 | import warnings
 4 | 
 5 | 
 6 | def load_jupyter_server_extension(app):
 7 |     warnings.warn(
 8 |         "Using ipyparallel.nbextension as server extension is deprecated in IPython Parallel 7.0. Use top-level ipyparallel.",
 9 |         DeprecationWarning,
10 |     )
11 |     from ipyparallel import _load_jupyter_server_extension
12 | 
13 |     return _load_jupyter_server_extension(app)
14 | 


--------------------------------------------------------------------------------
/ipyparallel/nbextension/base.py:
--------------------------------------------------------------------------------
 1 | """Place to put the base Handler"""
 2 | 
 3 | import warnings
 4 | from functools import lru_cache
 5 | 
 6 | _APIHandler = None
 7 | 
 8 | 
 9 | @lru_cache
10 | def _guess_api_handler():
11 |     """Fallback to guess API handler by availability"""
12 |     try:
13 |         from notebook.base.handlers import APIHandler
14 |     except ImportError:
15 |         from jupyter_server.base.handlers import APIHandler
16 |     global _APIHandler
17 |     _APIHandler = APIHandler
18 |     return _APIHandler
19 | 
20 | 
21 | def get_api_handler(app=None):
22 |     """Get the base APIHandler class to use
23 | 
24 |     Inferred from app class (either jupyter_server or notebook app)
25 |     """
26 |     global _APIHandler
27 |     if _APIHandler is not None:
28 |         return _APIHandler
29 |     if app is None:
30 |         warnings.warn(
31 |             "Guessing base APIHandler class. Specify an app to ensure the right APIHandler is used.",
32 |             stacklevel=2,
33 |         )
34 |         return _guess_api_handler()
35 | 
36 |     top_modules = {cls.__module__.split(".", 1)[0] for cls in app.__class__.mro()}
37 |     if "jupyter_server" in top_modules:
38 |         from jupyter_server.base.handlers import APIHandler
39 | 
40 |         _APIHandler = APIHandler
41 |         return APIHandler
42 |     if "notebook" in top_modules:
43 |         from notebook.base.handlers import APIHandler
44 | 
45 |         _APIHandler = APIHandler
46 |         return APIHandler
47 | 
48 |     warnings.warn(f"Failed to detect base APIHandler class for {app}.", stacklevel=2)
49 |     return _guess_api_handler()
50 | 


--------------------------------------------------------------------------------
/ipyparallel/nbextension/install.py:
--------------------------------------------------------------------------------
 1 | """Install the IPython clusters tab in the Jupyter notebook dashboard
 2 | 
 3 | Only applicable for notebook < 7
 4 | """
 5 | 
 6 | # Copyright (c) IPython Development Team.
 7 | # Distributed under the terms of the Modified BSD License.
 8 | 
 9 | 
10 | def install_extensions(enable=True, user=False):
11 |     """Register ipyparallel clusters tab as notebook extensions
12 | 
13 |     Toggle with enable=True/False.
14 |     """
15 |     import notebook  # noqa
16 | 
17 |     from notebook.nbextensions import (
18 |         disable_nbextension,
19 |         enable_nbextension,
20 |         install_nbextension_python,
21 |     )
22 |     from notebook.serverextensions import toggle_serverextension_python
23 | 
24 |     toggle_serverextension_python('ipyparallel', user=user)
25 |     install_nbextension_python('ipyparallel', user=user)
26 |     if enable:
27 |         enable_nbextension('tree', 'ipyparallel/main', user=user)
28 |     else:
29 |         disable_nbextension('tree', 'ipyparallel/main')
30 | 
31 | 
32 | install_server_extension = install_extensions
33 | 


--------------------------------------------------------------------------------
/ipyparallel/nbextension/static/clusterlist.css:
--------------------------------------------------------------------------------
1 | .action_col {
2 |   text-align: right;
3 | }
4 | 
5 | input.engine_num_input {
6 |   width: 60px;
7 | }
8 | 


--------------------------------------------------------------------------------
/ipyparallel/nbextension/static/main.js:
--------------------------------------------------------------------------------
 1 | // Copyright (c) IPython Development Team.
 2 | // Distributed under the terms of the Modified BSD License.
 3 | 
 4 | define(function (require) {
 5 |   var $ = require("jquery");
 6 |   var IPython = require("base/js/namespace");
 7 |   var clusterlist = require("./clusterlist");
 8 | 
 9 |   var cluster_html = $(
10 |     [
11 |       '
',
12 |       '  
',
13 |       '    
',
14 |       '      IPython parallel computing clusters',
15 |       "    
",
16 |       '    
',
17 |       '      ',
18 |       '      ',
19 |       "      ",
20 |       "    
",
21 |       "  
",
22 |       '  
',
23 |       '    
',
24 |       '      
profile
',
25 |       '      
cluster id
',
26 |       '      
status
',
27 |       '      
# of engines
',
28 |       '      
action
',
29 |       "    
",
30 |       "  
",
31 |       "
",
32 |     ].join("\n"),
33 |   );
34 | 
35 |   function load() {
36 |     if (!IPython.notebook_list) return;
37 |     var base_url = IPython.notebook_list.base_url;
38 |     // hide the deprecated clusters tab
39 |     $("#tabs").find('[href="#clusters"]').hide();
40 |     $("head").append(
41 |       $("")
42 |         .attr("rel", "stylesheet")
43 |         .attr("type", "text/css")
44 |         .attr("href", base_url + "nbextensions/ipyparallel/clusterlist.css"),
45 |     );
46 |     $(".tab-content").append(cluster_html);
47 |     $("#tabs").append(
48 |       $("
  • ").append(
49 |         $("")
50 |           .attr("href", "#ipyclusters")
51 |           .attr("data-toggle", "tab")
52 |           .text("IPython Clusters")
53 |           .click(function (e) {
54 |             window.history.pushState(null, null, "#ipyclusters");
55 |           }),
56 |       ),
57 |     );
58 |     var cluster_list = new clusterlist.ClusterList("#cluster_list", {
59 |       base_url: IPython.notebook_list.base_url,
60 |       notebook_path: IPython.notebook_list.notebook_path,
61 |     });
62 |     cluster_list.load_list();
63 |     if (document.location.hash === "#ipyclusters") {
64 |       // focus clusters tab once it exists
65 |       // since it won't have been created when the browser first handles the document hash
66 |       $("[href='#ipyclusters']").tab("show");
67 |     }
68 |   }
69 |   return {
70 |     load_ipython_extension: load,
71 |   };
72 | });
73 | 


--------------------------------------------------------------------------------
/ipyparallel/serialize/__init__.py:
--------------------------------------------------------------------------------
 1 | from .canning import (
 2 |     Reference,
 3 |     can,
 4 |     can_map,
 5 |     uncan,
 6 |     uncan_map,
 7 |     use_cloudpickle,
 8 |     use_dill,
 9 |     use_pickle,
10 | )
11 | from .serialize import (
12 |     PrePickled,
13 |     deserialize_object,
14 |     pack_apply_message,
15 |     serialize_object,
16 |     unpack_apply_message,
17 | )
18 | 
19 | __all__ = (
20 |     'Reference',
21 |     'PrePickled',
22 |     'can_map',
23 |     'uncan_map',
24 |     'can',
25 |     'uncan',
26 |     'use_dill',
27 |     'use_cloudpickle',
28 |     'use_pickle',
29 |     'serialize_object',
30 |     'deserialize_object',
31 |     'pack_apply_message',
32 |     'unpack_apply_message',
33 | )
34 | 


--------------------------------------------------------------------------------
/ipyparallel/serialize/codeutil.py:
--------------------------------------------------------------------------------
 1 | """Utilities to enable code objects to be pickled.
 2 | 
 3 | Any process that import this module will be able to pickle code objects.  This
 4 | includes the func_code attribute of any function.  Once unpickled, new
 5 | functions can be built using new.function(code, globals()).  Eventually
 6 | we need to automate all of this so that functions themselves can be pickled.
 7 | 
 8 | Reference: A. Tremols, P Cogolo, "Python Cookbook," p 302-305
 9 | """
10 | 
11 | # Copyright (c) IPython Development Team.
12 | # Distributed under the terms of the Modified BSD License.
13 | import copyreg
14 | import inspect
15 | import sys
16 | import types
17 | 
18 | 
19 | def code_ctor(*args):
20 |     return types.CodeType(*args)
21 | 
22 | 
23 | # map CodeType constructor args to code co_ attribute names
24 | # (they _almost_ all match, and new ones probably will)
25 | _code_attr_map = {
26 |     "codestring": "code",
27 |     "constants": "consts",
28 | }
29 | # pass every supported arg to the code constructor
30 | # this should be more forward-compatible
31 | # (broken on pypy: https://github.com/ipython/ipyparallel/issues/845)
32 | if sys.version_info >= (3, 10) and getattr(sys, "pypy_version_info", (7, 3, 19)) >= (
33 |     7,
34 |     3,
35 |     19,
36 | ):
37 |     _code_attr_names = tuple(
38 |         _code_attr_map.get(name, name)
39 |         for name, param in inspect.signature(types.CodeType).parameters.items()
40 |         if (param.POSITIONAL_ONLY or param.POSITIONAL_OR_KEYWORD)
41 |         and not (hasattr(sys, "pypy_version_info") and name == "magic")
42 |     )
43 | else:
44 |     # can't inspect types.CodeType on Python < 3.10
45 |     _code_attr_names = [
46 |         "argcount",
47 |         "kwonlyargcount",
48 |         "nlocals",
49 |         "stacksize",
50 |         "flags",
51 |         "code",
52 |         "consts",
53 |         "names",
54 |         "varnames",
55 |         "filename",
56 |         "name",
57 |         "firstlineno",
58 |         "lnotab",
59 |         "freevars",
60 |         "cellvars",
61 |     ]
62 |     if hasattr(types.CodeType, "co_posonlyargcount"):
63 |         _code_attr_names.insert(1, "posonlyargcount")
64 | 
65 |     _code_attr_names = tuple(_code_attr_names)
66 | 
67 | 
68 | def reduce_code(obj):
69 |     """codeobject reducer"""
70 |     return code_ctor, tuple(getattr(obj, f'co_{name}') for name in _code_attr_names)
71 | 
72 | 
73 | copyreg.pickle(types.CodeType, reduce_code)
74 | 


--------------------------------------------------------------------------------
/ipyparallel/shellcmd.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | """
 3 | Commandline interface to OS independent shell commands
 4 | 
 5 | Currently the following command are supported:
 6 | * start:   starts a process into background and returns its pid
 7 | * running: check if a process with a given pid is running
 8 | * kill:    kill a process with a given pid
 9 | * mkdir:   creates directory recursively (no error if directory already exists)
10 | * rmdir:   remove directory recursively
11 | * exists:  checks if a file/directory exists
12 | * remove:  removes a file
13 | """
14 | 
15 | import os
16 | import sys
17 | from argparse import ArgumentParser
18 | 
19 | from .cluster.shellcmd import ShellCommandReceive
20 | 
21 | 
22 | def main():
23 |     parser = ArgumentParser(
24 |         description='Perform some standard shell command in a platform independent way'
25 |     )
26 |     parser.add_argument(
27 |         '--debug',
28 |         action='store_true',
29 |         help='append command line parameters to \'sys_arg.txt\'',
30 |     )
31 |     subparsers = parser.add_subparsers(dest='cmd', help='sub-command help')
32 | 
33 |     # create the parser for the "a" command
34 |     parser_start = subparsers.add_parser(
35 |         'start', help='start a process into background'
36 |     )
37 |     parser_start.add_argument('--env', help='optional environment dictionary')
38 |     parser_start.add_argument(
39 |         '--output_file', help='optional output redirection (for stdout and stderr)'
40 |     )
41 |     parser_start.add_argument('start_cmd', nargs='+', help='command that help')
42 | 
43 |     parser_running = subparsers.add_parser(
44 |         'running', help='check if a process is running'
45 |     )
46 |     parser_running.add_argument(
47 |         'pid', type=int, help='pid of process that should be checked'
48 |     )
49 | 
50 |     parser_kill = subparsers.add_parser('kill', help='kill a process')
51 |     parser_kill.add_argument(
52 |         'pid', type=int, help='pid of process that should be killed'
53 |     )
54 |     parser_kill.add_argument('--sig', type=int, help='signals to send')
55 | 
56 |     parser_mkdir = subparsers.add_parser('mkdir', help='create directory recursively')
57 |     parser_mkdir.add_argument('path', help='directory path to be created')
58 | 
59 |     parser_rmdir = subparsers.add_parser('rmdir', help='remove directory recursively')
60 |     parser_rmdir.add_argument('path', help='directory path to be removed')
61 | 
62 |     parser_exists = subparsers.add_parser(
63 |         'exists', help='checks if a file/directory exists'
64 |     )
65 |     parser_exists.add_argument('path', help='path to check')
66 | 
67 |     parser_remove = subparsers.add_parser('remove', help='removes a file')
68 |     parser_remove.add_argument('path', help='path to remove')
69 | 
70 |     if len(sys.argv) == 1:
71 |         parser.print_help()
72 |         sys.exit(0)
73 | 
74 |     args = parser.parse_args()
75 |     cmd = args.__dict__.pop('cmd')
76 | 
77 |     if args.debug:
78 |         with open("sys_arg.txt", "a") as f:
79 |             f.write(f"'{__file__}' started in '{os.getcwd()}':\n")
80 |             for idx, arg in enumerate(sys.argv[1:]):
81 |                 f.write(f"\t{idx}:{arg}$\n")
82 |     del args.debug
83 | 
84 |     receiver = ShellCommandReceive()
85 |     with receiver as r:
86 |         recevier_method = getattr(r, f"cmd_{cmd}")
87 |         recevier_method(**vars(args))
88 | 
89 | 
90 | if __name__ == '__main__':
91 |     main()
92 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/_test_startup_crash.py:
--------------------------------------------------------------------------------
1 | import os
2 | import time
3 | 
4 | time.sleep(int(os.environ.get("CRASH_DELAY") or "1"))
5 | os._exit(1)
6 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_async.py:
--------------------------------------------------------------------------------
 1 | """tests for async utilities"""
 2 | 
 3 | import pytest
 4 | 
 5 | from ipyparallel._async import AsyncFirst
 6 | 
 7 | 
 8 | class A(AsyncFirst):
 9 |     a = 5
10 | 
11 |     def sync_method(self):
12 |         return 'sync'
13 | 
14 |     async def async_method(self):
15 |         return 'async'
16 | 
17 | 
18 | def test_async_first_dir():
19 |     a = A()
20 |     attrs = sorted(dir(a))
21 |     real_attrs = [a for a in attrs if not a.startswith("_")]
22 |     assert real_attrs == ['a', 'async_method', 'async_method_sync', 'sync_method']
23 | 
24 | 
25 | def test_getattr():
26 |     a = A()
27 |     assert a.a == 5
28 |     with pytest.raises(AttributeError):
29 |         a.a_sync
30 |     with pytest.raises(AttributeError):
31 |         a.some_other_sync
32 |     with pytest.raises(AttributeError):
33 |         a.sync_method_sync
34 | 
35 | 
36 | def test_sync_no_asyncio():
37 |     a = A()
38 |     assert a.async_method_sync() == 'async'
39 |     assert a._async_thread is None
40 | 
41 | 
42 | async def test_sync_asyncio():
43 |     a = A()
44 |     assert a.async_method_sync() == 'async'
45 |     assert a._async_thread is not None
46 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_canning.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import pickle
  3 | from binascii import b2a_hex
  4 | from functools import partial
  5 | 
  6 | from ipyparallel.serialize import canning
  7 | from ipyparallel.serialize.canning import can, uncan
  8 | 
  9 | 
 10 | def interactive(f):
 11 |     f.__module__ = '__main__'
 12 |     return f
 13 | 
 14 | 
 15 | def dumps(obj):
 16 |     return pickle.dumps(can(obj))
 17 | 
 18 | 
 19 | def loads(obj):
 20 |     return uncan(pickle.loads(obj))
 21 | 
 22 | 
 23 | def roundtrip(obj):
 24 |     return loads(dumps(obj))
 25 | 
 26 | 
 27 | def test_no_closure():
 28 |     @interactive
 29 |     def foo():
 30 |         a = 5
 31 |         return a
 32 | 
 33 |     pfoo = dumps(foo)
 34 |     bar = loads(pfoo)
 35 |     assert foo() == bar()
 36 | 
 37 | 
 38 | def test_generator_closure():
 39 |     # this only creates a closure on Python 3
 40 |     @interactive
 41 |     def foo():
 42 |         i = 'i'
 43 |         r = [i for j in (1, 2)]
 44 |         return r
 45 | 
 46 |     pfoo = dumps(foo)
 47 |     bar = loads(pfoo)
 48 |     assert foo() == bar()
 49 | 
 50 | 
 51 | def test_nested_closure():
 52 |     @interactive
 53 |     def foo():
 54 |         i = 'i'
 55 | 
 56 |         def g():
 57 |             return i
 58 | 
 59 |         return g()
 60 | 
 61 |     pfoo = dumps(foo)
 62 |     bar = loads(pfoo)
 63 |     assert foo() == bar()
 64 | 
 65 | 
 66 | def test_closure():
 67 |     i = 'i'
 68 | 
 69 |     @interactive
 70 |     def foo():
 71 |         return i
 72 | 
 73 |     pfoo = dumps(foo)
 74 |     bar = loads(pfoo)
 75 |     assert foo() == bar()
 76 | 
 77 | 
 78 | def test_uncan_bytes_buffer():
 79 |     data = b'data'
 80 |     canned = can(data)
 81 |     canned.buffers = [memoryview(buf) for buf in canned.buffers]
 82 |     out = uncan(canned)
 83 |     assert out == data
 84 | 
 85 | 
 86 | def test_can_partial():
 87 |     def foo(x, y, z):
 88 |         return x * y * z
 89 | 
 90 |     partial_foo = partial(foo, 2, y=5)
 91 |     canned = can(partial_foo)
 92 |     assert isinstance(canned, canning.CannedPartial)
 93 |     dumped = pickle.dumps(canned)
 94 |     loaded = pickle.loads(dumped)
 95 |     pfoo2 = uncan(loaded)
 96 |     assert pfoo2(z=3) == partial_foo(z=3)
 97 | 
 98 | 
 99 | def test_can_partial_buffers():
100 |     def foo(arg1, arg2, kwarg1, kwarg2):
101 |         return f'{arg1}{arg2[:32]}{b2a_hex(kwarg1.tobytes()[:32])}{kwarg2}'
102 | 
103 |     buf1 = os.urandom(1024 * 1024)
104 |     buf2 = memoryview(os.urandom(1024 * 1024))
105 |     partial_foo = partial(foo, 5, buf1, kwarg1=buf2, kwarg2=10)
106 |     canned = can(partial_foo)
107 |     assert len(canned.buffers) == 2
108 |     assert isinstance(canned, canning.CannedPartial)
109 |     buffers, canned.buffers = canned.buffers, []
110 |     dumped = pickle.dumps(canned)
111 |     loaded = pickle.loads(dumped)
112 |     loaded.buffers = buffers
113 |     pfoo2 = uncan(loaded)
114 |     assert pfoo2() == partial_foo()
115 | 
116 | 
117 | def test_keyword_only_arguments():
118 |     def compute(a, *, b=3):
119 |         return a * b
120 | 
121 |     assert compute(2) == 6
122 |     compute_2 = roundtrip(compute)
123 |     assert compute_2(2) == 6
124 | 
125 | 
126 | def test_annotations():
127 |     def f(a: int):
128 |         return a
129 | 
130 |     f2 = roundtrip(f)
131 |     assert f2.__annotations__ == f.__annotations__
132 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_dependency.py:
--------------------------------------------------------------------------------
  1 | """Tests for dependency.py"""
  2 | 
  3 | # Copyright (c) IPython Development Team.
  4 | # Distributed under the terms of the Modified BSD License.
  5 | import ipyparallel as ipp
  6 | from ipyparallel.serialize import can, uncan
  7 | from ipyparallel.util import interactive
  8 | 
  9 | from .clienttest import ClusterTestCase, raises_remote
 10 | 
 11 | 
 12 | @ipp.require('time')
 13 | def wait(n):
 14 |     time.sleep(n)  # noqa: F821
 15 |     return n
 16 | 
 17 | 
 18 | @ipp.interactive
 19 | def func(x):
 20 |     return x * x
 21 | 
 22 | 
 23 | mixed = list(map(str, range(10)))
 24 | completed = list(map(str, range(0, 10, 2)))
 25 | failed = list(map(str, range(1, 10, 2)))
 26 | 
 27 | 
 28 | class TestDependency(ClusterTestCase):
 29 |     def setup_method(self):
 30 |         super().setup_method()
 31 |         self.user_ns = {'__builtins__': __builtins__}
 32 |         self.view = self.client.load_balanced_view()
 33 |         self.dview = self.client[-1]
 34 |         self.succeeded = set(map(str, range(0, 25, 2)))
 35 |         self.failed = set(map(str, range(1, 25, 2)))
 36 | 
 37 |     def assertMet(self, dep):
 38 |         assert dep.check(self.succeeded, self.failed), "Dependency should be met"
 39 | 
 40 |     def assertUnmet(self, dep):
 41 |         assert not dep.check(self.succeeded, self.failed), (
 42 |             "Dependency should not be met"
 43 |         )
 44 | 
 45 |     def assertUnreachable(self, dep):
 46 |         assert dep.unreachable(self.succeeded, self.failed), (
 47 |             "Dependency should be unreachable"
 48 |         )
 49 | 
 50 |     def assertReachable(self, dep):
 51 |         assert not dep.unreachable(self.succeeded, self.failed), (
 52 |             "Dependency should be reachable"
 53 |         )
 54 | 
 55 |     def cancan(self, f):
 56 |         """decorator to pass through canning into self.user_ns"""
 57 |         return uncan(can(f), self.user_ns)
 58 | 
 59 |     def test_require_imports(self):
 60 |         """test that @require imports names"""
 61 | 
 62 |         @self.cancan
 63 |         @ipp.require('base64')
 64 |         @interactive
 65 |         def encode(arg):
 66 |             return base64.b64encode(arg)  # noqa: F821
 67 | 
 68 |         # must pass through canning to properly connect namespaces
 69 |         assert encode(b'foo') == b'Zm9v'
 70 | 
 71 |     def test_success_only(self):
 72 |         dep = ipp.Dependency(mixed, success=True, failure=False)
 73 |         self.assertUnmet(dep)
 74 |         self.assertUnreachable(dep)
 75 |         dep.all = False
 76 |         self.assertMet(dep)
 77 |         self.assertReachable(dep)
 78 |         dep = ipp.Dependency(completed, success=True, failure=False)
 79 |         self.assertMet(dep)
 80 |         self.assertReachable(dep)
 81 |         dep.all = False
 82 |         self.assertMet(dep)
 83 |         self.assertReachable(dep)
 84 | 
 85 |     def test_failure_only(self):
 86 |         dep = ipp.Dependency(mixed, success=False, failure=True)
 87 |         self.assertUnmet(dep)
 88 |         self.assertUnreachable(dep)
 89 |         dep.all = False
 90 |         self.assertMet(dep)
 91 |         self.assertReachable(dep)
 92 |         dep = ipp.Dependency(completed, success=False, failure=True)
 93 |         self.assertUnmet(dep)
 94 |         self.assertUnreachable(dep)
 95 |         dep.all = False
 96 |         self.assertUnmet(dep)
 97 |         self.assertUnreachable(dep)
 98 | 
 99 |     def test_require_function(self):
100 |         @ipp.interactive
101 |         def bar(a):
102 |             return func(a)
103 | 
104 |         @ipp.require(func)
105 |         @ipp.interactive
106 |         def bar2(a):
107 |             return func(a)
108 | 
109 |         self.client[:].clear()
110 |         with raises_remote(NameError):
111 |             self.view.apply_sync(bar, 5)
112 |         ar = self.view.apply_async(bar2, 5)
113 |         assert ar.get(5) == func(5)
114 | 
115 |     def test_require_object(self):
116 |         @ipp.require(foo=func)
117 |         @ipp.interactive
118 |         def bar(a):
119 |             return foo(a)  # noqa: F821
120 | 
121 |         ar = self.view.apply_async(bar, 5)
122 |         assert ar.get(5) == func(5)
123 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_executor.py:
--------------------------------------------------------------------------------
 1 | """Tests for Executor API"""
 2 | 
 3 | # Copyright (c) IPython Development Team.
 4 | # Distributed under the terms of the Modified BSD License.
 5 | import time
 6 | 
 7 | from ipyparallel.client.view import LazyMapIterator, LoadBalancedView
 8 | 
 9 | from .clienttest import ClusterTestCase
10 | 
11 | 
12 | def wait(n):
13 |     import time
14 | 
15 |     time.sleep(n)
16 |     return n
17 | 
18 | 
19 | def echo(x):
20 |     return x
21 | 
22 | 
23 | class TestExecutor(ClusterTestCase):
24 |     def test_client_executor(self):
25 |         executor = self.client.executor()
26 |         assert isinstance(executor.view, LoadBalancedView)
27 |         f = executor.submit(lambda x: 2 * x, 5)
28 |         r = f.result()
29 |         assert r == 10
30 | 
31 |     def test_view_executor(self):
32 |         view = self.client.load_balanced_view()
33 |         executor = view.executor
34 |         assert executor.view is view
35 | 
36 |     def test_executor_submit(self):
37 |         view = self.client.load_balanced_view()
38 |         executor = view.executor
39 |         f = executor.submit(lambda x, y: x * y, 2, 3)
40 |         r = f.result()
41 |         assert r == 6
42 | 
43 |     def test_executor_map(self):
44 |         view = self.client.load_balanced_view()
45 |         executor = view.executor
46 |         gen = executor.map(lambda x: x, range(5))
47 |         assert isinstance(gen, LazyMapIterator)
48 |         for i, r in enumerate(gen):
49 |             assert i == r
50 | 
51 |     def test_executor_context(self):
52 |         view = self.client.load_balanced_view()
53 |         executor = view.executor
54 |         with executor:
55 |             f = executor.submit(time.sleep, 0.5)
56 |             assert not f.done()
57 |             m = executor.map(lambda x: x, range(10))
58 |         assert len(view.history) == 11
59 |         # Executor context calls shutdown
60 |         # shutdown doesn't shutdown engines,
61 |         # but it should at least wait for results to finish
62 |         assert f.done()
63 |         tic = time.perf_counter()
64 |         list(m)
65 |         toc = time.perf_counter()
66 |         assert toc - tic < 0.5
67 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_joblib.py:
--------------------------------------------------------------------------------
 1 | import warnings
 2 | from unittest import mock
 3 | 
 4 | import pytest
 5 | 
 6 | import ipyparallel as ipp
 7 | 
 8 | from .clienttest import ClusterTestCase, add_engines
 9 | 
10 | try:
11 |     with warnings.catch_warnings():
12 |         warnings.simplefilter("ignore")
13 |         from joblib import Parallel, delayed
14 | 
15 |         from ipyparallel.client._joblib import IPythonParallelBackend  # noqa
16 | except (ImportError, TypeError):
17 |     have_joblib = False
18 | else:
19 |     have_joblib = True
20 | 
21 | 
22 | def neg(x):
23 |     return -1 * x
24 | 
25 | 
26 | class TestJobLib(ClusterTestCase):
27 |     def setup_method(self):
28 |         if not have_joblib:
29 |             pytest.skip("Requires joblib >= 0.10")
30 |         super().setup_method()
31 |         add_engines(1, total=True)
32 | 
33 |     def test_default_backend(self):
34 |         """ipyparallel.register_joblib_backend() registers default backend"""
35 |         ipp.register_joblib_backend()
36 |         with mock.patch.object(ipp.Client, "__new__", lambda *a, **kw: self.client):
37 |             p = Parallel(backend='ipyparallel')
38 |             assert p._backend._view.client is self.client
39 | 
40 |     def test_register_backend(self):
41 |         view = self.client.load_balanced_view()
42 |         view.register_joblib_backend('view')
43 |         p = Parallel(backend='view')
44 |         assert p._backend._view is view
45 | 
46 |     def test_joblib_backend(self):
47 |         view = self.client.load_balanced_view()
48 |         view.register_joblib_backend('view')
49 |         p = Parallel(backend='view')
50 |         result = p(delayed(neg)(i) for i in range(10))
51 |         assert result == [neg(i) for i in range(10)]
52 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_mongodb.py:
--------------------------------------------------------------------------------
 1 | """Tests for mongodb backend"""
 2 | 
 3 | # Copyright (c) IPython Development Team.
 4 | # Distributed under the terms of the Modified BSD License.
 5 | import os
 6 | from unittest import TestCase
 7 | 
 8 | import pytest
 9 | 
10 | from . import test_db
11 | 
12 | c = None
13 | 
14 | 
15 | @pytest.fixture(scope='module')
16 | def mongo_conn(request):
17 |     global c
18 |     try:
19 |         from pymongo import MongoClient
20 |     except ImportError:
21 |         pytest.skip("Requires mongodb")
22 | 
23 |     conn_kwargs = {}
24 |     if 'DB_IP' in os.environ:
25 |         conn_kwargs['host'] = os.environ['DB_IP']
26 |     if 'DBA_MONGODB_ADMIN_URI' in os.environ:
27 |         # On ShiningPanda, we need a username and password to connect. They are
28 |         # passed in a mongodb:// URI.
29 |         conn_kwargs['host'] = os.environ['DBA_MONGODB_ADMIN_URI']
30 |     if 'DB_PORT' in os.environ:
31 |         conn_kwargs['port'] = int(os.environ['DB_PORT'])
32 | 
33 |     try:
34 |         c = MongoClient(**conn_kwargs, serverSelectionTimeoutMS=2000)
35 |         servinfo = c.server_info()
36 |     except Exception:
37 |         c = None
38 |     if c is not None:
39 |         request.addfinalizer(lambda: c.drop_database('iptestdb'))
40 |     return c
41 | 
42 | 
43 | @pytest.mark.usefixtures('mongo_conn')
44 | class TestMongoBackend(test_db.TaskDBTest, TestCase):
45 |     """MongoDB backend tests"""
46 | 
47 |     def create_db(self):
48 |         try:
49 |             from ipyparallel.controller.mongodb import MongoDB
50 | 
51 |             return MongoDB(database='iptestdb', _connection=c)
52 |         except Exception:
53 |             pytest.skip("Couldn't connect to mongodb")
54 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_mpi.py:
--------------------------------------------------------------------------------
 1 | import shutil
 2 | 
 3 | import pytest
 4 | 
 5 | from .test_cluster import (
 6 |     test_get_output,  # noqa: F401
 7 |     test_restart_engines,  # noqa: F401
 8 |     test_signal_engines,  # noqa: F401
 9 |     test_start_stop_cluster,  # noqa: F401
10 |     test_to_from_dict,  # noqa: F401
11 | )
12 | 
13 | # import tests that use engine_launcher_class fixture
14 | 
15 | 
16 | # override engine_launcher_class
17 | @pytest.fixture
18 | def engine_launcher_class():
19 |     if shutil.which("mpiexec") is None:
20 |         pytest.skip("Requires mpiexec")
21 |     return 'mpi'
22 | 
23 | 
24 | @pytest.fixture
25 | def controller_launcher_class():
26 |     if shutil.which("mpiexec") is None:
27 |         pytest.skip("Requires mpiexec")
28 |     return 'mpi'
29 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_remotefunction.py:
--------------------------------------------------------------------------------
 1 | """Tests for remote functions"""
 2 | 
 3 | # Copyright (c) IPython Development Team.
 4 | # Distributed under the terms of the Modified BSD License.
 5 | import ipyparallel as ipp
 6 | 
 7 | from .clienttest import ClusterTestCase
 8 | 
 9 | 
10 | class TestRemoteFunctions(ClusterTestCase):
11 |     def test_remote(self):
12 |         v = self.client[-1]
13 | 
14 |         @ipp.remote(v, block=True)
15 |         def foo(x, y=5):
16 |             """multiply x * y"""
17 |             return x * y
18 | 
19 |         assert foo.__name__ == 'foo'
20 |         assert 'RemoteFunction' in foo.__doc__
21 |         assert 'multiply x' in foo.__doc__
22 | 
23 |         z = foo(5)
24 |         assert z == 25
25 |         z = foo(2, 3)
26 |         assert z == 6
27 |         z = foo(x=5, y=2)
28 |         assert z == 10
29 | 
30 |     def test_parallel(self):
31 |         n = 2
32 |         v = self.client[:n]
33 | 
34 |         @ipp.parallel(v, block=True)
35 |         def foo(x):
36 |             """multiply x * y"""
37 |             return x * 2
38 | 
39 |         assert foo.__name__ == 'foo'
40 |         assert 'ParallelFunction' in foo.__doc__
41 |         assert 'multiply x' in foo.__doc__
42 | 
43 |         z = foo([1, 2, 3, 4])
44 |         assert z, [1, 2, 1, 2, 3, 4, 3 == 4]
45 | 
46 |     def test_parallel_map(self):
47 |         v = self.client.load_balanced_view()
48 | 
49 |         @ipp.parallel(v, block=True)
50 |         def foo(x, y=5):
51 |             """multiply x * y"""
52 |             return x * y
53 | 
54 |         z = foo.map([1, 2, 3])
55 |         assert z, [5, 10 == 15]
56 |         z = foo.map([1, 2, 3], [1, 2, 3])
57 |         assert z, [1, 4 == 9]
58 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_slurm.py:
--------------------------------------------------------------------------------
 1 | import shutil
 2 | from unittest import mock
 3 | 
 4 | import pytest
 5 | from traitlets.config import Config
 6 | 
 7 | from . import test_cluster
 8 | from .conftest import temporary_ipython_dir
 9 | from .test_cluster import (
10 |     test_get_output,  # noqa: F401
11 |     test_restart_engines,  # noqa: F401
12 |     test_signal_engines,  # noqa: F401
13 |     test_start_stop_cluster,  # noqa: F401
14 |     test_to_from_dict,  # noqa: F401
15 | )
16 | 
17 | 
18 | @pytest.fixture(autouse=True, scope="module")
19 | def longer_timeout():
20 |     # slurm tests started failing with timeouts
21 |     # when adding timeout to test_restart_engines
22 |     # maybe it's just slow...
23 |     with mock.patch.object(test_cluster, "_timeout", 120):
24 |         yield
25 | 
26 | 
27 | # put ipython dir on shared filesystem
28 | @pytest.fixture(autouse=True, scope="module")
29 | def ipython_dir(request):
30 |     if shutil.which("sbatch") is None:
31 |         pytest.skip("Requires slurm")
32 |     with temporary_ipython_dir(prefix="/data/") as ipython_dir:
33 |         yield ipython_dir
34 | 
35 | 
36 | @pytest.fixture
37 | def cluster_config():
38 |     c = Config()
39 |     c.Cluster.controller_ip = '0.0.0.0'
40 |     return c
41 | 
42 | 
43 | # override launcher classes
44 | @pytest.fixture
45 | def engine_launcher_class():
46 |     if shutil.which("sbatch") is None:
47 |         pytest.skip("Requires slurm")
48 |     return 'slurm'
49 | 
50 | 
51 | @pytest.fixture
52 | def controller_launcher_class():
53 |     if shutil.which("sbatch") is None:
54 |         pytest.skip("Requires slurm")
55 |     return 'slurm'
56 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_ssh.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | from functools import partial
 3 | 
 4 | import pytest
 5 | from traitlets.config import Config
 6 | 
 7 | from .conftest import Cluster as BaseCluster  # noqa: F401
 8 | from .test_cluster import (
 9 |     test_get_output,  # noqa: F401
10 |     test_restart_engines,  # noqa: F401
11 |     test_signal_engines,  # noqa: F401
12 |     test_start_stop_cluster,  # noqa: F401
13 |     test_to_from_dict,  # noqa: F401
14 | )
15 | 
16 | # import tests that use engine_launcher_class fixture
17 | 
18 | 
19 | @pytest.fixture(params=["SSH", "SSHProxy"])
20 | def ssh_config(ssh_key, request):
21 |     windows = os.name == "nt"
22 | 
23 |     if (
24 |         windows and request.param == "SSHProxy"
25 |     ):  # SSHProxy currently not working under Windows
26 |         pytest.skip("Proxy tests currently not working under Windows")
27 | 
28 |     c = Config()
29 |     c.Cluster.controller_ip = '0.0.0.0'
30 |     c.Cluster.engine_launcher_class = request.param
31 |     engine_set_cfg = c[f"{request.param}EngineSetLauncher"]
32 |     engine_set_cfg.ssh_args = []
33 |     if not windows:
34 |         engine_set_cfg.ssh_args.extend(
35 |             [
36 |                 "-o",
37 |                 "UserKnownHostsFile=/dev/null",
38 |                 "-o",
39 |                 "StrictHostKeyChecking=no",
40 |                 "-i",
41 |                 ssh_key,
42 |             ]
43 |         )
44 |     engine_set_cfg.scp_args = list(engine_set_cfg.ssh_args)  # copy
45 |     if windows:
46 |         engine_set_cfg.remote_python = "python"
47 |         engine_set_cfg.remote_profile_dir = "C:/Users/ciuser/.ipython/profile_default"
48 |     else:
49 |         engine_set_cfg.remote_python = "/opt/conda/bin/python3"
50 |         engine_set_cfg.remote_profile_dir = "/home/ciuser/.ipython/profile_default"
51 |     engine_set_cfg.engine_args = ['--debug']
52 |     c.SSHProxyEngineSetLauncher.hostname = "127.0.0.1"
53 |     c.SSHProxyEngineSetLauncher.ssh_args.append("-p2222")
54 |     c.SSHProxyEngineSetLauncher.scp_args.append("-P2222")
55 |     c.SSHProxyEngineSetLauncher.user = "ciuser"
56 |     c.SSHEngineSetLauncher.engines = {"ciuser@127.0.0.1:2222": 4}
57 |     return c
58 | 
59 | 
60 | @pytest.fixture
61 | def Cluster(ssh_config, BaseCluster):  # noqa: F811
62 |     """Override Cluster to add ssh config"""
63 |     return partial(BaseCluster, config=ssh_config)
64 | 
65 | 
66 | # override engine_launcher_class
67 | @pytest.fixture
68 | def engine_launcher_class(ssh_config):
69 |     return ssh_config.Cluster.engine_launcher_class
70 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_util.py:
--------------------------------------------------------------------------------
 1 | import socket
 2 | 
 3 | import pytest
 4 | from jupyter_client.localinterfaces import localhost, public_ips
 5 | 
 6 | from ipyparallel import util
 7 | 
 8 | 
 9 | def test_disambiguate_ip():
10 |     # garbage in, garbage out
11 |     assert util.disambiguate_ip_address('garbage') == 'garbage'
12 |     assert util.disambiguate_ip_address('0.0.0.0', socket.gethostname()) == localhost()
13 |     wontresolve = 'this.wontresolve.dns'
14 |     with pytest.warns(
15 |         RuntimeWarning, match=f"IPython could not determine IPs for {wontresolve}"
16 |     ):
17 |         assert util.disambiguate_ip_address('0.0.0.0', wontresolve) == wontresolve
18 |     if public_ips():
19 |         public_ip = public_ips()[0]
20 |         assert util.disambiguate_ip_address('0.0.0.0', public_ip) == localhost()
21 | 


--------------------------------------------------------------------------------
/ipyparallel/tests/test_view_broadcast.py:
--------------------------------------------------------------------------------
 1 | """test BroadcastView objects"""
 2 | 
 3 | import pytest
 4 | 
 5 | from . import test_view
 6 | 
 7 | needs_map = pytest.mark.xfail(reason="map not yet implemented")
 8 | 
 9 | 
10 | @pytest.mark.usefixtures('ipython')
11 | class TestBroadcastView(test_view.TestView):
12 |     is_coalescing = False
13 | 
14 |     def setup_method(self):
15 |         super().setup_method()
16 |         self._broadcast_view_used = False
17 |         # use broadcast view for direct API
18 |         real_direct_view = self.client.real_direct_view = self.client.direct_view
19 | 
20 |         def broadcast_or_direct(targets):
21 |             if isinstance(targets, int):
22 |                 return real_direct_view(targets)
23 |             else:
24 |                 self._broadcast_view_used = True
25 |                 return self.client.broadcast_view(
26 |                     targets, is_coalescing=self.is_coalescing
27 |                 )
28 | 
29 |         self.client.direct_view = broadcast_or_direct
30 | 
31 |     def teardown_method(self):
32 |         super().teardown_method()
33 |         # note that a test didn't use a broadcast view
34 |         if not self._broadcast_view_used:
35 |             pytest.skip("No broadcast view used")
36 | 
37 |     @pytest.mark.xfail(reason="Tracking gets disconnected from original message")
38 |     def test_scatter_tracked(self):
39 |         pass
40 | 
41 | 
42 | class TestBroadcastViewCoalescing(TestBroadcastView):
43 |     is_coalescing = True
44 | 
45 |     @pytest.mark.xfail(reason="coalescing view doesn't preserve target order")
46 |     def test_target_ordering(self):
47 |         self.minimum_engines(4)
48 |         ids_in_order = self.client.ids
49 |         dv = self.client.real_direct_view(ids_in_order)
50 | 
51 |         dv.scatter('rank', ids_in_order, flatten=True, block=True)
52 |         assert dv['rank'] == ids_in_order
53 | 
54 |         view = self.client.broadcast_view(ids_in_order, is_coalescing=True)
55 |         assert view['rank'] == ids_in_order
56 | 
57 |         view = self.client.broadcast_view(ids_in_order[::-1], is_coalescing=True)
58 |         assert view['rank'] == ids_in_order[::-1]
59 | 
60 |         view = self.client.broadcast_view(ids_in_order[::2], is_coalescing=True)
61 |         assert view['rank'] == ids_in_order[::2]
62 | 
63 |         view = self.client.broadcast_view(ids_in_order[::-2], is_coalescing=True)
64 |         assert view['rank'] == ids_in_order[::-2]
65 | 
66 |     def test_engine_metadata(self):
67 |         self.minimum_engines(4)
68 |         ids_in_order = sorted(self.client.ids)
69 |         dv = self.client.real_direct_view(ids_in_order)
70 |         dv.scatter('rank', ids_in_order, flatten=True, block=True)
71 |         view = self.client.broadcast_view(ids_in_order, is_coalescing=True)
72 |         ar = view.pull('rank', block=False)
73 |         result = ar.get(timeout=10)
74 |         assert isinstance(ar.engine_id, list)
75 |         assert isinstance(ar.engine_uuid, list)
76 |         assert result == ar.engine_id
77 |         assert sorted(ar.engine_id) == ids_in_order
78 | 
79 |         even_ids = ids_in_order[::-2]
80 |         view = self.client.broadcast_view(even_ids, is_coalescing=True)
81 |         ar = view.pull('rank', block=False)
82 |         result = ar.get(timeout=10)
83 |         assert isinstance(ar.engine_id, list)
84 |         assert isinstance(ar.engine_uuid, list)
85 |         assert result == ar.engine_id
86 |         assert sorted(ar.engine_id) == sorted(even_ids)
87 | 
88 |     @pytest.mark.xfail(reason="displaypub ordering not preserved")
89 |     def test_apply_displaypub(self):
90 |         pass
91 | 
92 | 
93 | # FIXME
94 | del TestBroadcastView
95 | 


--------------------------------------------------------------------------------
/ipyparallel/traitlets.py:
--------------------------------------------------------------------------------
  1 | """Custom ipyparallel trait types"""
  2 | 
  3 | import sys
  4 | 
  5 | if sys.version_info < (3, 10):
  6 |     from importlib_metadata import entry_points
  7 | else:
  8 |     from importlib.metadata import entry_points
  9 | 
 10 | from traitlets import List, TraitError, Type
 11 | 
 12 | 
 13 | class Launcher(Type):
 14 |     """Entry point-extended Type
 15 | 
 16 |     classes can be registered via entry points
 17 |     in addition to standard 'mypackage.MyClass' strings
 18 |     """
 19 | 
 20 |     def __init__(self, *args, entry_point_group, **kwargs):
 21 |         self.entry_point_group = entry_point_group
 22 |         kwargs.setdefault('klass', 'ipyparallel.cluster.launcher.BaseLauncher')
 23 |         super().__init__(*args, **kwargs)
 24 | 
 25 |     _original_help = ''
 26 | 
 27 |     @property
 28 |     def help(self):
 29 |         """Extend help by listing currently installed choices"""
 30 |         chunks = [self._original_help]
 31 |         chunks.append("Currently installed: ")
 32 |         for key, entry_point in self.load_entry_points().items():
 33 |             chunks.append(f"  - {key}: {entry_point.value}")
 34 |         return '\n'.join(chunks)
 35 | 
 36 |     @help.setter
 37 |     def help(self, value):
 38 |         self._original_help = value
 39 | 
 40 |     def load_entry_points(self):
 41 |         """Load my entry point group"""
 42 |         return {
 43 |             entry_point.name.lower(): entry_point
 44 |             for entry_point in entry_points(group=self.entry_point_group)
 45 |         }
 46 | 
 47 |     def validate(self, obj, value):
 48 |         if isinstance(value, str):
 49 |             # first, look up in entry point registry
 50 |             registry = self.load_entry_points()
 51 |             key = value.lower()
 52 |             if key in registry:
 53 |                 value = registry[key].load()
 54 |         return super().validate(obj, value)
 55 | 
 56 | 
 57 | class PortList(List):
 58 |     """List of ports
 59 | 
 60 |     For use configuring a list of ports to consume
 61 | 
 62 |     Ports will be a list of valid ports
 63 | 
 64 |     Can be specified as a port-range string for convenience
 65 |     (mainly for use on the command-line)
 66 |     e.g. '10101-10105,10108'
 67 |     """
 68 | 
 69 |     @staticmethod
 70 |     def parse_port_range(s):
 71 |         """Parse a port range string in the form '1,3-5,6' into [1,3,4,5,6]"""
 72 |         ports = []
 73 |         ranges = s.split(",")
 74 |         for r in ranges:
 75 |             start, _, end = r.partition("-")
 76 |             start = int(start)
 77 |             if end:
 78 |                 end = int(end)
 79 |                 ports.extend(range(start, end + 1))
 80 |             else:
 81 |                 ports.append(start)
 82 |         return ports
 83 | 
 84 |     def from_string_list(self, s_list):
 85 |         ports = []
 86 |         for s in s_list:
 87 |             ports.extend(self.parse_port_range(s))
 88 |         return ports
 89 | 
 90 |     def validate(self, obj, value):
 91 |         if isinstance(value, str):
 92 |             value = self.parse_port_range(value)
 93 |         value = super().validate(obj, value)
 94 |         for item in value:
 95 |             if not isinstance(item, int):
 96 |                 raise TraitError(
 97 |                     f"Ports must be integers in range 1-65536, not {item!r}"
 98 |                 )
 99 |             if not 1 <= item <= 65536:
100 |                 raise TraitError(
101 |                     f"Ports must be integers in range 1-65536, not {item!r}"
102 |                 )
103 |         return value
104 | 


--------------------------------------------------------------------------------
/jupyter-config/jupyter_notebook_config.d/ipyparallel.json:
--------------------------------------------------------------------------------
1 | {
2 |   "NotebookApp": {
3 |     "nbserver_extensions": {
4 |       "ipyparallel": true
5 |     }
6 |   }
7 | }
8 | 


--------------------------------------------------------------------------------
/jupyter-config/jupyter_server_config.d/ipyparallel.json:
--------------------------------------------------------------------------------
1 | {
2 |   "ServerApp": {
3 |     "jpserver_extensions": {
4 |       "ipyparallel": true
5 |     }
6 |   }
7 | }
8 | 


--------------------------------------------------------------------------------
/jupyter-config/nbconfig/tree.d/ipyparallel.json:
--------------------------------------------------------------------------------
1 | {
2 |   "load_extensions": {
3 |     "ipyparallel/main": true
4 |   }
5 | }
6 | 


--------------------------------------------------------------------------------
/lab/schema/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 |   "jupyter.lab.setting-icon-class": "ipp-Logo",
3 |   "jupyter.lab.setting-icon-label": "IPython Parallel",
4 |   "title": "IPython Parallel",
5 |   "description": "Settings for the IPython Parallel plugin.",
6 |   "properties": {},
7 |   "type": "object"
8 | }
9 | 


--------------------------------------------------------------------------------
/lab/src/commands.ts:
--------------------------------------------------------------------------------
 1 | export namespace CommandIDs {
 2 |   /**
 3 |    * Inject client code into the active editor.
 4 |    */
 5 |   export const injectClientCode = "ipyparallel:inject-client-code";
 6 | 
 7 |   /**
 8 |    * Launch a new cluster.
 9 |    */
10 |   export const newCluster = "ipyparallel:new-cluster";
11 | 
12 |   /**
13 |    * Launch a new cluster.
14 |    */
15 |   export const startCluster = "ipyparallel:start-cluster";
16 | 
17 |   /**
18 |    * Shutdown a cluster.
19 |    */
20 |   export const stopCluster = "ipyparallel:stop-cluster";
21 | 
22 |   /**
23 |    * Scale a cluster.
24 |    */
25 |   export const scaleCluster = "ipyparallel:scale-cluster";
26 | 
27 |   /**
28 |    * Toggle the auto-starting of clients.
29 |    */
30 |   export const toggleAutoStartClient = "ipyparallel:toggle-auto-start-client";
31 | }
32 | 


--------------------------------------------------------------------------------
/lab/src/sidebar.ts:
--------------------------------------------------------------------------------
 1 | import { Widget, PanelLayout } from "@lumino/widgets";
 2 | import { CommandRegistry } from "@lumino/commands";
 3 | 
 4 | import { ClusterManager, IClusterModel } from "./clusters";
 5 | 
 6 | /**
 7 |  * A widget for hosting IPP cluster widgets
 8 |  */
 9 | export class Sidebar extends Widget {
10 |   /**
11 |    * Create a new IPP sidebar.
12 |    */
13 |   constructor(options: Sidebar.IOptions) {
14 |     super();
15 |     this.addClass("ipp-Sidebar");
16 |     let layout = (this.layout = new PanelLayout());
17 | 
18 |     const injectClientCodeForCluster = options.clientCodeInjector;
19 |     const getClientCodeForCluster = options.clientCodeGetter;
20 |     // Add the cluster manager component.
21 |     this._clusters = new ClusterManager({
22 |       registry: options.registry,
23 |       injectClientCodeForCluster,
24 |       getClientCodeForCluster,
25 |     });
26 |     layout.addWidget(this._clusters);
27 |   }
28 | 
29 |   /**
30 |    * Get the cluster manager associated with the sidebar.
31 |    */
32 |   get clusterManager(): ClusterManager {
33 |     return this._clusters;
34 |   }
35 | 
36 |   private _clusters: ClusterManager;
37 | }
38 | 
39 | /**
40 |  * A namespace for Sidebar statics.
41 |  */
42 | export namespace Sidebar {
43 |   /**
44 |    * Options for the constructor.
45 |    */
46 |   export interface IOptions {
47 |     /**
48 |      * Registry of all commands
49 |      */
50 |     registry: CommandRegistry;
51 | 
52 |     /**
53 |      * A function that injects client-connection code for a given cluster.
54 |      */
55 |     clientCodeInjector: (model: IClusterModel) => void;
56 | 
57 |     /**
58 |      * A function that gets client-connection code for a given cluster.
59 |      */
60 |     clientCodeGetter: (model: IClusterModel) => string;
61 |   }
62 | }
63 | 


--------------------------------------------------------------------------------
/lab/src/svg.d.ts:
--------------------------------------------------------------------------------
1 | // svg.d.ts
2 | 
3 | declare module "*.svg" {
4 |   const value: string;
5 |   export default value;
6 | }
7 | 


--------------------------------------------------------------------------------
/lab/style/code-dark.svg:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/lab/style/code-light.svg:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/lab/style/logo.svg:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 8 |    
 9 |     
11 |       
15 |       
19 |   
20 | 
21 | 


--------------------------------------------------------------------------------
/lab/webpack.config.js:
--------------------------------------------------------------------------------
1 | const crypto = require("crypto");
2 | 
3 | // Workaround for loaders using "md4" by default, which is not supported in FIPS-compliant OpenSSL
4 | const cryptoOrigCreateHash = crypto.createHash;
5 | crypto.createHash = (algorithm) =>
6 |   cryptoOrigCreateHash(algorithm == "md4" ? "sha256" : algorithm);
7 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "ipyparallel-labextension",
 3 |   "version": "9.1.0",
 4 |   "private": true,
 5 |   "description": "A JupyterLab extension for IPython Parallel.",
 6 |   "keywords": [
 7 |     "ipython",
 8 |     "jupyter",
 9 |     "jupyterlab",
10 |     "jupyterlab-extension"
11 |   ],
12 |   "homepage": "https://github.com/ipython/ipyparallel",
13 |   "bugs": {
14 |     "url": "https://github.com/ipython/ipyparallel/issues"
15 |   },
16 |   "repository": {
17 |     "type": "git",
18 |     "url": "https://github.com/ipython/ipyparallel"
19 |   },
20 |   "license": "BSD-3-Clause",
21 |   "author": "Min Ragan-Kelley",
22 |   "files": [
23 |     "lab/lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}",
24 |     "lab/schema/*.json",
25 |     "lab/style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}"
26 |   ],
27 |   "main": "lab/lib/index.js",
28 |   "types": "lab/lib/index.d.ts",
29 |   "scripts": {
30 |     "build": "jlpm run build:lib && jlpm run build:labextension:dev",
31 |     "build:labextension": "jupyter labextension build .",
32 |     "build:labextension:dev": "jupyter labextension build --development True .",
33 |     "build:lib": "tsc",
34 |     "build:prod": "jlpm run build:lib && jlpm run build:labextension",
35 |     "clean": "jlpm run clean:lib",
36 |     "clean:all": "jlpm run clean:lib && jlpm run clean:labextension",
37 |     "clean:labextension": "rimraf ipyparallel/labextension",
38 |     "clean:lib": "rimraf lab/lib tsconfig.tsbuildinfo",
39 |     "eslint": "eslint . --ext .ts,.tsx --fix",
40 |     "eslint:check": "eslint . --ext .ts,.tsx",
41 |     "install:extension": "jupyter labextension develop --overwrite .",
42 |     "lint": "prettier --check '**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}' && jlpm eslint:check",
43 |     "prettier": "prettier --write '**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}'",
44 |     "prettier:check": "prettier --list-different '**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}'",
45 |     "test": "mocha",
46 |     "watch": "run-p watch:src watch:labextension",
47 |     "watch:labextension": "jupyter labextension watch .",
48 |     "watch:src": "tsc -w"
49 |   },
50 |   "dependencies": {
51 |     "@jupyterlab/application": "^4.4.0",
52 |     "@jupyterlab/apputils": "^4.5.0",
53 |     "@jupyterlab/codeeditor": "^4.4.0",
54 |     "@jupyterlab/console": "^4.4.0",
55 |     "@jupyterlab/coreutils": "^6.4.0",
56 |     "@jupyterlab/nbformat": "^4.4.0",
57 |     "@jupyterlab/notebook": "^4.4.0",
58 |     "@jupyterlab/services": "^7.4.0",
59 |     "@jupyterlab/settingregistry": "^4.4.0",
60 |     "@jupyterlab/statedb": "^4.4.0",
61 |     "@jupyterlab/ui-components": "^4.4.0",
62 |     "@lumino/algorithm": "^2.0.3",
63 |     "@lumino/commands": "^2.3.2",
64 |     "@lumino/coreutils": "^2.2.1",
65 |     "@lumino/domutils": "^2.0.3",
66 |     "@lumino/dragdrop": "^2.1.6",
67 |     "@lumino/messaging": "^2.0.3",
68 |     "@lumino/polling": "^2.1.4",
69 |     "@lumino/signaling": "^2.1.4",
70 |     "@lumino/widgets": "^2.7.0",
71 |     "react": "^18.2.0",
72 |     "react-dom": "^18.2.0"
73 |   },
74 |   "devDependencies": {
75 |     "@jupyterlab/builder": "^4.4.0",
76 |     "@types/react": "^18.0.26",
77 |     "@types/react-dom": "~18.3.1",
78 |     "@typescript-eslint/eslint-plugin": "^8.30.1",
79 |     "@typescript-eslint/parser": "^8.30.1",
80 |     "eslint": "^9.24.0",
81 |     "eslint-config-prettier": "^10.1.2",
82 |     "eslint-plugin-prettier": "^5.2.6",
83 |     "eslint-plugin-react": "^7.37.5",
84 |     "npm-run-all2": "^7.0.1",
85 |     "prettier": "^3.5.3",
86 |     "rimraf": "^6.0.1",
87 |     "typescript": "~5.8.3"
88 |   },
89 |   "resolutions": {
90 |     "@types/react": "^18.0.26"
91 |   },
92 |   "jupyterlab": {
93 |     "extension": true,
94 |     "schemaDir": "lab/schema",
95 |     "webpackConfig": "lab/webpack.config.js",
96 |     "outputDir": "ipyparallel/labextension"
97 |   }
98 | }
99 | 


--------------------------------------------------------------------------------
/readthedocs.yml:
--------------------------------------------------------------------------------
 1 | version: 2
 2 | 
 3 | build:
 4 |   os: ubuntu-22.04
 5 |   tools:
 6 |     python: "3.10"
 7 |     nodejs: "18"
 8 | 
 9 | sphinx:
10 |   configuration: docs/source/conf.py
11 | 
12 | python:
13 |   install:
14 |     # install ipp itself
15 |     - path: .
16 |     - requirements: docs/requirements.txt
17 | 


--------------------------------------------------------------------------------
/tsconfig.eslint.json:
--------------------------------------------------------------------------------
1 | {
2 |   "extends": "./tsconfig",
3 |   "include": [".eslintrc.js", "*", "lab/src/*", "lab/*.js"]
4 | }
5 | 


--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "compilerOptions": {
 3 |     "allowSyntheticDefaultImports": true,
 4 |     "composite": true,
 5 |     "declaration": true,
 6 |     "esModuleInterop": true,
 7 |     "incremental": true,
 8 |     "lib": [
 9 |       "DOM",
10 |       "DOM.Iterable",
11 |       "ES2018",
12 |       "ES2020.BigInt",
13 |       "ES2020.Intl",
14 |       "ES2020.String"
15 |     ],
16 |     "jsx": "react",
17 |     "module": "esnext",
18 |     "moduleResolution": "node",
19 |     "noEmitOnError": true,
20 |     "noImplicitAny": true,
21 |     "noUnusedLocals": true,
22 |     "preserveWatchOutput": true,
23 |     "resolveJsonModule": true,
24 |     "outDir": "lab/lib",
25 |     "rootDir": "lab/src",
26 |     "strict": true,
27 |     "strictNullChecks": false,
28 |     "target": "es2018",
29 |     "types": []
30 |   },
31 |   "include": ["lab/src/*"]
32 | }
33 | 


--------------------------------------------------------------------------------