\d+)
13 | (?P((a|b|rc)\d+))?
14 | (\.
15 | (?Pdev\d*)
16 | )?
17 | ''',
18 | re.VERBOSE,
19 | )
20 |
21 | _version_fields = _version_regex.match(__version__).groupdict()
22 | version_info = tuple(
23 | field
24 | for field in (
25 | int(_version_fields["major"]),
26 | int(_version_fields["minor"]),
27 | int(_version_fields["patch"]),
28 | _version_fields["pre"],
29 | _version_fields["dev"],
30 | )
31 | if field is not None
32 | )
33 |
34 | __all__ = ["__version__", "version_info"]
35 |
--------------------------------------------------------------------------------
/ipyparallel/tests/test_util.py:
--------------------------------------------------------------------------------
1 | import socket
2 |
3 | import pytest
4 | from jupyter_client.localinterfaces import localhost, public_ips
5 |
6 | from ipyparallel import util
7 |
8 |
9 | def test_disambiguate_ip():
10 | # garbage in, garbage out
11 | assert util.disambiguate_ip_address('garbage') == 'garbage'
12 | assert util.disambiguate_ip_address('0.0.0.0', socket.gethostname()) == localhost()
13 | wontresolve = 'this.wontresolve.dns'
14 | with pytest.warns(
15 | RuntimeWarning, match=f"IPython could not determine IPs for {wontresolve}"
16 | ):
17 | assert util.disambiguate_ip_address('0.0.0.0', wontresolve) == wontresolve
18 | if public_ips():
19 | public_ip = public_ips()[0]
20 | assert util.disambiguate_ip_address('0.0.0.0', public_ip) == localhost()
21 |
--------------------------------------------------------------------------------
/docs/source/redirects.txt:
--------------------------------------------------------------------------------
1 | "asyncresult.rst" "tutorial/asyncresult.md"
2 | "dag_dependencies.rst" "reference/dag_dependencies.md"
3 | "db.rst" "reference/db.md"
4 | "demos.rst" "tutorial/demos.md"
5 | "details.rst" "reference/details.md"
6 | "development/connections.rst" "reference/connections.md"
7 | "development/messages.rst" "reference/messages.md"
8 | "intro.rst" "tutorial/intro.md"
9 | "magics.rst" "examples/Parallel Magics.ipynb"
10 | "tutorial/magics.md" "examples/Parallel Magics.ipynb"
11 | "mpi.rst" "reference/mpi.md"
12 | "multiengine.rst" "tutorial/direct.md"
13 | "process.rst" "tutorial/process.md"
14 | "security.rst" "reference/security.md"
15 | "task.rst" "tutorial/task.md"
16 | "transition.rst" "tutorial/index.md"
17 | "winhpc.rst" "reference/launchers.md"
18 |
19 | "examples/Index.ipynb" "examples/index.md"
20 |
--------------------------------------------------------------------------------
/lab/src/commands.ts:
--------------------------------------------------------------------------------
1 | export namespace CommandIDs {
2 | /**
3 | * Inject client code into the active editor.
4 | */
5 | export const injectClientCode = "ipyparallel:inject-client-code";
6 |
7 | /**
8 | * Launch a new cluster.
9 | */
10 | export const newCluster = "ipyparallel:new-cluster";
11 |
12 | /**
13 | * Launch a new cluster.
14 | */
15 | export const startCluster = "ipyparallel:start-cluster";
16 |
17 | /**
18 | * Shutdown a cluster.
19 | */
20 | export const stopCluster = "ipyparallel:stop-cluster";
21 |
22 | /**
23 | * Scale a cluster.
24 | */
25 | export const scaleCluster = "ipyparallel:scale-cluster";
26 |
27 | /**
28 | * Toggle the auto-starting of clients.
29 | */
30 | export const toggleAutoStartClient = "ipyparallel:toggle-auto-start-client";
31 | }
32 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | # dependabot.yaml reference: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
2 | version: 2
3 | updates:
4 | # Maintain dependencies in our GitHub Workflows
5 | - package-ecosystem: github-actions
6 | directory: "/"
7 | schedule:
8 | interval: monthly
9 |
10 | # jupyterlab extension
11 | - package-ecosystem: npm
12 | directory: "/"
13 | schedule:
14 | interval: monthly
15 | groups:
16 | # one big pull request
17 | lab-minor:
18 | patterns:
19 | - "*"
20 | update-types:
21 | - minor
22 | - patch
23 | # group major bumps
24 | lab-major:
25 | patterns:
26 | - "*"
27 | update-types:
28 | - major
29 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "allowSyntheticDefaultImports": true,
4 | "composite": true,
5 | "declaration": true,
6 | "esModuleInterop": true,
7 | "incremental": true,
8 | "lib": [
9 | "DOM",
10 | "DOM.Iterable",
11 | "ES2018",
12 | "ES2020.BigInt",
13 | "ES2020.Intl",
14 | "ES2020.String"
15 | ],
16 | "jsx": "react",
17 | "module": "esnext",
18 | "moduleResolution": "node",
19 | "noEmitOnError": true,
20 | "noImplicitAny": true,
21 | "noUnusedLocals": true,
22 | "preserveWatchOutput": true,
23 | "resolveJsonModule": true,
24 | "outDir": "lab/lib",
25 | "rootDir": "lab/src",
26 | "strict": true,
27 | "strictNullChecks": false,
28 | "target": "es2018",
29 | "types": []
30 | },
31 | "include": ["lab/src/*"]
32 | }
33 |
--------------------------------------------------------------------------------
/benchmarks/logger.py:
--------------------------------------------------------------------------------
1 | import os
2 | from datetime import date
3 |
4 | from ipyparallel_master_project.benchmarks.utils import get_time_stamp
5 |
6 | LOGS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logs')
7 | GCLOUD_DIR = os.path.join(LOGS_DIR, 'gcloud_output')
8 | PROFILING_DIR = os.path.join(LOGS_DIR, 'profiling')
9 | TODAY = str(date.today())
10 |
11 |
12 | def get_dir(main_dir):
13 | target_dir = f"{main_dir}/{TODAY}"
14 | if not os.path.exists(target_dir):
15 | os.makedirs(target_dir)
16 | return target_dir
17 |
18 |
19 | def get_profiling_log_file_name():
20 | return os.path.join(get_dir(PROFILING_DIR), f'profiling_{get_time_stamp()}')
21 |
22 |
23 | def get_gcloud_log_file_name(instance_name):
24 | return os.path.join(get_dir(GCLOUD_DIR), instance_name)
25 |
26 |
27 | if __name__ == "__main__":
28 | print(LOGS_DIR)
29 |
--------------------------------------------------------------------------------
/ci/slurm/etc_slurm/slurmdbd.conf:
--------------------------------------------------------------------------------
1 | #
2 | # Example slurmdbd.conf file.
3 | #
4 | # See the slurmdbd.conf man page for more information.
5 | #
6 | # Archive info
7 | #ArchiveJobs=yes
8 | #ArchiveDir="/tmp"
9 | #ArchiveSteps=yes
10 | #ArchiveScript=
11 | #JobPurge=12
12 | #StepPurge=1
13 | #
14 | # Authentication info
15 | AuthType=auth/munge
16 | #AuthInfo=/var/run/munge/munge.socket.2
17 | #
18 | # slurmDBD info
19 | DbdAddr=slurmdbd
20 | DbdHost=slurmdbd
21 | #DbdPort=6819
22 | SlurmUser=slurm
23 | #MessageTimeout=300
24 | DebugLevel=4
25 | #DefaultQOS=normal,standby
26 | LogFile=/var/log/slurm/slurmdbd.log
27 | PidFile=/var/run/slurmdbd/slurmdbd.pid
28 | #PluginDir=/usr/lib/slurm
29 | #PrivateData=accounts,users,usage,jobs
30 | #TrackWCKey=yes
31 | #
32 | # Database info
33 | StorageType=accounting_storage/mysql
34 | StorageHost=mysql
35 | StorageUser=slurm
36 | StoragePass=password
37 | #StorageLoc=slurm_acct_db
38 |
--------------------------------------------------------------------------------
/lab/style/logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
21 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Interactive Parallel Computing with IPython
2 |
3 | IPython Parallel (`ipyparallel`) is a Python package and collection of CLI scripts for controlling clusters of IPython processes, built on the Jupyter protocol.
4 |
5 | IPython Parallel provides the following commands:
6 |
7 | - ipcluster - start/stop/list clusters
8 | - ipcontroller - start a controller
9 | - ipengine - start an engine
10 |
11 | ## Install
12 |
13 | Install IPython Parallel:
14 |
15 | pip install ipyparallel
16 |
17 | This will install and enable the IPython Parallel extensions
18 | for Jupyter Notebook and (as of 7.0) Jupyter Lab 3.0.
19 |
20 | ## Run
21 |
22 | Start a cluster:
23 |
24 | ipcluster start
25 |
26 | Use it from Python:
27 |
28 | ```python
29 | import os
30 | import ipyparallel as ipp
31 |
32 | cluster = ipp.Cluster(n=4)
33 | with cluster as rc:
34 | ar = rc[:].apply_async(os.getpid)
35 | pid_map = ar.get_dict()
36 | ```
37 |
38 | See [the docs](https://ipyparallel.readthedocs.io) for more info.
39 |
--------------------------------------------------------------------------------
/benchmarks/profiling/view_profiling_results.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | master_project_path = os.path.abspath(
4 | os.path.join(os.path.dirname(__file__), os.pardir)
5 | )
6 | ALL_RESULTS_DIRECTORY = os.path.join(master_project_path, 'results', 'profiling')
7 |
8 |
9 | def get_latest_results_dir():
10 | return os.path.join(
11 | 'results',
12 | 'profiling',
13 | max(
14 | dirname
15 | for dirname in os.listdir(ALL_RESULTS_DIRECTORY)
16 | if 'initial_results' not in dirname
17 | ),
18 | )
19 |
20 |
21 | def get_initial_results_dir():
22 | return os.path.join(
23 | 'results',
24 | 'profiling',
25 | next(
26 | (
27 | dirname
28 | for dirname in os.listdir(ALL_RESULTS_DIRECTORY)
29 | if 'initial_results' in dirname
30 | ),
31 | max(dirname for dirname in os.listdir(ALL_RESULTS_DIRECTORY)),
32 | ),
33 | )
34 |
35 |
36 | if __name__ == '__main__':
37 | print(get_latest_results_dir())
38 |
--------------------------------------------------------------------------------
/docs/make.bat:
--------------------------------------------------------------------------------
1 | @ECHO OFF
2 |
3 | pushd %~dp0
4 |
5 | REM Command file for Sphinx documentation
6 |
7 | if "%SPHINXBUILD%" == "" (
8 | set SPHINXBUILD=--color -W --keep-going
9 | )
10 | if "%SPHINXBUILD%" == "" (
11 | set SPHINXBUILD=sphinx-build
12 | )
13 | set SOURCEDIR=source
14 | set BUILDDIR=_build
15 |
16 | if "%1" == "" goto help
17 | if "%1" == "devenv" goto devenv
18 | goto default
19 |
20 |
21 | :default
22 | %SPHINXBUILD% >NUL 2>NUL
23 | if errorlevel 9009 (
24 | echo.
25 | echo.The 'sphinx-build' command was not found. Open and read README.md!
26 | exit /b 1
27 | )
28 | %SPHINXBUILD% -M %1 "%SOURCEDIR%" "%BUILDDIR%" %SPHINXOPTS%
29 | goto end
30 |
31 |
32 | :help
33 | %SPHINXBUILD% -M help "%SOURCEDIR%" "%BUILDDIR%" %SPHINXOPTS%
34 | goto end
35 |
36 |
37 | :devenv
38 | sphinx-autobuild >NUL 2>NUL
39 | if errorlevel 9009 (
40 | echo.
41 | echo.The 'sphinx-autobuild' command was not found. Open and read README.md!
42 | exit /b 1
43 | )
44 | sphinx-autobuild -b html --open-browser "%SOURCEDIR%" "%BUILDDIR%/html"
45 | goto end
46 |
47 |
48 | :end
49 | popd
50 |
--------------------------------------------------------------------------------
/ipyparallel/nbextension/install.py:
--------------------------------------------------------------------------------
1 | """Install the IPython clusters tab in the Jupyter notebook dashboard
2 |
3 | Only applicable for notebook < 7
4 | """
5 |
6 | # Copyright (c) IPython Development Team.
7 | # Distributed under the terms of the Modified BSD License.
8 |
9 |
10 | def install_extensions(enable=True, user=False):
11 | """Register ipyparallel clusters tab as notebook extensions
12 |
13 | Toggle with enable=True/False.
14 | """
15 | import notebook # noqa
16 |
17 | from notebook.nbextensions import (
18 | disable_nbextension,
19 | enable_nbextension,
20 | install_nbextension_python,
21 | )
22 | from notebook.serverextensions import toggle_serverextension_python
23 |
24 | toggle_serverextension_python('ipyparallel', user=user)
25 | install_nbextension_python('ipyparallel', user=user)
26 | if enable:
27 | enable_nbextension('tree', 'ipyparallel/main', user=user)
28 | else:
29 | disable_nbextension('tree', 'ipyparallel/main')
30 |
31 |
32 | install_server_extension = install_extensions
33 |
--------------------------------------------------------------------------------
/ipyparallel/joblib.py:
--------------------------------------------------------------------------------
1 | """IPython parallel backend for joblib
2 |
3 | To enable the default view as a backend for joblib::
4 |
5 | import ipyparallel as ipp
6 | ipp.register_joblib_backend()
7 |
8 | Or to enable a particular View you have already set up::
9 |
10 | view.register_joblib_backend()
11 |
12 | At this point, you can use it with::
13 |
14 | with parallel_backend('ipyparallel'):
15 | Parallel(n_jobs=2)(delayed(some_function)(i) for i in range(10))
16 |
17 | .. versionadded:: 5.1
18 | """
19 |
20 | # Copyright (c) IPython Development Team.
21 | # Distributed under the terms of the Modified BSD License.
22 | from joblib.parallel import register_parallel_backend
23 |
24 | from .client._joblib import IPythonParallelBackend
25 |
26 |
27 | def register(name='ipyparallel', make_default=False):
28 | """Register the default ipyparallel Client as a joblib backend
29 |
30 | See joblib.parallel.register_parallel_backend for details.
31 | """
32 | return register_parallel_backend(
33 | name, IPythonParallelBackend, make_default=make_default
34 | )
35 |
--------------------------------------------------------------------------------
/benchmarks/explore/control_flow.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | import ipyparallel as ipp
4 |
5 |
6 | def main():
7 | client = ipp.Client(profile='asv')
8 | # client.debug = True
9 | direct_view = client[:]
10 | direct_view.targets = list(range(5))
11 | direct_view_result = direct_view.apply(lambda x: x * 2, 13)
12 | print(direct_view_result.get())
13 | # load_balanced_view = client.load_balanced_view()
14 | # result = load_balanced_view.apply(lambda x: x * 2, 13)
15 | # print(result.get())
16 | broadcast_view = client.broadcast_view(is_coalescing=True)
17 | broadcast_result = broadcast_view.apply_sync(
18 | lambda x: x, np.array([0] * 8, dtype=np.int8)
19 | )
20 |
21 | print(broadcast_result)
22 | print(len(broadcast_result))
23 | # broadcast_view2 = client.broadcast_view(is_coalescing=False)
24 | # broadcast_result = broadcast_view2.apply_sync(
25 | # lambda x: x, np.array([0] * 8, dtype=np.int8)
26 | # )
27 | #
28 | # print(broadcast_result)
29 | # print(len(broadcast_result))
30 |
31 |
32 | if __name__ == '__main__':
33 | main()
34 |
--------------------------------------------------------------------------------
/ipyparallel/tests/test_async.py:
--------------------------------------------------------------------------------
1 | """tests for async utilities"""
2 |
3 | import pytest
4 |
5 | from ipyparallel._async import AsyncFirst
6 |
7 |
8 | class A(AsyncFirst):
9 | a = 5
10 |
11 | def sync_method(self):
12 | return 'sync'
13 |
14 | async def async_method(self):
15 | return 'async'
16 |
17 |
18 | def test_async_first_dir():
19 | a = A()
20 | attrs = sorted(dir(a))
21 | real_attrs = [a for a in attrs if not a.startswith("_")]
22 | assert real_attrs == ['a', 'async_method', 'async_method_sync', 'sync_method']
23 |
24 |
25 | def test_getattr():
26 | a = A()
27 | assert a.a == 5
28 | with pytest.raises(AttributeError):
29 | a.a_sync
30 | with pytest.raises(AttributeError):
31 | a.some_other_sync
32 | with pytest.raises(AttributeError):
33 | a.sync_method_sync
34 |
35 |
36 | def test_sync_no_asyncio():
37 | a = A()
38 | assert a.async_method_sync() == 'async'
39 | assert a._async_thread is None
40 |
41 |
42 | async def test_sync_asyncio():
43 | a = A()
44 | assert a.async_method_sync() == 'async'
45 | assert a._async_thread is not None
46 |
--------------------------------------------------------------------------------
/docs/source/api/ipyparallel.rst:
--------------------------------------------------------------------------------
1 | API Reference
2 | =============
3 |
4 | .. module:: ipyparallel
5 |
6 | .. autodata:: version_info
7 |
8 | The IPython parallel version as a tuple of integers.
9 | There will always be 3 integers. Development releases will have 'dev' as a fourth element.
10 |
11 | Classes
12 | -------
13 |
14 | .. autoconfigurable:: Cluster
15 | :inherited-members:
16 |
17 | .. autoclass:: Client
18 | :inherited-members:
19 |
20 | .. autoclass:: DirectView
21 | :inherited-members:
22 |
23 | .. autoclass:: LoadBalancedView
24 | :inherited-members:
25 |
26 | .. autoclass:: BroadcastView
27 | :inherited-members:
28 |
29 | .. autoclass:: AsyncResult
30 | :inherited-members:
31 |
32 | .. autoclass:: ViewExecutor
33 | :inherited-members:
34 |
35 | Decorators
36 | ----------
37 |
38 | IPython parallel provides some decorators to assist in using your functions as tasks.
39 |
40 | .. autodecorator:: interactive
41 | .. autodecorator:: require
42 | .. autodecorator:: depend
43 | .. autodecorator:: remote
44 | .. autodecorator:: parallel
45 |
46 |
47 | Exceptions
48 | ----------
49 |
50 | .. autoexception:: RemoteError
51 | :no-members:
52 | .. autoexception:: CompositeError
53 | :no-members:
54 | .. autoexception:: NoEnginesRegistered
55 | .. autoexception:: ImpossibleDependency
56 | .. autoexception:: InvalidDependency
57 |
--------------------------------------------------------------------------------
/docs/source/examples/rmt/rmtkernel.py:
--------------------------------------------------------------------------------
1 | # -------------------------------------------------------------------------------
2 | # Core routines for computing properties of symmetric random matrices.
3 | # -------------------------------------------------------------------------------
4 | import numpy as np
5 |
6 | ra = np.random
7 | la = np.linalg
8 |
9 |
10 | def GOE(N):
11 | """Creates an NxN element of the Gaussian Orthogonal Ensemble"""
12 | m = ra.standard_normal((N, N))
13 | m += m.T
14 | return m / 2
15 |
16 |
17 | def center_eigenvalue_diff(mat):
18 | """Compute the eigvals of mat and then find the center eigval difference."""
19 | N = len(mat)
20 | evals = np.sort(la.eigvals(mat))
21 | diff = np.abs(evals[N / 2] - evals[N / 2 - 1])
22 | return diff
23 |
24 |
25 | def ensemble_diffs(num, N):
26 | """Return num eigenvalue diffs for the NxN GOE ensemble."""
27 | diffs = np.empty(num)
28 | for i in range(num):
29 | mat = GOE(N)
30 | diffs[i] = center_eigenvalue_diff(mat)
31 | return diffs
32 |
33 |
34 | def normalize_diffs(diffs):
35 | """Normalize an array of eigenvalue diffs."""
36 | return diffs / diffs.mean()
37 |
38 |
39 | def normalized_ensemble_diffs(num, N):
40 | """Return num *normalized* eigenvalue diffs for the NxN GOE ensemble."""
41 | diffs = ensemble_diffs(num, N)
42 | return normalize_diffs(diffs)
43 |
--------------------------------------------------------------------------------
/docs/source/examples/phistogram.py:
--------------------------------------------------------------------------------
1 | """Parallel histogram function"""
2 |
3 | from ipyparallel import Reference
4 |
5 |
6 | def phistogram(view, a, bins=10, rng=None, normed=False):
7 | """Compute the histogram of a remote array a.
8 |
9 | Parameters
10 | ----------
11 | view
12 | IPython DirectView instance
13 | a : str
14 | String name of the remote array
15 | bins : int
16 | Number of histogram bins
17 | rng : (float, float)
18 | Tuple of min, max of the range to histogram
19 | normed : boolean
20 | Should the histogram counts be normalized to 1
21 | """
22 | nengines = len(view.targets)
23 |
24 | # view.push(dict(bins=bins, rng=rng))
25 | with view.sync_imports():
26 | import numpy
27 | rets = view.apply_sync(
28 | lambda a, b, rng: numpy.histogram(a, b, rng), Reference(a), bins, rng
29 | )
30 | hists = [r[0] for r in rets]
31 | lower_edges = [r[1] for r in rets]
32 | # view.execute('hist, lower_edges = numpy.histogram(%s, bins, rng)' % a)
33 | lower_edges = view.pull('lower_edges', targets=0)
34 | hist_array = numpy.array(hists).reshape(nengines, -1)
35 | # hist_array.shape = (nengines,-1)
36 | total_hist = numpy.sum(hist_array, 0)
37 | if normed:
38 | total_hist = total_hist / numpy.sum(total_hist, dtype=float)
39 | return total_hist, lower_edges
40 |
--------------------------------------------------------------------------------
/ci/slurm/docker-compose.yaml:
--------------------------------------------------------------------------------
1 | services:
2 | mysql:
3 | image: mariadb:10.11
4 | hostname: mysql
5 | container_name: mysql
6 | environment:
7 | MYSQL_RANDOM_ROOT_PASSWORD: "yes"
8 | MYSQL_DATABASE: slurm_acct_db
9 | MYSQL_USER: slurm
10 | MYSQL_PASSWORD: password
11 | volumes:
12 | - var_lib_mysql:/var_lib/mysql
13 |
14 | slurmdbd:
15 | image: ipp-cluster:slurm
16 | build: .
17 | command:
18 | - slurmdbd
19 | container_name: slurmdbd
20 | hostname: slurmdbd
21 | volumes:
22 | - etc_munge:/etc/munge
23 | - var_log_slurm:/var/log/slurm
24 | depends_on:
25 | - mysql
26 |
27 | slurmctld:
28 | image: ipp-cluster:slurm
29 | build: .
30 | command:
31 | - slurmctld
32 | container_name: slurmctld
33 | hostname: slurmctld
34 | volumes:
35 | - etc_munge:/etc/munge
36 | - slurm_jobdir:/data
37 | - var_log_slurm:/var/log/slurm
38 | - ../..:/io
39 | depends_on:
40 | - slurmdbd
41 |
42 | compute:
43 | image: ipp-cluster:slurm
44 | build: .
45 | command:
46 | - slurmd
47 | deploy:
48 | replicas: 2
49 |
50 | volumes:
51 | - etc_munge:/etc/munge
52 | - slurm_jobdir:/data
53 | - var_log_slurm:/var/log/slurm
54 | - ../..:/io
55 | depends_on:
56 | - "slurmctld"
57 |
58 | volumes:
59 | etc_munge:
60 | # etc_slurm:
61 | slurm_jobdir:
62 | var_lib_mysql:
63 | var_log_slurm:
64 |
--------------------------------------------------------------------------------
/hatch_build.py:
--------------------------------------------------------------------------------
1 | """Custom build script for hatch backend"""
2 |
3 | import glob
4 | import os
5 | import subprocess
6 |
7 | from hatchling.builders.hooks.plugin.interface import BuildHookInterface
8 |
9 |
10 | class CustomHook(BuildHookInterface):
11 | def initialize(self, version, build_data):
12 | if self.target_name not in ["wheel", "sdist"]:
13 | return
14 | cmd = "build:prod" if version == "standard" else "build"
15 | osp = os.path
16 | here = osp.abspath(osp.dirname(__file__))
17 | lab_path = osp.join(here, 'ipyparallel', 'labextension')
18 |
19 | if os.environ.get("IPP_DISABLE_JS") == "1":
20 | print("Skipping js installation")
21 | return
22 |
23 | # this tells us if labextension is built at all, not if it's up-to-date
24 | labextension_built = glob.glob(osp.join(lab_path, "*"))
25 | needs_js = True
26 | if not osp.isdir(osp.join(here, ".git")):
27 | print("Installing from a dist, not a repo")
28 | # not in a repo, probably installing from sdist
29 | # could be git-archive, though!
30 | # skip rebuilding js if it's already present
31 | if labextension_built:
32 | print(f"Not regenerating labextension in {lab_path}")
33 | needs_js = False
34 |
35 | if needs_js:
36 | subprocess.check_call(['jlpm'], cwd=here)
37 | subprocess.check_call(['jlpm', 'run', cmd], cwd=here)
38 |
--------------------------------------------------------------------------------
/docs/source/examples/broadcast/docker-compose.yaml:
--------------------------------------------------------------------------------
1 | services:
2 | # need /ipython to be writable by the container user (1000)
3 | # can't seem to start volumes with the right permissions
4 | init-permissions:
5 | image: alpine
6 | volumes:
7 | - ipython:/ipython
8 | command: chown -Rv 1000:1000 /ipython
9 |
10 | controller:
11 | depends_on:
12 | - init-permissions
13 | # condition: service_completed_successfully
14 |
15 | build: .
16 | volumes:
17 | - ipython:/ipython
18 | environment:
19 | IPYTHONDIR: /ipython
20 | command: |
21 | ipcontroller --location controller --ip 0.0.0.0 --nodb --HubFactory.broadcast_scheduler_depth=2
22 | deploy:
23 | restart_policy:
24 | condition: on-failure
25 | engines:
26 | build: .
27 | depends_on:
28 | - controller
29 | volumes_from:
30 | - controller
31 | environment:
32 | IPYTHONDIR: /ipython
33 | links:
34 | - controller
35 | command: |
36 | ipcluster engines -n 16
37 | deploy:
38 | mode: replicated
39 | replicas: 4
40 | restart_policy:
41 | condition: on-failure
42 |
43 | client:
44 | build: .
45 | ports:
46 | - "9999:8888"
47 | links:
48 | - controller
49 | volumes_from:
50 | - controller
51 | volumes:
52 | - .:/home/jovyan/example
53 | environment:
54 | IPYTHONDIR: /ipython
55 | JUPYTER_CONFIG_DIR: /ipython/jupyter
56 |
57 | volumes:
58 | ipython: {}
59 |
--------------------------------------------------------------------------------
/ci/ssh/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM ubuntu:24.04
2 | RUN --mount=type=cache,target=/var/cache/apt \
3 | rm -f /etc/apt/apt.conf.d/docker-clean \
4 | && apt-get update \
5 | && apt-get -y install \
6 | iputils-ping \
7 | bind9-utils \
8 | wget \
9 | openssh-server
10 |
11 | ENV MAMBA_ROOT_PREFIX=/opt/conda
12 | ENV PATH=$MAMBA_ROOT_PREFIX/bin:$PATH
13 | ENV IPP_DISABLE_JS=1
14 | # x86_64 -> 64, aarch64 unmodified
15 | RUN ARCH=$(uname -m | sed s@x86_@@) \
16 | && wget -qO- https://github.com/mamba-org/micromamba-releases/releases/latest/download/micromamba-linux-${ARCH} > /usr/local/bin/micromamba \
17 | && chmod +x /usr/local/bin/micromamba
18 |
19 | RUN --mount=type=cache,target=${MAMBA_ROOT_PREFIX}/pkgs \
20 | micromamba install -y -p $MAMBA_ROOT_PREFIX -c conda-forge \
21 | python=3.10 \
22 | pip \
23 | ipyparallel
24 |
25 | # generate a user with home directory and trusted ssh keypair
26 | RUN useradd -m -s /bin/bash -N ciuser
27 | USER ciuser
28 | RUN mkdir ~/.ssh \
29 | && chmod 0700 ~/.ssh \
30 | && ssh-keygen -q -t rsa -N '' -f /home/ciuser/.ssh/id_rsa \
31 | && cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys \
32 | && chmod 0600 ~/.ssh/*
33 | USER root
34 |
35 |
36 | ENV PIP_CACHE_DIR=/tmp/pip-cache
37 | COPY . /src/ipyparallel
38 | RUN --mount=type=cache,target=${PIP_CACHE_DIR} python3 -m pip install -e 'file:///src/ipyparallel#egg=ipyparallel[test]'
39 |
40 | # needed for sshd to start
41 | RUN mkdir /run/sshd
42 | # run sshd in the foreground
43 | CMD /usr/sbin/sshd -D -e
44 |
--------------------------------------------------------------------------------
/docs/source/examples/index.md:
--------------------------------------------------------------------------------
1 | ---
2 | file_format: mystnb
3 | kernelspec:
4 | name: python3
5 | mystnb:
6 | execution_mode: "force"
7 | remove_code_source: true
8 | ---
9 |
10 | # examples
11 |
12 | Here you will find example notebooks and scripts for working with IPython Parallel.
13 |
14 | (example-tutorials)=
15 |
16 | ## Tutorials
17 |
18 | ```{toctree}
19 | Cluster API
20 | Parallel Magics
21 | progress
22 | Data Publication API
23 | visualizing-tasks
24 | broadcast/Broadcast view
25 | broadcast/memmap Broadcast
26 | broadcast/MPI Broadcast
27 | ```
28 |
29 | ## Examples
30 |
31 | ```{toctree}
32 | Monitoring an MPI Simulation - 1
33 | Monitoring an MPI Simulation - 2
34 | Parallel Decorator and map
35 | Using MPI with IPython Parallel
36 | Monte Carlo Options
37 | rmt/rmt
38 | ```
39 |
40 | ## Integrating IPython Parallel with other tools
41 |
42 | There are lots of cool tools for working with asynchronous and parallel execution. IPython Parallel aims to be fairly compatible with these, both by implementing explicit support via methods such as `Client.become_dask`, and by using standards such as the `concurrent.futures.Future` API.
43 |
44 | ```{toctree}
45 | Futures
46 | joblib
47 | dask
48 | Using Dill
49 | ```
50 |
51 | ## Non-notebook examples
52 |
53 | This directory also contains some examples that are scripts instead of notebooks.
54 |
55 | ```{code-cell}
56 | import glob
57 | import os
58 |
59 | from IPython.display import FileLink, FileLinks, display
60 |
61 | FileLinks(".", included_suffixes=[".py"], recursive=True)
62 | ```
63 |
--------------------------------------------------------------------------------
/docs/source/examples/interengine/interengine.py:
--------------------------------------------------------------------------------
1 | import ipyparallel as ipp
2 |
3 | rc = ipp.Client()
4 | rc.block = True
5 | view = rc[:]
6 | view.run('communicator.py')
7 | view.execute('com = EngineCommunicator()')
8 |
9 | # gather the connection information into a dict
10 | ar = view.apply_async(lambda: com.info) # noqa: F821
11 | peers = ar.get_dict()
12 | # this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators
13 |
14 | # connect the engines to each other:
15 | view.apply_sync(lambda pdict: com.connect(pdict), peers) # noqa: F821
16 |
17 | # now all the engines are connected, and we can communicate between them:
18 |
19 |
20 | def broadcast(client, sender, msg_name, dest_name=None, block=None):
21 | """broadcast a message from one engine to all others."""
22 | dest_name = msg_name if dest_name is None else dest_name
23 | client[sender].execute(f'com.publish({msg_name})', block=None)
24 | targets = client.ids
25 | targets.remove(sender)
26 | return client[targets].execute(f'{dest_name}=com.consume()', block=None)
27 |
28 |
29 | def send(client, sender, targets, msg_name, dest_name=None, block=None):
30 | """send a message from one to one-or-more engines."""
31 | dest_name = msg_name if dest_name is None else dest_name
32 |
33 | def _send(targets, m_name):
34 | msg = globals()[m_name]
35 | return com.send(targets, msg) # noqa: F821
36 |
37 | client[sender].apply_async(_send, targets, msg_name)
38 |
39 | return client[targets].execute(f'{dest_name}=com.recv()', block=None)
40 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing
2 |
3 | We follow the [Jupyter Contributing Guide](https://jupyter.readthedocs.io/en/latest/contributing/content-contributor.html).
4 |
5 | Make sure to follow the [Jupyter Code of Conduct](https://jupyter.org/conduct/).
6 |
7 | ## Development install
8 |
9 | A basic development install of IPython parallel
10 | is the same as almost all Python packages:
11 |
12 | ```bash
13 | pip install -e .
14 | ```
15 |
16 | To enable the server extension from a development install:
17 |
18 | ```bash
19 | # for jupyterlab
20 | jupyter server extension enable --sys-prefix ipyparallel
21 | # for classic notebook
22 | jupyter serverextension enable --sys-prefix ipyparallel
23 | ```
24 |
25 | As described in the [JupyterLab documentation](https://jupyterlab.readthedocs.io/en/stable/extension/extension_dev.html#developing-a-prebuilt-extension)
26 | for a development install of the jupyterlab extension you can run the following in this directory:
27 |
28 | ```bash
29 | jlpm # Install npm package dependencies
30 | jlpm build # Compile the TypeScript sources to Javascript
31 | jupyter labextension develop . --overwrite # Install the current directory as an extension
32 | ```
33 |
34 | If you are working on the lab extension,
35 | you can run jupyterlab in dev mode to always rebuild and reload your extensions.
36 | In two terminals, run:
37 |
38 | ```bash
39 | [term 1] $ jlpm watch
40 | [term 2] $ jupyter lab --extensions-in-dev-mode
41 | ```
42 |
43 | You should then be able to refresh the JupyterLab page
44 | and it will pick up the changes to the extension as you work.
45 |
--------------------------------------------------------------------------------
/ipyparallel/nbextension/base.py:
--------------------------------------------------------------------------------
1 | """Place to put the base Handler"""
2 |
3 | import warnings
4 | from functools import lru_cache
5 |
6 | _APIHandler = None
7 |
8 |
9 | @lru_cache
10 | def _guess_api_handler():
11 | """Fallback to guess API handler by availability"""
12 | try:
13 | from notebook.base.handlers import APIHandler
14 | except ImportError:
15 | from jupyter_server.base.handlers import APIHandler
16 | global _APIHandler
17 | _APIHandler = APIHandler
18 | return _APIHandler
19 |
20 |
21 | def get_api_handler(app=None):
22 | """Get the base APIHandler class to use
23 |
24 | Inferred from app class (either jupyter_server or notebook app)
25 | """
26 | global _APIHandler
27 | if _APIHandler is not None:
28 | return _APIHandler
29 | if app is None:
30 | warnings.warn(
31 | "Guessing base APIHandler class. Specify an app to ensure the right APIHandler is used.",
32 | stacklevel=2,
33 | )
34 | return _guess_api_handler()
35 |
36 | top_modules = {cls.__module__.split(".", 1)[0] for cls in app.__class__.mro()}
37 | if "jupyter_server" in top_modules:
38 | from jupyter_server.base.handlers import APIHandler
39 |
40 | _APIHandler = APIHandler
41 | return APIHandler
42 | if "notebook" in top_modules:
43 | from notebook.base.handlers import APIHandler
44 |
45 | _APIHandler = APIHandler
46 | return APIHandler
47 |
48 | warnings.warn(f"Failed to detect base APIHandler class for {app}.", stacklevel=2)
49 | return _guess_api_handler()
50 |
--------------------------------------------------------------------------------
/ipyparallel/tests/test_slurm.py:
--------------------------------------------------------------------------------
1 | import shutil
2 | from unittest import mock
3 |
4 | import pytest
5 | from traitlets.config import Config
6 |
7 | from . import test_cluster
8 | from .conftest import temporary_ipython_dir
9 | from .test_cluster import (
10 | test_get_output, # noqa: F401
11 | test_restart_engines, # noqa: F401
12 | test_signal_engines, # noqa: F401
13 | test_start_stop_cluster, # noqa: F401
14 | test_to_from_dict, # noqa: F401
15 | )
16 |
17 |
18 | @pytest.fixture(autouse=True, scope="module")
19 | def longer_timeout():
20 | # slurm tests started failing with timeouts
21 | # when adding timeout to test_restart_engines
22 | # maybe it's just slow...
23 | with mock.patch.object(test_cluster, "_timeout", 120):
24 | yield
25 |
26 |
27 | # put ipython dir on shared filesystem
28 | @pytest.fixture(autouse=True, scope="module")
29 | def ipython_dir(request):
30 | if shutil.which("sbatch") is None:
31 | pytest.skip("Requires slurm")
32 | with temporary_ipython_dir(prefix="/data/") as ipython_dir:
33 | yield ipython_dir
34 |
35 |
36 | @pytest.fixture
37 | def cluster_config():
38 | c = Config()
39 | c.Cluster.controller_ip = '0.0.0.0'
40 | return c
41 |
42 |
43 | # override launcher classes
44 | @pytest.fixture
45 | def engine_launcher_class():
46 | if shutil.which("sbatch") is None:
47 | pytest.skip("Requires slurm")
48 | return 'slurm'
49 |
50 |
51 | @pytest.fixture
52 | def controller_launcher_class():
53 | if shutil.which("sbatch") is None:
54 | pytest.skip("Requires slurm")
55 | return 'slurm'
56 |
--------------------------------------------------------------------------------
/ipyparallel/tests/test_remotefunction.py:
--------------------------------------------------------------------------------
1 | """Tests for remote functions"""
2 |
3 | # Copyright (c) IPython Development Team.
4 | # Distributed under the terms of the Modified BSD License.
5 | import ipyparallel as ipp
6 |
7 | from .clienttest import ClusterTestCase
8 |
9 |
10 | class TestRemoteFunctions(ClusterTestCase):
11 | def test_remote(self):
12 | v = self.client[-1]
13 |
14 | @ipp.remote(v, block=True)
15 | def foo(x, y=5):
16 | """multiply x * y"""
17 | return x * y
18 |
19 | assert foo.__name__ == 'foo'
20 | assert 'RemoteFunction' in foo.__doc__
21 | assert 'multiply x' in foo.__doc__
22 |
23 | z = foo(5)
24 | assert z == 25
25 | z = foo(2, 3)
26 | assert z == 6
27 | z = foo(x=5, y=2)
28 | assert z == 10
29 |
30 | def test_parallel(self):
31 | n = 2
32 | v = self.client[:n]
33 |
34 | @ipp.parallel(v, block=True)
35 | def foo(x):
36 | """multiply x * y"""
37 | return x * 2
38 |
39 | assert foo.__name__ == 'foo'
40 | assert 'ParallelFunction' in foo.__doc__
41 | assert 'multiply x' in foo.__doc__
42 |
43 | z = foo([1, 2, 3, 4])
44 | assert z, [1, 2, 1, 2, 3, 4, 3 == 4]
45 |
46 | def test_parallel_map(self):
47 | v = self.client.load_balanced_view()
48 |
49 | @ipp.parallel(v, block=True)
50 | def foo(x, y=5):
51 | """multiply x * y"""
52 | return x * y
53 |
54 | z = foo.map([1, 2, 3])
55 | assert z, [5, 10 == 15]
56 | z = foo.map([1, 2, 3], [1, 2, 3])
57 | assert z, [1, 4 == 9]
58 |
--------------------------------------------------------------------------------
/ipyparallel/tests/test_mongodb.py:
--------------------------------------------------------------------------------
1 | """Tests for mongodb backend"""
2 |
3 | # Copyright (c) IPython Development Team.
4 | # Distributed under the terms of the Modified BSD License.
5 | import os
6 | from unittest import TestCase
7 |
8 | import pytest
9 |
10 | from . import test_db
11 |
12 | c = None
13 |
14 |
15 | @pytest.fixture(scope='module')
16 | def mongo_conn(request):
17 | global c
18 | try:
19 | from pymongo import MongoClient
20 | except ImportError:
21 | pytest.skip("Requires mongodb")
22 |
23 | conn_kwargs = {}
24 | if 'DB_IP' in os.environ:
25 | conn_kwargs['host'] = os.environ['DB_IP']
26 | if 'DBA_MONGODB_ADMIN_URI' in os.environ:
27 | # On ShiningPanda, we need a username and password to connect. They are
28 | # passed in a mongodb:// URI.
29 | conn_kwargs['host'] = os.environ['DBA_MONGODB_ADMIN_URI']
30 | if 'DB_PORT' in os.environ:
31 | conn_kwargs['port'] = int(os.environ['DB_PORT'])
32 |
33 | try:
34 | c = MongoClient(**conn_kwargs, serverSelectionTimeoutMS=2000)
35 | servinfo = c.server_info()
36 | except Exception:
37 | c = None
38 | if c is not None:
39 | request.addfinalizer(lambda: c.drop_database('iptestdb'))
40 | return c
41 |
42 |
43 | @pytest.mark.usefixtures('mongo_conn')
44 | class TestMongoBackend(test_db.TaskDBTest, TestCase):
45 | """MongoDB backend tests"""
46 |
47 | def create_db(self):
48 | try:
49 | from ipyparallel.controller.mongodb import MongoDB
50 |
51 | return MongoDB(database='iptestdb', _connection=c)
52 | except Exception:
53 | pytest.skip("Couldn't connect to mongodb")
54 |
--------------------------------------------------------------------------------
/ipyparallel/tests/test_joblib.py:
--------------------------------------------------------------------------------
1 | import warnings
2 | from unittest import mock
3 |
4 | import pytest
5 |
6 | import ipyparallel as ipp
7 |
8 | from .clienttest import ClusterTestCase, add_engines
9 |
10 | try:
11 | with warnings.catch_warnings():
12 | warnings.simplefilter("ignore")
13 | from joblib import Parallel, delayed
14 |
15 | from ipyparallel.client._joblib import IPythonParallelBackend # noqa
16 | except (ImportError, TypeError):
17 | have_joblib = False
18 | else:
19 | have_joblib = True
20 |
21 |
22 | def neg(x):
23 | return -1 * x
24 |
25 |
26 | class TestJobLib(ClusterTestCase):
27 | def setup_method(self):
28 | if not have_joblib:
29 | pytest.skip("Requires joblib >= 0.10")
30 | super().setup_method()
31 | add_engines(1, total=True)
32 |
33 | def test_default_backend(self):
34 | """ipyparallel.register_joblib_backend() registers default backend"""
35 | ipp.register_joblib_backend()
36 | with mock.patch.object(ipp.Client, "__new__", lambda *a, **kw: self.client):
37 | p = Parallel(backend='ipyparallel')
38 | assert p._backend._view.client is self.client
39 |
40 | def test_register_backend(self):
41 | view = self.client.load_balanced_view()
42 | view.register_joblib_backend('view')
43 | p = Parallel(backend='view')
44 | assert p._backend._view is view
45 |
46 | def test_joblib_backend(self):
47 | view = self.client.load_balanced_view()
48 | view.register_joblib_backend('view')
49 | p = Parallel(backend='view')
50 | result = p(delayed(neg)(i) for i in range(10))
51 | assert result == [neg(i) for i in range(10)]
52 |
--------------------------------------------------------------------------------
/.github/workflows/windows-ssh-image.yaml:
--------------------------------------------------------------------------------
1 | # because it's too unbelievably slow to install ssh server on Windows
2 | name: Build image for Windows SSH CI
3 |
4 | on:
5 | workflow_dispatch:
6 |
7 | env:
8 | REGISTRY: ghcr.io
9 | IMAGE_NAME: ipyparallel-windows-ssh
10 | TAG: 3.12-2022
11 |
12 | jobs:
13 | build-and-push-image:
14 | runs-on: windows-2022
15 |
16 | permissions:
17 | contents: read
18 | packages: write
19 |
20 | steps:
21 | - name: Checkout repository
22 | uses: actions/checkout@v5
23 |
24 | - name: Log in to the Container registry
25 | uses: docker/login-action@v3
26 | with:
27 | registry: ${{ env.REGISTRY }}
28 | username: ${{ github.actor }}
29 | password: ${{ secrets.GITHUB_TOKEN }}
30 |
31 | # - name: Extract metadata (tags, labels) for Docker
32 | # id: meta
33 | # uses: docker/metadata-action@v5
34 | # with:
35 | # labels: |
36 | # org.opencontainers.image.title=${{ env.IMAGE_NAME }}
37 | #
38 | # images: ${{ github.repository_owner }}/${{ env.IMAGE_NAME }}
39 | # tags: |
40 | # type=raw,value=${{ env.TAG }}
41 |
42 | - name: Build image
43 | # can't use build-push-action on Windows
44 | # https://github.com/docker/build-push-action/issues/18
45 | run: |
46 | docker build -t ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.TAG }} -f ci/ssh/win_base_Dockerfile ci/ssh
47 | - name: Push image
48 | run: |
49 | docker push ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.TAG }}
50 |
--------------------------------------------------------------------------------
/benchmarks/instance_setup.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | DEFAULT_MINICONDA_PATH = os.path.join(os.getcwd(), "miniconda3/bin/:")
4 | env = os.environ.copy()
5 | env["PATH"] = DEFAULT_MINICONDA_PATH + env["PATH"]
6 |
7 | from subprocess import check_call
8 |
9 | GITHUB_TOKEN = "" # Token for machine user
10 | ASV_TESTS_REPO = "github.com/tomoboy/ipyparallel_master_project.git"
11 | IPYPARALLEL_REPO = "github.com/tomoboy/ipyparallel.git"
12 |
13 |
14 | def cmd_run(*args, log_filename=None, error_filename=None):
15 | if len(args) == 1:
16 | args = args[0].split(" ")
17 | print(f'$ {" ".join(args)}')
18 | if not log_filename and not error_filename:
19 | check_call(args, env=env)
20 | else:
21 | check_call(
22 | args,
23 | env=env,
24 | stdout=open(log_filename, 'w'),
25 | stderr=open(error_filename, 'w'),
26 | )
27 |
28 |
29 | if __name__ == "__main__":
30 | cmd_run(
31 | f"git clone -q https://{GITHUB_TOKEN}@{ASV_TESTS_REPO}"
32 | ) # Get benchmarks from repo
33 | print("Finished cloning benchmark repo")
34 | # Installing ipyparallel from the dev branch
35 | cmd_run(f"pip install -q git+https://{GITHUB_TOKEN}@{IPYPARALLEL_REPO}")
36 | print("Installed ipyparallel")
37 | # Create profile for ipyparallel, (should maybe be copied if we want some cusom values here)
38 | cmd_run("ipython profile create --parallel --profile=asv")
39 | cmd_run('echo 120000 > /proc/sys/kernel/threads-max')
40 | cmd_run('echo 600000 > /proc/sys/vm/max_map_count')
41 | cmd_run('echo 200000 > /proc/sys/kernel/piad_max')
42 | cmd_run('echo "* hard nproc 100000" > /etc/security/limits.d')
43 | cmd_run('echo "* soft nproc 100000" > /etc/security/limits.d')
44 |
--------------------------------------------------------------------------------
/lab/src/sidebar.ts:
--------------------------------------------------------------------------------
1 | import { Widget, PanelLayout } from "@lumino/widgets";
2 | import { CommandRegistry } from "@lumino/commands";
3 |
4 | import { ClusterManager, IClusterModel } from "./clusters";
5 |
6 | /**
7 | * A widget for hosting IPP cluster widgets
8 | */
9 | export class Sidebar extends Widget {
10 | /**
11 | * Create a new IPP sidebar.
12 | */
13 | constructor(options: Sidebar.IOptions) {
14 | super();
15 | this.addClass("ipp-Sidebar");
16 | let layout = (this.layout = new PanelLayout());
17 |
18 | const injectClientCodeForCluster = options.clientCodeInjector;
19 | const getClientCodeForCluster = options.clientCodeGetter;
20 | // Add the cluster manager component.
21 | this._clusters = new ClusterManager({
22 | registry: options.registry,
23 | injectClientCodeForCluster,
24 | getClientCodeForCluster,
25 | });
26 | layout.addWidget(this._clusters);
27 | }
28 |
29 | /**
30 | * Get the cluster manager associated with the sidebar.
31 | */
32 | get clusterManager(): ClusterManager {
33 | return this._clusters;
34 | }
35 |
36 | private _clusters: ClusterManager;
37 | }
38 |
39 | /**
40 | * A namespace for Sidebar statics.
41 | */
42 | export namespace Sidebar {
43 | /**
44 | * Options for the constructor.
45 | */
46 | export interface IOptions {
47 | /**
48 | * Registry of all commands
49 | */
50 | registry: CommandRegistry;
51 |
52 | /**
53 | * A function that injects client-connection code for a given cluster.
54 | */
55 | clientCodeInjector: (model: IClusterModel) => void;
56 |
57 | /**
58 | * A function that gets client-connection code for a given cluster.
59 | */
60 | clientCodeGetter: (model: IClusterModel) => string;
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/ci/slurm/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM ubuntu:24.04
2 |
3 | ENV DEBIAN_FRONTEND=noninteractive
4 | RUN --mount=type=cache,target=/var/cache/apt \
5 | rm -f /etc/apt/apt.conf.d/docker-clean \
6 | && apt-get update && apt-get -y install \
7 | gosu \
8 | mysql-client \
9 | python3-venv \
10 | python3-pip \
11 | slurm-wlm \
12 | slurmdbd \
13 | slurm
14 |
15 | ENV PIP_CACHE_DIR=/tmp/pip-cache \
16 | VIRTUAL_ENV=/srv/env \
17 | PATH=/srv/env/bin:${PATH} \
18 | IPP_DISABLE_JS=1
19 |
20 | RUN --mount=type=cache,target=${PIP_CACHE_DIR} \
21 | python3 -m venv $VIRTUAL_ENV \
22 | && $VIRTUAL_ENV/bin/python3 -m pip install ipyparallel pytest-asyncio pytest-cov
23 |
24 | # initialize some filesystem
25 | RUN mkdir -p /etc/sysconfig/slurm \
26 | /var/spool/slurmd \
27 | /var/run/slurmd \
28 | /var/run/slurmdbd \
29 | /var/lib/slurmd \
30 | /data \
31 | && touch /var/lib/slurmd/node_state \
32 | /var/lib/slurmd/front_end_state \
33 | /var/lib/slurmd/job_state \
34 | /var/lib/slurmd/resv_state \
35 | /var/lib/slurmd/trigger_state \
36 | /var/lib/slurmd/assoc_mgr_state \
37 | /var/lib/slurmd/assoc_usage \
38 | /var/lib/slurmd/qos_usage \
39 | /var/lib/slurmd/fed_mgr_state \
40 | && chown -R slurm:slurm /var/*/slurm* \
41 | && mkdir /run/munge \
42 | && chown munge:munge /run/munge \
43 | && chmod a+rwxt /run/munge
44 | # && mungekey -c
45 |
46 | COPY --chown=slurm:slurm --chmod=0600 etc_slurm/ /etc/slurm/
47 |
48 |
49 | COPY entrypoint.sh /entrypoint
50 | ENTRYPOINT ["/entrypoint"]
51 |
52 | # the mounted directory
53 | RUN mkdir /io
54 | ENV PYTHONPATH=/io
55 | WORKDIR "/io"
56 |
57 | CMD [ "tail", "-f", "/var/log/slurm/slurmd.log" ]
58 |
--------------------------------------------------------------------------------
/docs/source/examples/throughput.py:
--------------------------------------------------------------------------------
1 | import time
2 |
3 | import numpy as np
4 |
5 | import ipyparallel as parallel
6 |
7 | nlist = map(int, np.logspace(2, 9, 16, base=2))
8 | nlist2 = map(int, np.logspace(2, 8, 15, base=2))
9 | tlist = map(int, np.logspace(7, 22, 16, base=2))
10 | nt = 16
11 |
12 |
13 | def wait(t=0):
14 | import time
15 |
16 | time.sleep(t)
17 |
18 |
19 | def echo(s=''):
20 | return s
21 |
22 |
23 | def time_throughput(nmessages, t=0, f=wait):
24 | client = parallel.Client()
25 | view = client.load_balanced_view()
26 | # do one ping before starting timing
27 | if f is echo:
28 | t = np.random.random(t / 8)
29 | view.apply_sync(echo, '')
30 | client.spin()
31 | tic = time.time()
32 | for i in range(nmessages):
33 | view.apply(f, t)
34 | lap = time.time()
35 | client.wait()
36 | toc = time.time()
37 | return lap - tic, toc - tic
38 |
39 |
40 | def do_runs(nlist, t=0, f=wait, trials=2, runner=time_throughput):
41 | A = np.zeros((len(nlist), 2))
42 | for i, n in enumerate(nlist):
43 | t1 = t2 = 0
44 | for _ in range(trials):
45 | time.sleep(0.25)
46 | ts = runner(n, t, f)
47 | t1 += ts[0]
48 | t2 += ts[1]
49 | t1 /= trials
50 | t2 /= trials
51 | A[i] = (t1, t2)
52 | A[i] = n / A[i]
53 | print(n, A[i])
54 | return A
55 |
56 |
57 | def do_echo(n, tlist=[0], f=echo, trials=2, runner=time_throughput):
58 | A = np.zeros((len(tlist), 2))
59 | for i, t in enumerate(tlist):
60 | t1 = t2 = 0
61 | for _ in range(trials):
62 | time.sleep(0.25)
63 | ts = runner(n, t, f)
64 | t1 += ts[0]
65 | t2 += ts[1]
66 | t1 /= trials
67 | t2 /= trials
68 | A[i] = (t1, t2)
69 | A[i] = n / A[i]
70 | print(t, A[i])
71 | return A
72 |
--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | ci:
2 | autoupdate_schedule: monthly
3 |
4 | repos:
5 | - repo: https://github.com/astral-sh/ruff-pre-commit
6 | rev: v0.14.3
7 | hooks:
8 | - id: ruff
9 | args:
10 | - "--fix"
11 | exclude_types:
12 | - jupyter
13 | - id: ruff
14 | name: ruff (notebooks)
15 | args:
16 | - "--fix"
17 | types:
18 | - jupyter
19 | - id: ruff-format
20 | # run via nbqa to handle cell magics
21 | exclude_types:
22 | - jupyter
23 | # run ruff via nbqa, which has better and configurable handling of cell magics
24 | # than ruff itself
25 | # but has trouble with isort rules
26 | - repo: https://github.com/nbQA-dev/nbQA
27 | rev: 1.9.1
28 | hooks:
29 | - id: nbqa-ruff-format
30 | - id: nbqa-ruff-check
31 | args:
32 | - --fix
33 | # isort rules get confused in the temporary files
34 | - --ignore=I
35 | - repo: https://github.com/rbubley/mirrors-prettier
36 | rev: v3.6.2
37 | hooks:
38 | - id: prettier
39 | - repo: https://github.com/pre-commit/pre-commit-hooks
40 | rev: v6.0.0
41 | hooks:
42 | - id: end-of-file-fixer
43 | - id: check-case-conflict
44 | - id: check-executables-have-shebangs
45 | - id: requirements-txt-fixer
46 | - repo: https://github.com/pre-commit/mirrors-eslint
47 | rev: v9.39.0
48 | hooks:
49 | - id: eslint
50 | files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx
51 | exclude: ipyparallel/nbextension/.*
52 | types: [file]
53 | additional_dependencies:
54 | - "@typescript-eslint/eslint-plugin@2.27.0"
55 | - "@typescript-eslint/parser@2.27.0"
56 | - eslint@^6.0.0
57 | - eslint-config-prettier@6.10.1
58 | - eslint-plugin-prettier@3.1.4
59 | - eslint-plugin-react@7.21.5
60 | - typescript@4.1.3
61 |
--------------------------------------------------------------------------------
/ipyparallel/engine/datapub.py:
--------------------------------------------------------------------------------
1 | """Publishing native (typically pickled) objects."""
2 |
3 | # Copyright (c) IPython Development Team.
4 | # Distributed under the terms of the Modified BSD License.
5 | from ipykernel.jsonutil import json_clean
6 | from jupyter_client.session import Session, extract_header
7 | from traitlets import Any, CBytes, Dict, Instance
8 | from traitlets.config import Configurable
9 |
10 | from ipyparallel.serialize import serialize_object
11 |
12 |
13 | class ZMQDataPublisher(Configurable):
14 | topic = topic = CBytes(b'datapub')
15 | session = Instance(Session, allow_none=True)
16 | pub_socket = Any(allow_none=True)
17 | parent_header = Dict({})
18 |
19 | def set_parent(self, parent):
20 | """Set the parent for outbound messages."""
21 | self.parent_header = extract_header(parent)
22 |
23 | def publish_data(self, data):
24 | """publish a data_message on the IOPub channel
25 |
26 | Parameters
27 | ----------
28 | data : dict
29 | The data to be published. Think of it as a namespace.
30 | """
31 | session = self.session
32 | buffers = serialize_object(
33 | data,
34 | buffer_threshold=session.buffer_threshold,
35 | item_threshold=session.item_threshold,
36 | )
37 | content = json_clean(dict(keys=list(data.keys())))
38 | session.send(
39 | self.pub_socket,
40 | 'data_message',
41 | content=content,
42 | parent=self.parent_header,
43 | buffers=buffers,
44 | ident=self.topic,
45 | )
46 |
47 |
48 | def publish_data(data):
49 | """publish a data_message on the IOPub channel
50 |
51 | Parameters
52 | ----------
53 | data : dict
54 | The data to be published. Think of it as a namespace.
55 | """
56 | from ipykernel.zmqshell import ZMQInteractiveShell
57 |
58 | ZMQInteractiveShell.instance().data_pub.publish_data(data)
59 |
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Makefile for Sphinx documentation generated by sphinx-quickstart
2 | # ----------------------------------------------------------------------------
3 |
4 | # You can set these variables from the command line, and also
5 | # from the environment for the first two.
6 | SPHINXOPTS ?= --color -W --keep-going
7 | SPHINXBUILD ?= sphinx-build
8 | SOURCEDIR = source
9 | BUILDDIR = _build
10 |
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
14 |
15 | .PHONY: help Makefile
16 |
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option.
19 | #
20 | # Several sphinx-build commands can be used through this, for example:
21 | #
22 | # - make clean
23 | # - make linkcheck
24 | # - make spelling
25 | #
26 | %: Makefile
27 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
28 |
29 |
30 | # Manually added targets - related to code generation
31 | # ----------------------------------------------------------------------------
32 |
33 | # For local development:
34 | # - builds the html
35 | # - NOTE: If the pre-requisites for the html target is updated, also update the
36 | # Read The Docs section in docs/source/conf.py.
37 | #
38 | html:
39 | $(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS)
40 | @echo
41 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
42 |
43 |
44 | # Manually added targets - related to development
45 | # ----------------------------------------------------------------------------
46 |
47 | # For local development:
48 | # - requires sphinx-autobuild, see
49 | # https://sphinxcontrib-spelling.readthedocs.io/en/latest/
50 | # - builds and rebuilds html on changes to source, but does not re-generate
51 | # metrics/scopes files
52 | # - starts a livereload enabled webserver and opens up a browser
53 | devenv: html
54 | sphinx-autobuild -b html --open-browser "$(SOURCEDIR)" "$(BUILDDIR)/html"
55 |
--------------------------------------------------------------------------------
/ci/slurm/entrypoint.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -ex
3 |
4 | if [ "$1" = "slurmdbd" ]
5 | then
6 | echo "---> Starting the MUNGE Authentication service (munged) ..."
7 | gosu munge /usr/sbin/munged
8 |
9 | echo "---> Starting the Slurm Database Daemon (slurmdbd) ..."
10 |
11 | {
12 | . /etc/slurm/slurmdbd.conf
13 | until echo "SELECT 1" | mysql -h $StorageHost -u$StorageUser -p$StoragePass 2>&1 > /dev/null
14 | do
15 | echo "-- Waiting for database to become active ..."
16 | sleep 2
17 | done
18 | }
19 | echo "-- Database is now active ..."
20 |
21 | exec gosu slurm /usr/sbin/slurmdbd -Dvvv
22 | fi
23 |
24 | if [ "$1" = "slurmctld" ]
25 | then
26 | echo "---> Starting the MUNGE Authentication service (munged) ..."
27 | gosu munge /usr/sbin/munged
28 |
29 | echo "---> Waiting for slurmdbd to become active before starting slurmctld ..."
30 |
31 | until 2>/dev/null >/dev/tcp/slurmdbd/6819
32 | do
33 | echo "-- slurmdbd is not available. Sleeping ..."
34 | sleep 2
35 | done
36 | echo "-- slurmdbd is now active ..."
37 |
38 | echo "---> Starting the Slurm Controller Daemon (slurmctld) ..."
39 | if /usr/sbin/slurmctld -V | grep -q '17.02' ; then
40 | exec gosu slurm /usr/sbin/slurmctld -Dvvv
41 | else
42 | exec gosu slurm /usr/sbin/slurmctld -i -Dvvv
43 | fi
44 | fi
45 |
46 | if [ "$1" = "slurmd" ]
47 | then
48 | echo "---> Starting the MUNGE Authentication service (munged) ..."
49 | gosu munge /usr/sbin/munged
50 |
51 | echo "---> Waiting for slurmctld to become active before starting slurmd..."
52 |
53 | until 2>/dev/null >/dev/tcp/slurmctld/6817
54 | do
55 | echo "-- slurmctld is not available. Sleeping ..."
56 | sleep 2
57 | done
58 | echo "-- slurmctld is now active ..."
59 |
60 | echo "---> Starting the Slurm Node Daemon (slurmd) ..."
61 | exec /usr/sbin/slurmd -Z --conf "CPUs=2 RealMemory=1000 Feature=compute" -Dvvv
62 | fi
63 |
64 | exec "$@"
65 |
--------------------------------------------------------------------------------
/docs/source/examples/customresults.py:
--------------------------------------------------------------------------------
1 | """An example for handling results in a way that AsyncMapResult doesn't provide
2 |
3 | Specifically, out-of-order results with some special handing of metadata.
4 |
5 | This just submits a bunch of jobs, waits on the results, and prints the stdout
6 | and results of each as they finish.
7 |
8 | Authors
9 | -------
10 | * MinRK
11 | """
12 |
13 | import random
14 |
15 | import ipyparallel as ipp
16 |
17 | # create client & views
18 | rc = ipp.Client()
19 | dv = rc[:]
20 | v = rc.load_balanced_view()
21 |
22 |
23 | # scatter 'id', so id=0,1,2 on engines 0,1,2
24 | dv.scatter('id', rc.ids, flatten=True)
25 | print(dv['id'])
26 |
27 |
28 | def sleep_here(count, t):
29 | """simple function that takes args, prints a short message, sleeps for a time, and returns the same args"""
30 | import sys
31 | import time
32 |
33 | print(f"hi from engine {id}")
34 | sys.stdout.flush()
35 | time.sleep(t)
36 | return count, t
37 |
38 |
39 | amr = v.map(sleep_here, range(100), [random.random() for i in range(100)], chunksize=2)
40 |
41 | pending = set(amr.msg_ids)
42 | while pending:
43 | try:
44 | rc.wait(pending, 1e-3)
45 | except TimeoutError:
46 | # ignore timeouterrors, since they only mean that at least one isn't done
47 | pass
48 | # finished is the set of msg_ids that are complete
49 | finished = pending.difference(rc.outstanding)
50 | # update pending to exclude those that just finished
51 | pending = pending.difference(finished)
52 | for msg_id in finished:
53 | # we know these are done, so don't worry about blocking
54 | ar = rc.get_result(msg_id)
55 | print(f"job id {msg_id} finished on engine {ar.engine_id}")
56 | print("with stdout:")
57 | print(' ' + ar.stdout.replace('\n', '\n ').rstrip())
58 | print("and results:")
59 |
60 | # note that each job in a map always returns a list of length chunksize
61 | # even if chunksize == 1
62 | for count, t in ar.get():
63 | print(f" item {count}: slept for {t:.2f}s")
64 |
--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | # Build releases and (on tags) publish to PyPI
2 | name: Release
3 |
4 | # always build releases (to make sure wheel-building works)
5 | # but only publish to PyPI on tags
6 | on:
7 | push:
8 | branches-ignore:
9 | - "pre-commit-ci*"
10 | tags:
11 | - "*"
12 | pull_request:
13 |
14 | concurrency:
15 | group: >-
16 | ${{ github.workflow }}-
17 | ${{ github.ref_type }}-
18 | ${{ github.event.pull_request.number || github.sha }}
19 | cancel-in-progress: true
20 |
21 | jobs:
22 | build-release:
23 | runs-on: ubuntu-22.04
24 | steps:
25 | - uses: actions/checkout@v5
26 | - uses: actions/setup-python@v6
27 | with:
28 | python-version: 3.11
29 | cache: pip
30 |
31 | - name: install build package
32 | run: |
33 | pip install --upgrade pip
34 | pip install build
35 | pip freeze
36 |
37 | - name: build release
38 | run: |
39 | python -m build --sdist --wheel .
40 | ls -l dist
41 |
42 | - name: verify wheel
43 | run: |
44 | cd dist
45 | pip install ./*.whl jupyterlab==4.*
46 | ipcluster --help-all
47 | ipcontroller --help-all
48 | ipengine --help-all
49 | jupyter labextension list 2>&1 | grep ipyparallel
50 | jupyter server extension list 2>&1 | grep ipyparallel
51 |
52 | # ref: https://github.com/actions/upload-artifact#readme
53 | - uses: actions/upload-artifact@v5
54 | with:
55 | name: ipyparallel-${{ github.sha }}
56 | path: "dist/*"
57 | if-no-files-found: error
58 |
59 | upload-pypi:
60 | permissions:
61 | id-token: write
62 | environment: release
63 | runs-on: ubuntu-22.04
64 | if: startsWith(github.ref, 'refs/tags/')
65 | needs:
66 | - build-release
67 | steps:
68 | - uses: actions/download-artifact@v6
69 | with:
70 | path: dist
71 | merge-multiple: true
72 | - name: Publish wheels to PyPI
73 | uses: pypa/gh-action-pypi-publish@release/v1
74 |
--------------------------------------------------------------------------------
/ipyparallel/tests/test_executor.py:
--------------------------------------------------------------------------------
1 | """Tests for Executor API"""
2 |
3 | # Copyright (c) IPython Development Team.
4 | # Distributed under the terms of the Modified BSD License.
5 | import time
6 |
7 | from ipyparallel.client.view import LazyMapIterator, LoadBalancedView
8 |
9 | from .clienttest import ClusterTestCase
10 |
11 |
12 | def wait(n):
13 | import time
14 |
15 | time.sleep(n)
16 | return n
17 |
18 |
19 | def echo(x):
20 | return x
21 |
22 |
23 | class TestExecutor(ClusterTestCase):
24 | def test_client_executor(self):
25 | executor = self.client.executor()
26 | assert isinstance(executor.view, LoadBalancedView)
27 | f = executor.submit(lambda x: 2 * x, 5)
28 | r = f.result()
29 | assert r == 10
30 |
31 | def test_view_executor(self):
32 | view = self.client.load_balanced_view()
33 | executor = view.executor
34 | assert executor.view is view
35 |
36 | def test_executor_submit(self):
37 | view = self.client.load_balanced_view()
38 | executor = view.executor
39 | f = executor.submit(lambda x, y: x * y, 2, 3)
40 | r = f.result()
41 | assert r == 6
42 |
43 | def test_executor_map(self):
44 | view = self.client.load_balanced_view()
45 | executor = view.executor
46 | gen = executor.map(lambda x: x, range(5))
47 | assert isinstance(gen, LazyMapIterator)
48 | for i, r in enumerate(gen):
49 | assert i == r
50 |
51 | def test_executor_context(self):
52 | view = self.client.load_balanced_view()
53 | executor = view.executor
54 | with executor:
55 | f = executor.submit(time.sleep, 0.5)
56 | assert not f.done()
57 | m = executor.map(lambda x: x, range(10))
58 | assert len(view.history) == 11
59 | # Executor context calls shutdown
60 | # shutdown doesn't shutdown engines,
61 | # but it should at least wait for results to finish
62 | assert f.done()
63 | tic = time.perf_counter()
64 | list(m)
65 | toc = time.perf_counter()
66 | assert toc - tic < 0.5
67 |
--------------------------------------------------------------------------------
/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | env: {
3 | browser: true,
4 | es6: true,
5 | commonjs: true,
6 | node: true,
7 | },
8 | root: true,
9 | extends: [
10 | "eslint:recommended",
11 | "plugin:@typescript-eslint/eslint-recommended",
12 | "plugin:@typescript-eslint/recommended",
13 | "prettier",
14 | "plugin:react/recommended",
15 | ],
16 | parser: "@typescript-eslint/parser",
17 | parserOptions: {
18 | project: "tsconfig.eslint.json",
19 | },
20 | plugins: ["@typescript-eslint"],
21 | rules: {
22 | "@typescript-eslint/no-floating-promises": ["error", { ignoreVoid: true }],
23 | "@typescript-eslint/naming-convention": [
24 | "error",
25 | {
26 | selector: "interface",
27 | format: ["PascalCase"],
28 | custom: {
29 | regex: "^I[A-Z]",
30 | match: true,
31 | },
32 | },
33 | ],
34 | "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }],
35 | "@typescript-eslint/no-use-before-define": "off",
36 | "@typescript-eslint/camelcase": "off",
37 | "@typescript-eslint/no-explicit-any": "off",
38 | "@typescript-eslint/no-non-null-assertion": "off",
39 | "@typescript-eslint/no-namespace": "off",
40 | "@typescript-eslint/interface-name-prefix": "off",
41 | "@typescript-eslint/explicit-function-return-type": "off",
42 | "@typescript-eslint/ban-ts-comment": ["warn", { "ts-ignore": true }],
43 | "@typescript-eslint/ban-types": "warn",
44 | "@typescript-eslint/no-non-null-asserted-optional-chain": "warn",
45 | "@typescript-eslint/no-var-requires": "off",
46 | "@typescript-eslint/no-empty-interface": "off",
47 | "@typescript-eslint/triple-slash-reference": "warn",
48 | "@typescript-eslint/no-inferrable-types": "off",
49 | "no-inner-declarations": "off",
50 | "no-prototype-builtins": "off",
51 | "no-control-regex": "warn",
52 | "no-undef": "warn",
53 | "no-case-declarations": "warn",
54 | "no-useless-escape": "off",
55 | "prefer-const": "off",
56 | "react/prop-types": "warn",
57 | },
58 | settings: {
59 | react: {
60 | version: "detect",
61 | },
62 | },
63 | };
64 |
--------------------------------------------------------------------------------
/docs/source/examples/wave2D/communicator.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """A simple Communicator class that has N,E,S,W neighbors connected via 0MQ PEER sockets"""
3 |
4 | import socket
5 |
6 | import zmq
7 |
8 | from ipyparallel.util import disambiguate_url
9 |
10 |
11 | class EngineCommunicator:
12 | """An object that connects Engines to each other.
13 | north and east sockets listen, while south and west sockets connect.
14 |
15 | This class is useful in cases where there is a set of nodes that
16 | must communicate only with their nearest neighbors.
17 | """
18 |
19 | def __init__(self, interface='tcp://*', identity=None):
20 | self._ctx = zmq.Context()
21 | self.north = self._ctx.socket(zmq.PAIR)
22 | self.west = self._ctx.socket(zmq.PAIR)
23 | self.south = self._ctx.socket(zmq.PAIR)
24 | self.east = self._ctx.socket(zmq.PAIR)
25 |
26 | # bind to ports
27 | northport = self.north.bind_to_random_port(interface)
28 | eastport = self.east.bind_to_random_port(interface)
29 |
30 | self.north_url = f"{interface}:{northport}"
31 | self.east_url = f"{interface}:{eastport}"
32 |
33 | # guess first public IP from socket
34 | self.location = socket.gethostbyname_ex(socket.gethostname())[-1][0]
35 |
36 | def __del__(self):
37 | self.north.close()
38 | self.south.close()
39 | self.east.close()
40 | self.west.close()
41 | self._ctx.term()
42 |
43 | @property
44 | def info(self):
45 | """return the connection info for this object's sockets."""
46 | return (self.location, self.north_url, self.east_url)
47 |
48 | def connect(self, south_peer=None, west_peer=None):
49 | """connect to peers. `peers` will be a 3-tuples, of the form:
50 | (location, north_addr, east_addr)
51 | as produced by
52 | """
53 | if south_peer is not None:
54 | location, url, _ = south_peer
55 | self.south.connect(disambiguate_url(url, location))
56 | if west_peer is not None:
57 | location, _, url = west_peer
58 | self.west.connect(disambiguate_url(url, location))
59 |
--------------------------------------------------------------------------------
/docs/source/examples/pi/parallelpi.py:
--------------------------------------------------------------------------------
1 | """Calculate statistics on the digits of pi in parallel.
2 |
3 | This program uses the functions in :file:`pidigits.py` to calculate
4 | the frequencies of 2 digit sequences in the digits of pi. The
5 | results are plotted using matplotlib.
6 |
7 | To run, text files from https://www.super-computing.org/
8 | must be installed in the working directory of the IPython engines.
9 | The actual filenames to be used can be set with the ``filestring``
10 | variable below.
11 |
12 | The dataset we have been using for this is the 200 million digit one here:
13 | ftp://pi.super-computing.org/.2/pi200m/
14 |
15 | and the files used will be downloaded if they are not in the working directory
16 | of the IPython engines.
17 | """
18 |
19 | from timeit import default_timer as clock
20 |
21 | from matplotlib import pyplot as plt
22 | from pidigits import (
23 | compute_two_digit_freqs,
24 | fetch_pi_file,
25 | plot_two_digit_freqs,
26 | reduce_freqs,
27 | )
28 |
29 | import ipyparallel as ipp
30 |
31 | # Files with digits of pi (10m digits each)
32 | filestring = 'pi200m.ascii.%(i)02dof20'
33 | files = [filestring % {'i': i} for i in range(1, 21)]
34 |
35 | # Connect to the IPython cluster
36 | c = ipp.Client()
37 | c[:].run('pidigits.py')
38 |
39 | # the number of engines
40 | n = len(c)
41 | id0 = c.ids[0]
42 | v = c[:]
43 | v.block = True
44 | # fetch the pi-files
45 | print(f"downloading {n} files of pi")
46 | v.map(fetch_pi_file, files[:n]) # noqa: F821
47 | print("done")
48 |
49 | # Run 10m digits on 1 engine
50 | t1 = clock()
51 | freqs10m = c[id0].apply_sync(compute_two_digit_freqs, files[0])
52 | t2 = clock()
53 | digits_per_second1 = 10.0e6 / (t2 - t1)
54 | print("Digits per second (1 core, 10m digits): ", digits_per_second1)
55 |
56 |
57 | # Run n*10m digits on all engines
58 | t1 = clock()
59 | freqs_all = v.map(compute_two_digit_freqs, files[:n])
60 | freqs150m = reduce_freqs(freqs_all)
61 | t2 = clock()
62 | digits_per_second8 = n * 10.0e6 / (t2 - t1)
63 | print(f"Digits per second ({n} engines, {n}0m digits): ", digits_per_second8)
64 |
65 | print("Speedup: ", digits_per_second8 / digits_per_second1)
66 |
67 | plot_two_digit_freqs(freqs150m)
68 | plt.title(f"2 digit sequences in {n}0m digits of pi")
69 | plt.show()
70 |
--------------------------------------------------------------------------------
/docs/source/examples/daVinci Word Count/wordfreq.py:
--------------------------------------------------------------------------------
1 | """Count the frequencies of words in a string"""
2 |
3 |
4 | def wordfreq(text, is_filename=False):
5 | """Return a dictionary of words and word counts in a string."""
6 | if is_filename:
7 | with open(text) as f:
8 | text = f.read()
9 | freqs = {}
10 | for word in text.split():
11 | lword = word.lower()
12 | freqs[lword] = freqs.get(lword, 0) + 1
13 | return freqs
14 |
15 |
16 | def print_wordfreq(freqs, n=10):
17 | """Print the n most common words and counts in the freqs dict."""
18 |
19 | words, counts = freqs.keys(), freqs.values()
20 | items = zip(counts, words)
21 | items = sorted(items, reverse=True)
22 | for count, word in items[:n]:
23 | print(word, count)
24 |
25 |
26 | def wordfreq_to_weightsize(
27 | worddict, minsize=25, maxsize=50, minalpha=0.5, maxalpha=1.0
28 | ):
29 | mincount = min(worddict.itervalues())
30 | maxcount = max(worddict.itervalues())
31 | weights = {}
32 | for k, v in worddict.iteritems():
33 | w = (v - mincount) / (maxcount - mincount)
34 | alpha = minalpha + (maxalpha - minalpha) * w
35 | size = minsize + (maxsize - minsize) * w
36 | weights[k] = (alpha, size)
37 | return weights
38 |
39 |
40 | def tagcloud(worddict, n=10, minsize=25, maxsize=50, minalpha=0.5, maxalpha=1.0):
41 | import random
42 |
43 | from matplotlib import pyplot as plt
44 |
45 | worddict = wordfreq_to_weightsize(worddict, minsize, maxsize, minalpha, maxalpha)
46 |
47 | fig = plt.figure()
48 | ax = fig.add_subplot(111)
49 | ax.set_position([0.0, 0.0, 1.0, 1.0])
50 | plt.xticks([])
51 | plt.yticks([])
52 |
53 | words = worddict.keys()
54 | alphas = [v[0] for v in worddict.values()]
55 | sizes = [v[1] for v in worddict.values()]
56 | items = zip(alphas, sizes, words)
57 | items.sort(reverse=True)
58 | for alpha, size, word in items[:n]:
59 | # xpos = random.normalvariate(0.5, 0.3)
60 | # ypos = random.normalvariate(0.5, 0.3)
61 | xpos = random.uniform(0.0, 1.0)
62 | ypos = random.uniform(0.0, 1.0)
63 | ax.text(xpos, ypos, word.lower(), alpha=alpha, fontsize=size)
64 | ax.autoscale_view()
65 | return ax
66 |
--------------------------------------------------------------------------------
/ipyparallel/serialize/codeutil.py:
--------------------------------------------------------------------------------
1 | """Utilities to enable code objects to be pickled.
2 |
3 | Any process that import this module will be able to pickle code objects. This
4 | includes the func_code attribute of any function. Once unpickled, new
5 | functions can be built using new.function(code, globals()). Eventually
6 | we need to automate all of this so that functions themselves can be pickled.
7 |
8 | Reference: A. Tremols, P Cogolo, "Python Cookbook," p 302-305
9 | """
10 |
11 | # Copyright (c) IPython Development Team.
12 | # Distributed under the terms of the Modified BSD License.
13 | import copyreg
14 | import inspect
15 | import sys
16 | import types
17 |
18 |
19 | def code_ctor(*args):
20 | return types.CodeType(*args)
21 |
22 |
23 | # map CodeType constructor args to code co_ attribute names
24 | # (they _almost_ all match, and new ones probably will)
25 | _code_attr_map = {
26 | "codestring": "code",
27 | "constants": "consts",
28 | }
29 | # pass every supported arg to the code constructor
30 | # this should be more forward-compatible
31 | # (broken on pypy: https://github.com/ipython/ipyparallel/issues/845)
32 | if sys.version_info >= (3, 10) and getattr(sys, "pypy_version_info", (7, 3, 19)) >= (
33 | 7,
34 | 3,
35 | 19,
36 | ):
37 | _code_attr_names = tuple(
38 | _code_attr_map.get(name, name)
39 | for name, param in inspect.signature(types.CodeType).parameters.items()
40 | if (param.POSITIONAL_ONLY or param.POSITIONAL_OR_KEYWORD)
41 | and not (hasattr(sys, "pypy_version_info") and name == "magic")
42 | )
43 | else:
44 | # can't inspect types.CodeType on Python < 3.10
45 | _code_attr_names = [
46 | "argcount",
47 | "kwonlyargcount",
48 | "nlocals",
49 | "stacksize",
50 | "flags",
51 | "code",
52 | "consts",
53 | "names",
54 | "varnames",
55 | "filename",
56 | "name",
57 | "firstlineno",
58 | "lnotab",
59 | "freevars",
60 | "cellvars",
61 | ]
62 | if hasattr(types.CodeType, "co_posonlyargcount"):
63 | _code_attr_names.insert(1, "posonlyargcount")
64 |
65 | _code_attr_names = tuple(_code_attr_names)
66 |
67 |
68 | def reduce_code(obj):
69 | """codeobject reducer"""
70 | return code_ctor, tuple(getattr(obj, f'co_{name}') for name in _code_attr_names)
71 |
72 |
73 | copyreg.pickle(types.CodeType, reduce_code)
74 |
--------------------------------------------------------------------------------
/.github/workflows/test-docs.yml:
--------------------------------------------------------------------------------
1 | name: Test docs
2 |
3 | on:
4 | pull_request:
5 | push:
6 | branches-ignore:
7 | - "dependabot/**"
8 | - "pre-commit-ci-update-config"
9 | tags:
10 | - "**"
11 | workflow_dispatch:
12 |
13 | env:
14 | # UTF-8 content may be interpreted as ascii and causes errors without this.
15 | LANG: C.UTF-8
16 | PYTEST_ADDOPTS: "--verbose --color=yes"
17 |
18 | jobs:
19 | test-docs:
20 | runs-on: ubuntu-24.04
21 | steps:
22 | - uses: actions/checkout@v5
23 | with:
24 | # make rediraffecheckdiff requires git history to compare current
25 | # commit with the main branch and previous releases.
26 | fetch-depth: 0
27 |
28 | - uses: actions/setup-python@v6
29 | with:
30 | python-version: "3.13"
31 |
32 | - name: Install requirements
33 | run: |
34 | pip install -r docs/requirements.txt .
35 |
36 | # readthedocs doesn't halt on warnings,
37 | # so raise any warnings here
38 | - name: build docs
39 | run: |
40 | cd docs
41 | make html
42 |
43 | - name: check links
44 | run: |
45 | cd docs
46 | make linkcheck
47 |
48 | # make rediraffecheckdiff compares files for different changesets
49 | # these diff targets aren't always available
50 | # - compare with base ref (usually 'main', always on 'origin') for pull requests
51 | # - only compare with tags when running against main repo
52 | # to avoid errors on forks, which often lack tags
53 | - name: check redirects for this PR
54 | if: github.event_name == 'pull_request'
55 | run: |
56 | cd docs
57 | export REDIRAFFE_BRANCH=origin/${{ github.base_ref }}
58 | make rediraffecheckdiff
59 |
60 | # this should check currently published 'stable' links for redirects
61 | - name: check redirects since last release
62 | if: github.repository == 'ipython/ipyparallel'
63 | run: |
64 | cd docs
65 | export REDIRAFFE_BRANCH=$(git describe --tags --abbrev=0)
66 | make rediraffecheckdiff
67 |
68 | # longer-term redirect check (fixed version) for older links
69 | - name: check redirects since 8.6.0
70 | if: github.repository == 'ipython/ipyparallel'
71 | run: |
72 | cd docs
73 | export REDIRAFFE_BRANCH=8.6.0
74 | make rediraffecheckdiff
75 |
--------------------------------------------------------------------------------
/benchmarks/profiling/profiling_code.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | import numpy as np
4 |
5 | import ipyparallel as ipp
6 |
7 |
8 | def echo(delay=0):
9 | def inner_echo(x, **kwargs):
10 | import time
11 |
12 | if delay:
13 | time.sleep(delay)
14 | return x
15 |
16 | return inner_echo
17 |
18 |
19 | def profile_many_empty_tasks(lview, n, block=True):
20 | lview.map(echo(0), [None] * n, block=block)
21 |
22 |
23 | # def profile_echo_many_arguments(lview, number_of_arguments):
24 | # lview.map(
25 | # lambda x: echo_many_arguments(*x),
26 | # [
27 | # tuple(np.empty(1, dtype=np.int8) for n in range(number_of_arguments))
28 | # for x in range(16)
29 | # ],
30 | # block=False,
31 | # )
32 |
33 |
34 | def profile_tasks_with_large_data(lview, num_bytes):
35 | for _ in range(10):
36 | for i in range(10):
37 | lview.apply_sync(echo(0), np.array([0] * num_bytes, dtype=np.int8))
38 |
39 |
40 | def run_profiling(selected_profiling_task, selected_view):
41 | client = ipp.Client(profile='asv')
42 | # add it to path on the engines
43 | # client[:].apply_sync(add_to_path, master_project_parent)
44 | print('profiling task: ', selected_profiling_task)
45 | print('profiling view: ', selected_view)
46 | if selected_view == 'direct':
47 | view = client[:]
48 | elif selected_view == 'spanning_tree':
49 | view = client.spanning_tree_view()
50 | else:
51 | view = client.load_balanced_view()
52 |
53 | # view = client[:] if selected_view == 'direct' else client.load_balanced_view()
54 |
55 | if selected_profiling_task == 'many_empty_tasks':
56 | for x in range(1, 5):
57 | profile_many_empty_tasks(view, 10**x)
58 | elif selected_profiling_task == 'many_empty_tasks_non_blocking':
59 | for x in range(1, 5):
60 | profile_many_empty_tasks(view, 10**x, block=False)
61 | elif selected_profiling_task == 'tasks_with_large_data':
62 | for x in range(1, 8):
63 | profile_tasks_with_large_data(view, 10**x)
64 | # elif selected_profiling_task == 'echo_many_arguments':
65 | # for i in range(100):
66 | # for number_of_arguments in ((2 ** x) - 1 for x in range(1, 9)):
67 | # profile_echo_many_arguments(view, number_of_arguments)
68 |
69 |
70 | if __name__ == "__main__":
71 | run_profiling(sys.argv[1], sys.argv[2])
72 |
--------------------------------------------------------------------------------
/ipyparallel/tests/test_ssh.py:
--------------------------------------------------------------------------------
1 | import os
2 | from functools import partial
3 |
4 | import pytest
5 | from traitlets.config import Config
6 |
7 | from .conftest import Cluster as BaseCluster # noqa: F401
8 | from .test_cluster import (
9 | test_get_output, # noqa: F401
10 | test_restart_engines, # noqa: F401
11 | test_signal_engines, # noqa: F401
12 | test_start_stop_cluster, # noqa: F401
13 | test_to_from_dict, # noqa: F401
14 | )
15 |
16 | # import tests that use engine_launcher_class fixture
17 |
18 |
19 | @pytest.fixture(params=["SSH", "SSHProxy"])
20 | def ssh_config(ssh_key, request):
21 | windows = os.name == "nt"
22 |
23 | if (
24 | windows and request.param == "SSHProxy"
25 | ): # SSHProxy currently not working under Windows
26 | pytest.skip("Proxy tests currently not working under Windows")
27 |
28 | c = Config()
29 | c.Cluster.controller_ip = '0.0.0.0'
30 | c.Cluster.engine_launcher_class = request.param
31 | engine_set_cfg = c[f"{request.param}EngineSetLauncher"]
32 | engine_set_cfg.ssh_args = []
33 | if not windows:
34 | engine_set_cfg.ssh_args.extend(
35 | [
36 | "-o",
37 | "UserKnownHostsFile=/dev/null",
38 | "-o",
39 | "StrictHostKeyChecking=no",
40 | "-i",
41 | ssh_key,
42 | ]
43 | )
44 | engine_set_cfg.scp_args = list(engine_set_cfg.ssh_args) # copy
45 | if windows:
46 | engine_set_cfg.remote_python = "python"
47 | engine_set_cfg.remote_profile_dir = "C:/Users/ciuser/.ipython/profile_default"
48 | else:
49 | engine_set_cfg.remote_python = "/opt/conda/bin/python3"
50 | engine_set_cfg.remote_profile_dir = "/home/ciuser/.ipython/profile_default"
51 | engine_set_cfg.engine_args = ['--debug']
52 | c.SSHProxyEngineSetLauncher.hostname = "127.0.0.1"
53 | c.SSHProxyEngineSetLauncher.ssh_args.append("-p2222")
54 | c.SSHProxyEngineSetLauncher.scp_args.append("-P2222")
55 | c.SSHProxyEngineSetLauncher.user = "ciuser"
56 | c.SSHEngineSetLauncher.engines = {"ciuser@127.0.0.1:2222": 4}
57 | return c
58 |
59 |
60 | @pytest.fixture
61 | def Cluster(ssh_config, BaseCluster): # noqa: F811
62 | """Override Cluster to add ssh config"""
63 | return partial(BaseCluster, config=ssh_config)
64 |
65 |
66 | # override engine_launcher_class
67 | @pytest.fixture
68 | def engine_launcher_class(ssh_config):
69 | return ssh_config.Cluster.engine_launcher_class
70 |
--------------------------------------------------------------------------------
/benchmarks/cluster_start.py:
--------------------------------------------------------------------------------
1 | import atexit
2 | import sys
3 | import time
4 | from subprocess import Popen
5 |
6 | from benchmarks.throughput import wait_for
7 |
8 | import ipyparallel as ipp
9 |
10 |
11 | def start_cluster(depth, number_of_engines, path='', log_output_to_file=False):
12 | ipcontroller_cmd = (
13 | f'{path}ipcontroller --profile=asv --nodb '
14 | f'--cluster-id=depth_{depth} '
15 | f'--HubFactory.broadcast_scheduler_depth={depth} '
16 | f'--HubFactory.db_class=NoDB'
17 | )
18 | print(ipcontroller_cmd)
19 | ipengine_cmd = f'{path}ipengine --profile=asv --cluster-id=depth_{depth} '
20 | ps = [
21 | Popen(
22 | ipcontroller_cmd.split(),
23 | stdout=(
24 | open('ipcontroller_output.log', 'a+')
25 | if log_output_to_file
26 | else sys.stdout
27 | ),
28 | stderr=(
29 | open('ipcontroller_error_output.log', 'a+')
30 | if log_output_to_file
31 | else sys.stdout
32 | ),
33 | stdin=sys.stdin,
34 | )
35 | ]
36 | time.sleep(2)
37 | client = ipp.Client(profile='asv', cluster_id=f'depth_{depth}')
38 | print(ipengine_cmd)
39 | for i in range(number_of_engines):
40 | ps.append(
41 | Popen(
42 | ipengine_cmd.split(),
43 | stdout=(
44 | open('ipengine_output.log', 'a+')
45 | if log_output_to_file
46 | else sys.stdout
47 | ),
48 | stderr=(
49 | open('ipengine_error_output.log', 'a+')
50 | if log_output_to_file
51 | else sys.stdout
52 | ),
53 | stdin=sys.stdin,
54 | )
55 | )
56 | if i % 10 == 0:
57 | wait_for(lambda: len(client) >= i - 10)
58 | if i % 20 == 0:
59 | time.sleep(2)
60 | print(f'{len(client)} engines started')
61 |
62 | return ps
63 |
64 |
65 | if __name__ == '__main__':
66 | if len(sys.argv) > 3:
67 | depth = sys.argv[1]
68 | number_of_engines = int(sys.argv[3])
69 | else:
70 | depth = 3
71 | number_of_engines = 30
72 |
73 | ps = start_cluster(depth, number_of_engines)
74 |
75 | for p in ps:
76 | p.wait()
77 |
78 | def clean_up():
79 | for p in ps:
80 | p.kill()
81 |
82 | atexit.register(clean_up)
83 |
--------------------------------------------------------------------------------
/ipyparallel/client/_joblib.py:
--------------------------------------------------------------------------------
1 | """joblib parallel backend for IPython Parallel"""
2 |
3 | # Copyright (c) IPython Development Team.
4 | # Distributed under the terms of the Modified BSD License.
5 | from joblib.parallel import AutoBatchingMixin, ParallelBackendBase
6 |
7 | import ipyparallel as ipp
8 |
9 |
10 | class IPythonParallelBackend(AutoBatchingMixin, ParallelBackendBase):
11 | def __init__(self, view=None, **kwargs):
12 | super().__init__(**kwargs)
13 | self._cluster_owner = False
14 | self._client_owner = False
15 | if view is None:
16 | self._client_owner = True
17 | try:
18 | # load the default cluster
19 | cluster = ipp.Cluster.from_file()
20 | except FileNotFoundError:
21 | # other load errors?
22 | cluster = self._cluster = ipp.Cluster()
23 | self._cluster_owner = True
24 | cluster.start_cluster_sync()
25 | else:
26 | # cluster running, ensure some engines are, too
27 | if not cluster.engines:
28 | cluster.start_engines_sync()
29 | rc = cluster.connect_client_sync()
30 | rc.wait_for_engines(cluster.n or 1)
31 | view = rc.load_balanced_view()
32 |
33 | # use cloudpickle or dill for closures, if available.
34 | # joblib tends to create closures default pickle can't handle.
35 | try:
36 | import cloudpickle # noqa
37 | except ImportError:
38 | try:
39 | import dill # noqa
40 | except ImportError:
41 | pass
42 | else:
43 | view.client[:].use_dill()
44 | else:
45 | view.client[:].use_cloudpickle()
46 | self._view = view
47 |
48 | def effective_n_jobs(self, n_jobs):
49 | """A View can run len(view) jobs at a time"""
50 | return len(self._view)
51 |
52 | def terminate(self):
53 | """Close the client if we created it"""
54 | if self._client_owner:
55 | self._view.client.close()
56 | if self._cluster_owner:
57 | self._cluster.stop_cluster_sync()
58 |
59 | def apply_async(self, func, callback=None):
60 | """Schedule a func to be run"""
61 | future = self._view.apply_async(func)
62 | if callback:
63 | future.add_done_callback(lambda f: callback(f.result()))
64 | return future
65 |
--------------------------------------------------------------------------------
/ipyparallel/apps/iploggerapp.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | A simple IPython logger application
4 | """
5 |
6 | from IPython.core.profiledir import ProfileDir
7 | from traitlets import Dict
8 |
9 | from ipyparallel.apps.baseapp import (
10 | BaseParallelApplication,
11 | base_aliases,
12 | catch_config_error,
13 | )
14 | from ipyparallel.apps.logwatcher import LogWatcher
15 |
16 | # -----------------------------------------------------------------------------
17 | # Module level variables
18 | # -----------------------------------------------------------------------------
19 |
20 | #: The default config file name for this application
21 | _description = """Start an IPython logger for parallel computing.
22 |
23 | IPython controllers and engines (and your own processes) can broadcast log messages
24 | by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
25 | logger can be configured using command line options or using a cluster
26 | directory. Cluster directories contain config, log and security files and are
27 | usually located in your ipython directory and named as "profile_name".
28 | See the `profile` and `profile-dir` options for details.
29 | """
30 |
31 |
32 | # -----------------------------------------------------------------------------
33 | # Main application
34 | # -----------------------------------------------------------------------------
35 | aliases = {}
36 | aliases.update(base_aliases)
37 | aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics'))
38 |
39 |
40 | class IPLoggerApp(BaseParallelApplication):
41 | name = 'iplogger'
42 | description = _description
43 | classes = [LogWatcher, ProfileDir]
44 | aliases = Dict(aliases)
45 |
46 | @catch_config_error
47 | def initialize(self, argv=None):
48 | super().initialize(argv)
49 | self.init_watcher()
50 |
51 | def init_watcher(self):
52 | try:
53 | self.watcher = LogWatcher(parent=self, log=self.log)
54 | except BaseException:
55 | self.log.error("Couldn't start the LogWatcher", exc_info=True)
56 | self.exit(1)
57 | self.log.info(f"Listening for log messages on {self.watcher.url!r}")
58 |
59 | def start(self):
60 | self.watcher.start()
61 | try:
62 | self.watcher.loop.start()
63 | except KeyboardInterrupt:
64 | self.log.critical("Logging Interrupted, shutting down...\n")
65 |
66 |
67 | launch_new_instance = IPLoggerApp.launch_instance
68 |
69 |
70 | if __name__ == '__main__':
71 | launch_new_instance()
72 |
--------------------------------------------------------------------------------
/docs/source/examples/task_profiler.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """Test the performance of the task farming system.
3 |
4 | This script submits a set of tasks via a LoadBalancedView. The tasks
5 | are basically just a time.sleep(t), where t is a random number between
6 | two limits that can be configured at the command line. To run
7 | the script there must first be an IPython controller and engines running::
8 |
9 | ipcluster start -n 16
10 |
11 | A good test to run with 16 engines is::
12 |
13 | python task_profiler.py -n 128 -t 0.01 -T 1.0
14 |
15 | This should show a speedup of 13-14x. The limitation here is that the
16 | overhead of a single task is about 0.001-0.01 seconds.
17 | """
18 |
19 | import random
20 | import time
21 | from optparse import OptionParser
22 |
23 | import ipyparallel as ipp
24 |
25 |
26 | def main():
27 | parser = OptionParser()
28 | parser.set_defaults(n=100)
29 | parser.set_defaults(tmin=1e-3)
30 | parser.set_defaults(tmax=1)
31 | parser.set_defaults(profile='default')
32 |
33 | parser.add_option("-n", type='int', dest='n', help='the number of tasks to run')
34 | parser.add_option(
35 | "-t", type='float', dest='tmin', help='the minimum task length in seconds'
36 | )
37 | parser.add_option(
38 | "-T", type='float', dest='tmax', help='the maximum task length in seconds'
39 | )
40 | parser.add_option(
41 | "-p",
42 | '--profile',
43 | type='str',
44 | dest='profile',
45 | help="the cluster profile [default: 'default']",
46 | )
47 |
48 | (opts, args) = parser.parse_args()
49 | assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin"
50 |
51 | rc = ipp.Client()
52 | view = rc.load_balanced_view()
53 | print(view)
54 | rc.block = True
55 | nengines = len(rc.ids)
56 |
57 | # the jobs should take a random time within a range
58 | times = [
59 | random.random() * (opts.tmax - opts.tmin) + opts.tmin for i in range(opts.n)
60 | ]
61 | stime = sum(times)
62 |
63 | print(f"executing {opts.n} tasks, totalling {stime:.1f} secs on {nengines} engines")
64 | time.sleep(1)
65 | start = time.perf_counter()
66 | amr = view.map(time.sleep, times)
67 | amr.get()
68 | stop = time.perf_counter()
69 |
70 | ptime = stop - start
71 | scale = stime / ptime
72 |
73 | print(f"executed {stime:.1f} secs in {ptime:.1f} secs")
74 | print(f"{scale:3f}x parallel performance on {nengines} engines")
75 | print(f"{scale / nengines:.0%} of theoretical max")
76 |
77 |
78 | if __name__ == '__main__':
79 | main()
80 |
--------------------------------------------------------------------------------
/docs/source/examples/itermapresult.py:
--------------------------------------------------------------------------------
1 | """Example of iteration through AsyncMapResults, without waiting for all results
2 |
3 | When you call view.map(func, sequence), you will receive a special AsyncMapResult
4 | object. These objects are used to reconstruct the results of the split call.
5 | One feature AsyncResults provide is that they are iterable *immediately*, so
6 | you can iterate through the actual results as they complete.
7 |
8 | This is useful if you submit a large number of tasks that may take some time,
9 | but want to perform logic on elements in the result, or even abort subsequent
10 | tasks in cases where you are searching for the first affirmative result.
11 |
12 | By default, the results will match the ordering of the submitted sequence, but
13 | if you call `map(...ordered=False)`, then results will be provided to the iterator
14 | on a first come first serve basis.
15 |
16 | Authors
17 | -------
18 | * MinRK
19 | """
20 |
21 | import time
22 |
23 | import ipyparallel as ipp
24 |
25 | # create client & view
26 | rc = ipp.Client()
27 | dv = rc[:]
28 | v = rc.load_balanced_view()
29 |
30 | # scatter 'id', so id=0,1,2 on engines 0,1,2
31 | dv.scatter('id', rc.ids, flatten=True)
32 | print("Engine IDs: ", dv['id'])
33 |
34 | # create a Reference to `id`. This will be a different value on each engine
35 | ref = ipp.Reference('id')
36 | print("sleeping for `id` seconds on each engine")
37 | tic = time.perf_counter()
38 | ar = dv.apply(time.sleep, ref)
39 | for i, r in enumerate(ar):
40 | print(f"{i}: {time.perf_counter() - tic:.3f}")
41 |
42 |
43 | def sleep_here(t):
44 | import time
45 |
46 | time.sleep(t)
47 | return id, t
48 |
49 |
50 | # one call per task
51 | print("running with one call per task")
52 | amr = v.map(sleep_here, [0.01 * t for t in range(100)])
53 | tic = time.perf_counter()
54 | for i, r in enumerate(amr):
55 | print(f"task {i} on engine {r[0]}: {time.perf_counter() - tic:.3f}")
56 |
57 | print("running with four calls per task")
58 | # with chunksize, we can have four calls per task
59 | amr = v.map(sleep_here, [0.01 * t for t in range(100)], chunksize=4)
60 | tic = time.perf_counter()
61 | for i, r in enumerate(amr):
62 | print(f"task {i} on engine {r[0]}: {time.perf_counter() - tic:.3f}")
63 |
64 | print("running with two calls per task, with unordered results")
65 | # We can even iterate through faster results first, with ordered=False
66 | amr = v.map(
67 | sleep_here, [0.01 * t for t in range(100, 0, -1)], ordered=False, chunksize=2
68 | )
69 | tic = time.perf_counter()
70 | for i, r in enumerate(amr):
71 | print(f"slept {r[1]:.2f}s on engine {r[0]}: {time.perf_counter() - tic:.3f}")
72 |
--------------------------------------------------------------------------------
/benchmarks/results/asv_testing-16-2020-05-04-17-43/results.json:
--------------------------------------------------------------------------------
1 | {
2 | "results": {
3 | "throughput.CoalescingBroadcast.time_broadcast": {
4 | "result": null,
5 | "params": [
6 | ["0", "0.1"],
7 | ["1", "10", "50", "100", "200"],
8 | ["10", "100", "1000", "10000", "100000", "1000000"]
9 | ]
10 | },
11 | "throughput.DirectViewBroadCast.time_broadcast": {
12 | "result": null,
13 | "params": [
14 | ["0", "0.1"],
15 | ["1", "10", "50", "100", "200"],
16 | ["10", "100", "1000", "10000", "100000", "1000000"]
17 | ]
18 | },
19 | "throughput.NonCoalescingBroadcast.time_broadcast": {
20 | "result": null,
21 | "params": [
22 | ["0", "0.1"],
23 | ["1", "10", "50", "100", "200"],
24 | ["10", "100", "1000", "10000", "100000", "1000000"]
25 | ]
26 | },
27 | "throughput.SpanningTreeBroadCast.time_broadcast": {
28 | "result": null,
29 | "params": [
30 | ["0", "0.1"],
31 | ["1", "10", "50", "100", "200"],
32 | ["10", "100", "1000", "10000", "100000", "1000000"]
33 | ]
34 | }
35 | },
36 | "params": {
37 | "arch": "x86_64",
38 | "cpu": "Intel(R) Xeon(R) CPU @ 2.30GHz",
39 | "machine": "asv-testing-16",
40 | "os": "Linux 4.15.0-1029-gcp",
41 | "ram": "16423396",
42 | "python": "3.7",
43 | "numpy": ""
44 | },
45 | "requirements": {
46 | "numpy": ""
47 | },
48 | "commit_hash": "766bdabf611694c8bf1807677ed3a501bf0b87f2",
49 | "date": 1588108958000,
50 | "env_name": "conda-py3.7-numpy",
51 | "python": "3.7",
52 | "profiles": {},
53 | "started_at": {
54 | "throughput.CoalescingBroadcast.time_broadcast": 1588607341576,
55 | "throughput.DirectViewBroadCast.time_broadcast": 1588607371588,
56 | "throughput.NonCoalescingBroadcast.time_broadcast": 1588607401077,
57 | "throughput.SpanningTreeBroadCast.time_broadcast": 1588607436141
58 | },
59 | "ended_at": {
60 | "throughput.CoalescingBroadcast.time_broadcast": 1588607371588,
61 | "throughput.DirectViewBroadCast.time_broadcast": 1588607401077,
62 | "throughput.NonCoalescingBroadcast.time_broadcast": 1588607436140,
63 | "throughput.SpanningTreeBroadCast.time_broadcast": 1588607459984
64 | },
65 | "benchmark_version": {
66 | "throughput.CoalescingBroadcast.time_broadcast": "fa34b3be29f11e8e92ff3f69ee3db3cef435c17dc271602df2b926f774e93a8e",
67 | "throughput.DirectViewBroadCast.time_broadcast": "fa34b3be29f11e8e92ff3f69ee3db3cef435c17dc271602df2b926f774e93a8e",
68 | "throughput.NonCoalescingBroadcast.time_broadcast": "fa34b3be29f11e8e92ff3f69ee3db3cef435c17dc271602df2b926f774e93a8e",
69 | "throughput.SpanningTreeBroadCast.time_broadcast": "fa34b3be29f11e8e92ff3f69ee3db3cef435c17dc271602df2b926f774e93a8e"
70 | },
71 | "version": 1
72 | }
73 |
--------------------------------------------------------------------------------
/docs/source/examples/daVinci Word Count/pwordfreq.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """Parallel word frequency counter.
3 |
4 | This only works for a local cluster, because the filenames are local paths.
5 | """
6 |
7 | import os
8 | import time
9 | from itertools import repeat
10 |
11 | import requests
12 | from wordfreq import print_wordfreq, wordfreq
13 |
14 | import ipyparallel as ipp
15 |
16 | davinci_url = "https://www.gutenberg.org/files/5000/5000-8.txt"
17 |
18 |
19 | def pwordfreq(view, fnames):
20 | """Parallel word frequency counter.
21 |
22 | view - An IPython DirectView
23 | fnames - The filenames containing the split data.
24 | """
25 | assert len(fnames) == len(view.targets)
26 | view.scatter('fname', fnames, flatten=True)
27 | ar = view.apply(wordfreq, ipp.Reference('fname'))
28 | freqs_list = ar.get()
29 | word_set = set()
30 | for f in freqs_list:
31 | word_set.update(f.keys())
32 | freqs = dict(zip(word_set, repeat(0)))
33 | for f in freqs_list:
34 | for word, count in f.items():
35 | freqs[word] += count
36 | return freqs
37 |
38 |
39 | if __name__ == '__main__':
40 | # Create a Client and View
41 | rc = ipp.Client()
42 |
43 | view = rc[:]
44 | view.apply_sync(os.chdir, os.getcwd())
45 |
46 | if not os.path.exists('davinci.txt'):
47 | # download from project gutenberg
48 | print("Downloading Da Vinci's notebooks from Project Gutenberg")
49 | r = requests.get(davinci_url)
50 | with open('davinci.txt', 'w', encoding='utf8') as f:
51 | f.write(r.text)
52 |
53 | # Run the serial version
54 | print("Serial word frequency count:")
55 | text = open('davinci.txt', encoding='latin1').read()
56 | tic = time.time()
57 | freqs = wordfreq(text)
58 | toc = time.time()
59 | print_wordfreq(freqs, 10)
60 | print(f"Took {toc - tic:.3f}s to calculate")
61 |
62 | # The parallel version
63 | print("\nParallel word frequency count:")
64 | # split the davinci.txt into one file per engine:
65 | lines = text.splitlines()
66 | nlines = len(lines)
67 | n = len(rc)
68 | block = nlines // n
69 | for i in range(n):
70 | chunk = lines[i * block : i * (block + 1)]
71 | with open(f'davinci{i}.txt', 'w', encoding='utf8') as f:
72 | f.write('\n'.join(chunk))
73 |
74 | try: # python2
75 | cwd = os.path.abspath(os.getcwdu())
76 | except AttributeError: # python3
77 | cwd = os.path.abspath(os.getcwd())
78 | fnames = [os.path.join(cwd, f'davinci{i}.txt') for i in range(n)]
79 | tic = time.time()
80 | pfreqs = pwordfreq(view, fnames)
81 | toc = time.time()
82 | print_wordfreq(freqs)
83 | print(f"Took {toc - tic:.3f} s to calculate on {len(view.targets)} engines")
84 | # cleanup split files
85 | map(os.remove, fnames)
86 |
--------------------------------------------------------------------------------
/ci/ssh/win_Dockerfile_template:
--------------------------------------------------------------------------------
1 | # syntax = docker/dockerfile:1.2.1
2 | FROM python:3.12-windowsservercore-ltsc2022
3 | SHELL ["powershell"]
4 |
5 | # using docker env doesn't seem to work fully correctly. The environment variable is set in the powershell
6 | # but somehow it is not present during pip install process, which will cause the ipyparallel installation
7 | # to fail. Therefore, we use the SetEnvironmentVariable command (for the machine) to make this 'permanent'.
8 | # See run command below.
9 | ENV IPP_DISABLE_JS=1
10 |
11 | # the following env variable values will be 'statically' replaced by the corresponding github workflow script
12 | # if the values aren't replaced the container hosts file isn't changed which is typically no problem in a local
13 | # environment. but it's a necessity for github runners, since the host name resolution inside the container doesn't
14 | # work correctly while trying to register with the ipyparallel controller (for linux runners this isn't an issue)
15 | ENV docker_host_ip ${docker_host_ip}
16 | ENV docker_host_name ${docker_host_name}
17 |
18 | # set IPP_DISABLE_JS=1 and install latest node js version
19 | RUN [System.Environment]::SetEnvironmentVariable('IPP_DISABLE_JS','1', [EnvironmentVariableTarget]::Machine); \
20 | #add the docker host name and ip to the container hosts file (needed for the github runners since the docker host name resolution doesn't work there)
21 | $hostsfile='C:\Windows\System32\drivers\etc\hosts'; \
22 | $line=\"$env:docker_host_ip $env:docker_host_name\"; \
23 | if ($line.Trim().Length -eq 0) { \
24 | Write-Host 'Environment variables docker_host_[name|ip] not set. Hosts file unchanged!'; \
25 | } else { \
26 | Write-Host 'Adapting hosts file '; \
27 | $h=(Get-Content $hostsfile)+$line; \
28 | echo $h | out-file -encoding ASCII $hostsfile; \
29 | type $hostsfile; \
30 | }
31 |
32 | RUN Get-WindowsCapability -Online | Where-Object Name -like 'OpenSSH*'; \
33 | Write-Host 'Install OpenSSH Server...'; \
34 | Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0; \
35 | Write-Host 'Initializing OpenSSH Server...'; \
36 | Start-Service sshd; \
37 | Stop-Service sshd
38 |
39 |
40 | # create ciuser including key pair
41 | RUN Write-Host 'Create user ciuser...';\
42 | NET USER ciuser /add
43 | USER ciuser
44 | RUN Write-Host 'Create key pair and copy public key...';\
45 | ssh-keygen -t rsa -N '\"\"' -f $env:USERPROFILE/.ssh/id_rsa; \
46 | cp $env:USERPROFILE/.ssh/id_rsa.pub $env:USERPROFILE/.ssh/authorized_keys
47 |
48 | # switch back to the admin user
49 | USER containeradministrator
50 |
51 | # This is apparently the only way to keep the sshd service running.
52 | # Running sshd in the foreground in the context of a user (as it is done for linux), doesn't work under Windows.
53 | # Even if it is started as admin user, errors occur during logon (lack of some system rights)
54 | CMD powershell -NoExit -Command "Start-Service sshd"
55 |
--------------------------------------------------------------------------------
/docs/source/examples/interengine/communicator.py:
--------------------------------------------------------------------------------
1 | import socket
2 | import uuid
3 |
4 | import zmq
5 |
6 | from ipyparallel.util import disambiguate_url
7 |
8 |
9 | class EngineCommunicator:
10 | def __init__(self, interface='tcp://*', identity=None):
11 | self._ctx = zmq.Context()
12 | self.socket = self._ctx.socket(zmq.XREP)
13 | self.pub = self._ctx.socket(zmq.PUB)
14 | self.sub = self._ctx.socket(zmq.SUB)
15 |
16 | # configure sockets
17 | self.identity = identity or bytes(uuid.uuid4())
18 | self.socket.IDENTITY = self.identity
19 | self.sub.SUBSCRIBE = b''
20 |
21 | # bind to ports
22 | port = self.socket.bind_to_random_port(interface)
23 | pub_port = self.pub.bind_to_random_port(interface)
24 | self.url = f"{interface}:{port}"
25 | self.pub_url = f"{interface}:{pub_port}"
26 | # guess first public IP from socket
27 | self.location = socket.gethostbyname_ex(socket.gethostname())[-1][0]
28 | self.peers = {}
29 |
30 | def __del__(self):
31 | self.socket.close()
32 | self.pub.close()
33 | self.sub.close()
34 | self._ctx.term()
35 |
36 | @property
37 | def info(self):
38 | """return the connection info for this object's sockets."""
39 | return (self.identity, self.url, self.pub_url, self.location)
40 |
41 | def connect(self, peers):
42 | """connect to peers. `peers` will be a dict of 4-tuples, keyed by name.
43 | {peer : (ident, addr, pub_addr, location)}
44 | where peer is the name, ident is the XREP identity, addr,pub_addr are the
45 | """
46 | for peer, (ident, url, pub_url, location) in peers.items():
47 | self.peers[peer] = ident
48 | if ident != self.identity:
49 | self.sub.connect(disambiguate_url(pub_url, location))
50 | if ident > self.identity:
51 | # prevent duplicate xrep, by only connecting
52 | # engines to engines with higher IDENTITY
53 | # a doubly-connected pair will crash
54 | self.socket.connect(disambiguate_url(url, location))
55 |
56 | def send(self, peers, msg, flags=0, copy=True):
57 | if not isinstance(peers, list):
58 | peers = [peers]
59 | if not isinstance(msg, list):
60 | msg = [msg]
61 | for p in peers:
62 | ident = self.peers[p]
63 | self.socket.send_multipart([ident] + msg, flags=flags, copy=copy)
64 |
65 | def recv(self, flags=0, copy=True):
66 | return self.socket.recv_multipart(flags=flags, copy=copy)[1:]
67 |
68 | def publish(self, msg, flags=0, copy=True):
69 | if not isinstance(msg, list):
70 | msg = [msg]
71 | self.pub.send_multipart(msg, copy=copy)
72 |
73 | def consume(self, flags=0, copy=True):
74 | return self.sub.recv_multipart(flags=flags, copy=copy)
75 |
--------------------------------------------------------------------------------
/ipyparallel/nbextension/static/main.js:
--------------------------------------------------------------------------------
1 | // Copyright (c) IPython Development Team.
2 | // Distributed under the terms of the Modified BSD License.
3 |
4 | define(function (require) {
5 | var $ = require("jquery");
6 | var IPython = require("base/js/namespace");
7 | var clusterlist = require("./clusterlist");
8 |
9 | var cluster_html = $(
10 | [
11 | '',
12 | '
",
22 | '
',
23 | ' ",
30 | "
",
31 | "