52 |
53 | ::: info Some research code
54 |
55 | Running code ...
56 |
57 | `python regression.py --rate=0.1 --logs=1 --name=run-01`
58 |
59 | ... and loading the corresponding results ...
60 |
61 | `python plot_regression_result.py --component=run-01`
62 |
63 | ... are distinct and often redundant.
64 |
65 | This means you have to manually keep track by remembering what the component with `rate=0.1` was called.
66 |
67 | :::
68 |
69 |
70 |
71 | ::: tip machinable research code
72 |
73 | Running code ...
74 |
75 | `machinable regression rate=0.1 logs_=True --launch`
76 |
77 | ... and loading the corresponding results ...
78 |
79 | `machinable regression rate=0.1 logs_=True --launch --plot`
80 |
81 | ... are distinct but use the same abstraction.
82 |
83 | This means no need to worry about names as machinable automatically keeps track if you ran `rate=0.1` before.
84 |
85 | :::
86 |
87 |
88 |
89 | :arrow_right: [Learn more about machinable's approach](./about/approach.md)
90 |
91 |
92 |
93 |
4 | 
5 |
6 |
7 |
5 |
8 |
9 | [](https://github.com/machinable-org/machinable/actions?query=workflow%3Abuild)
10 | [](https://github.com/machinable-org/machinable/pulls?utf8=%E2%9C%93&q=is%3Apr%20author%3Aapp%2Fdependabot)
11 | [](https://github.com/psf/black)
12 | [](https://github.com/machinable-org/machinable/blob/main/LICENSE)
13 |
14 |
15 |
16 | 17 | 18 | **machinable** provides a system to manage research code effectively. Using a unified and modular representation, machinable can help structure your projects in a principled way so you can move quickly while enabling reuse and collaboration. 19 | 20 | Visit the [documentation](https://machinable.org/) to get started. 21 | -------------------------------------------------------------------------------- /src/machinable/__init__.py: -------------------------------------------------------------------------------- 1 | """machinable""" 2 | __all__ = [ 3 | "Element", 4 | "Interface", 5 | "Execution", 6 | "Component", 7 | "Project", 8 | "Storage", 9 | "Mixin", 10 | "Index", 11 | "Scope", 12 | "mixin", 13 | "Schedule", 14 | "get", 15 | ] 16 | __doc__ = """A modular system for machinable research code""" 17 | 18 | from importlib import metadata as importlib_metadata 19 | 20 | from machinable.cli import from_cli 21 | from machinable.component import Component 22 | from machinable.element import Element 23 | from machinable.execution import Execution 24 | from machinable.index import Index 25 | from machinable.interface import Interface 26 | from machinable.mixin import Mixin, mixin 27 | from machinable.project import Project 28 | from machinable.query import Query 29 | from machinable.schedule import Schedule 30 | from machinable.scope import Scope 31 | from machinable.storage import Storage 32 | 33 | 34 | def get_version() -> str: 35 | try: 36 | return importlib_metadata.version(__name__) 37 | except importlib_metadata.PackageNotFoundError: # pragma: no cover 38 | return "unknown" 39 | 40 | 41 | __version__: str = get_version() 42 | 43 | get = Query() 44 | -------------------------------------------------------------------------------- /tests/samples/project/.gitignore: -------------------------------------------------------------------------------- 1 | vendor/.cache/*.p 2 | vendor/fooba/vendor/.cache/*.p 3 | # Temporary and binary files 4 | *~ 5 | *.py[cod] 6 | *.so 7 | *.cfg 8 | !.isort.cfg 9 | !setup.cfg 10 | *.orig 11 | *.log 12 | *.pot 13 | __pycache__/* 14 | .cache/* 15 | .*.swp 16 | */.ipynb_checkpoints/* 17 | 18 | # Project files 19 | .ropeproject 20 | .project 21 | .pydevproject 22 | .settings 23 | .idea 24 | tags 25 | 26 | # Package files 27 | *.egg 28 | *.eggs/ 29 | .installed.cfg 30 | *.egg-info 31 | 32 | # Unittest and coverage 33 | htmlcov/* 34 | .coverage 35 | .tox 36 | junit.xml 37 | coverage.xml 38 | .pytest_cache/ 39 | .hypothesis 40 | 41 | # Build and docs folder/files 42 | build/* 43 | dist/* 44 | sdist/* 45 | docs/api/* 46 | docs/_rst/* 47 | docs/_build/* 48 | cover/* 49 | MANIFEST 50 | 51 | # Per-project virtualenvs 52 | .venv*/ 53 | 54 | 55 | # ---> macOS 56 | *.DS_Store 57 | .AppleDouble 58 | .LSOverride 59 | 60 | # Icon must end with two \r 61 | Icon 62 | 63 | 64 | # Thumbnails 65 | ._* 66 | 67 | # Files that might appear in the root of a volume 68 | .DocumentRevisions-V100 69 | .fseventsd 70 | .Spotlight-V100 71 | .TemporaryItems 72 | .Trashes 73 | .VolumeIcon.icns 74 | .com.apple.timemachine.donotpresent 75 | 76 | # Directories potentially created on remote AFP share 77 | .AppleDB 78 | .AppleDesktop 79 | Network Trash Folder 80 | Temporary Items 81 | .apdisk 82 | -------------------------------------------------------------------------------- /docs/.vitepress/components/ApiExplorer.vue: -------------------------------------------------------------------------------- 1 | 2 |
3 |
4 |
6 |
7 |
8 |
44 |
--------------------------------------------------------------------------------
/docs/examples/globus-storage/test_globus.py:
--------------------------------------------------------------------------------
1 | import json
2 | import os
3 |
4 | import pytest
5 | from machinable import Component, Storage
6 |
7 | try:
8 | import globus_sdk
9 | except ImportError:
10 | globus_sdk = None
11 |
12 |
13 | @pytest.mark.skipif(
14 | globus_sdk is None or "MACHINABLE_GLOBUS_TEST_CONFIG" not in os.environ,
15 | reason="Test requires globus environment",
16 | )
17 | def test_globus_storage():
18 | storage = Storage.make(
19 | "globus",
20 | json.loads(os.environ.get("MACHINABLE_GLOBUS_TEST_CONFIG", "{}")),
21 | ).__enter__()
22 | try:
23 | c = Component()
24 | c.save_file(".test", "test")
25 | c.local_directory("test", create=True)
26 | c.save_file("test/tada", "test")
27 | c.save_file("test/.tada", "test")
28 | c.launch()
29 |
30 | storage.tasks_wait()
31 |
32 | assert storage.contains(c.uuid)
33 |
34 | fetched = os.path.abspath("./storage/fetched")
35 | os.makedirs(fetched, exist_ok=True)
36 | storage.retrieve(c.uuid, fetched)
37 |
38 | cf = Component.from_directory(fetched)
39 | assert cf.load_file(".test") == "test"
40 | assert cf.load_file("test/tada") == "test"
41 | assert cf.load_file("test/.tada") == "test"
42 | assert cf.execution.is_finished()
43 | finally:
44 | storage.__exit__()
45 |
--------------------------------------------------------------------------------
/src/machinable/query.py:
--------------------------------------------------------------------------------
1 | from machinable.collection import InterfaceCollection
2 | from machinable.element import extend, normversion
3 | from machinable.interface import Interface
4 | from machinable.types import Optional, Union, VersionType
5 |
6 |
7 | class Query:
8 | def by_id(self, uuid: str) -> Optional[Interface]:
9 | return Interface.find_by_id(uuid)
10 |
11 | def from_directory(self, directory: str) -> Interface:
12 | return Interface.from_directory(directory)
13 |
14 | def __call__(
15 | self,
16 | module: Union[str, Interface, None] = None,
17 | version: VersionType = None,
18 | **kwargs,
19 | ) -> Interface:
20 | module, version = extend(module, version)
21 | return Interface.get(module, version, **kwargs)
22 |
23 | # modifiers
24 |
25 | def all(
26 | self,
27 | module: Union[None, str, Interface] = None,
28 | version: VersionType = None,
29 | **kwargs,
30 | ) -> "InterfaceCollection":
31 | module, version = extend(module, version)
32 | return Interface.find(module, version, **kwargs)
33 |
34 | def new(
35 | self,
36 | module: Union[None, str, Interface] = None,
37 | version: VersionType = None,
38 | **kwargs,
39 | ) -> Interface:
40 | module, version = extend(module, version)
41 | return Interface.make(module, version, **kwargs)
42 |
--------------------------------------------------------------------------------
/docs/logo/logo.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/tests/test_config.py:
--------------------------------------------------------------------------------
1 | import omegaconf
2 | from machinable.config import (
3 | from_element,
4 | match_method,
5 | rewrite_config_methods,
6 | to_dict,
7 | )
8 |
9 |
10 | def test_config_from_element():
11 | class Dummy:
12 | pass
13 |
14 | assert from_element(Dummy) == ({}, None)
15 |
16 | class HasConf:
17 | class Config:
18 | q: int = 1
19 |
20 | assert from_element(HasConf)[0]["q"] == 1
21 | assert from_element(HasConf())[0]["q"] == 1
22 |
23 | class DictConf:
24 | Config = {"a": 2}
25 |
26 | assert from_element(DictConf)[0]["a"] == 2
27 |
28 |
29 | def test_match_method():
30 | assert match_method("find_me(a=1)") == ("find_me", "a=1")
31 | assert match_method("test(1)") == ("test", "1")
32 | assert match_method("foo") is None
33 | assert match_method("ma$formed()") is None
34 | assert match_method("foo(1) < 1") is None
35 | assert match_method("foo(1) + bar(2)") is None
36 | assert match_method(" foo(1)") is None
37 |
38 |
39 | def test_rewrite_config_methods():
40 | rewrite_config_methods({"test": "test_me(1)"}) == {
41 | "test": "${config_method:test_me,1}"
42 | }
43 |
44 |
45 | def test_to_dict():
46 | assert to_dict({"a": 1}) == {"a": 1}
47 | assert to_dict(omegaconf.DictConfig({"a": 1})) == {"a": 1}
48 | assert to_dict(omegaconf.ListConfig([1, 2, 3])) == [1, 2, 3]
49 | assert to_dict({"a": omegaconf.DictConfig({"b": 1})}) == {"a": {"b": 1}}
50 | assert to_dict([1, 2, 3]) == [1, 2, 3]
51 | assert to_dict((1, 2, 3)) == (1, 2, 3)
52 | assert to_dict(1) == 1
53 | assert to_dict("foo") == "foo"
54 |
--------------------------------------------------------------------------------
/.github/workflows/docs.yml:
--------------------------------------------------------------------------------
1 | name: docs
2 | on:
3 | push:
4 | branches: [main]
5 | jobs:
6 | build-and-deploy:
7 | concurrency: ci-${{ github.ref }}
8 | runs-on: ubuntu-latest
9 | steps:
10 | - name: Checkout
11 | uses: actions/checkout@v3
12 | - name: Set up Python
13 | uses: actions/setup-python@v4.7.0
14 | with:
15 | python-version: "3.x"
16 | - name: Install Poetry
17 | uses: snok/install-poetry@v1.3
18 | with:
19 | virtualenvs-create: true
20 | virtualenvs-in-project: true
21 | - name: Set up cache
22 | id: cached-poetry-dependencies
23 | uses: actions/cache@v3.3.1
24 | with:
25 | path: .venv
26 | key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
27 | - name: Install dependencies
28 | run: poetry install --extras "all"
29 | if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
30 | - name: Autogenerate reference documentation
31 | run: |
32 | source .venv/bin/activate
33 | python docs/.vitepress/pydocgen.py
34 | - name: Install vitepress
35 | uses: bahmutov/npm-install@v1.8.34
36 | - name: Build
37 | run: |
38 | /home/runner/work/machinable/machinable/node_modules/vitepress/bin/vitepress.js build docs
39 | cp -r docs/.vitepress/public/* docs/.vitepress/dist/
40 | cp -r docs/logo docs/.vitepress/dist/
41 | - name: Deploy
42 | uses: JamesIves/github-pages-deploy-action@v4.4.3
43 | with:
44 | branch: gh-pages
45 | folder: docs/.vitepress/dist
46 |
--------------------------------------------------------------------------------
/docs/examples/scopes/group.py:
--------------------------------------------------------------------------------
1 | from typing import Optional, Tuple
2 |
3 | from datetime import datetime
4 |
5 | from machinable import schema
6 | from machinable.element import get_dump, get_lineage
7 | from machinable.scope import Scope
8 |
9 |
10 | def normgroup(group: Optional[str]) -> str:
11 | if group is None:
12 | return ""
13 | if not isinstance(group, str):
14 | raise ValueError(f"Invalid group. Expected str but got '{group}'")
15 | return group.lstrip("/")
16 |
17 |
18 | def resolve_group(group: str) -> Tuple[str, str]:
19 | group = normgroup(group)
20 | resolved = datetime.now().strftime(group)
21 | return group, resolved
22 |
23 |
24 | class Group(schema.Scope):
25 | kind: str = "Group"
26 | pattern: str
27 | path: Optional[str] = None
28 |
29 |
30 | class Group(Scope):
31 | """Group element"""
32 |
33 | kind = "Group"
34 |
35 | def __init__(self, group: Optional[str] = None):
36 | super().__init__(version=None)
37 | pattern, path = resolve_group(group)
38 | self.__model__ = schema.Group(
39 | kind=self.kind,
40 | module=self.__model__.module,
41 | config=self.__model__.config,
42 | version=self.__model__.version,
43 | pattern=pattern,
44 | path=path,
45 | lineage=get_lineage(self),
46 | )
47 | self.__model__._dump = get_dump(self)
48 |
49 | @property
50 | def pattern(self) -> str:
51 | return self.__model__.pattern
52 |
53 | @property
54 | def path(self) -> str:
55 | return self.__model__.path
56 |
57 | def __call__(self):
58 | return {"group": self.path}
59 |
--------------------------------------------------------------------------------
/docs/guide/component.md:
--------------------------------------------------------------------------------
1 | # Component
2 |
3 | While interfaces are designed to associate data with code, 33 | 34 |
35 | 36 |
37 | 38 |
39 | 40 | 41 |
44 |
94 |
95 |
45 |
46 |
47 |
48 | 💻
49 |
50 |
51 | 98 |
99 |
100 | 101 |
102 |
103 |
104 |
159 |
--------------------------------------------------------------------------------
/tests/test_storage.py:
--------------------------------------------------------------------------------
1 | import os
2 | import shutil
3 |
4 | from machinable import Storage
5 |
6 |
7 | class CopyStorage(Storage):
8 | class Config:
9 | directory: str = ""
10 |
11 | def commit(self, interface) -> bool:
12 | directory = os.path.join(self.config.directory, interface.uuid)
13 | if not os.path.exists(directory):
14 | os.makedirs(directory)
15 | interface.to_directory(directory)
16 |
17 | def contains(self, uuid):
18 | return os.path.exists(os.path.join(self.config.directory, uuid))
19 |
20 | def retrieve(self, uuid, local_directory) -> bool:
21 | if not self.contains(uuid):
22 | return False
23 |
24 | shutil.copytree(
25 | os.path.join(self.config.directory, uuid),
26 | local_directory,
27 | dirs_exist_ok=True,
28 | )
29 |
30 | return True
31 |
32 |
33 | def test_storage(tmp_path):
34 | from machinable import Index, get
35 |
36 | primary = str(tmp_path / "primary")
37 | secondary = str(tmp_path / "secondary")
38 |
39 | i = Index(
40 | {"directory": primary, "database": str(tmp_path / "index.sqlite")}
41 | ).__enter__()
42 |
43 | st2 = CopyStorage({"directory": secondary}).__enter__()
44 | st1 = Storage().__enter__()
45 |
46 | project = get("machinable.project", "tests/samples/project").__enter__()
47 |
48 | interface1 = get("dummy").commit()
49 | interface2 = get("dummy", {"a": 5}).commit()
50 |
51 | assert os.path.exists(os.path.join(primary, interface1.uuid))
52 | assert os.path.exists(os.path.join(secondary, interface1.uuid))
53 |
54 | # delete primary source and reload from remote
55 | shutil.rmtree(primary)
56 | assert not os.path.exists(interface1.local_directory())
57 | assert not os.path.exists(interface2.local_directory())
58 | interface1_reload = get("dummy")
59 | interface1_reload.fetch()
60 | assert os.path.exists(interface1_reload.local_directory())
61 | assert not os.path.exists(interface2.local_directory())
62 | interface2_reload = get("dummy", {"a": 5})
63 | interface2_reload.fetch()
64 | assert os.path.exists(interface2.local_directory())
65 |
66 | project.__exit__()
67 | st1.__exit__()
68 | st2.__exit__()
69 | i.__exit__()
70 |
71 |
72 | def test_storage_upload_and_download(tmp_path):
73 | from machinable import Index, get
74 |
75 | primary = str(tmp_path / "primary")
76 | secondary = str(tmp_path / "secondary")
77 |
78 | i = Index(primary).__enter__()
79 | local = Index(str(tmp_path / "download"))
80 |
81 | storage = CopyStorage({"directory": secondary})
82 |
83 | project = get("machinable.project", "tests/samples/project").__enter__()
84 |
85 | interface1 = get("dummy").launch()
86 | interface2 = get("dummy", {"a": 5}, uses=interface1).launch()
87 |
88 | assert not os.path.exists(tmp_path / "secondary" / interface2.uuid)
89 | storage.upload(interface2)
90 | assert os.path.exists(tmp_path / "secondary" / interface1.uuid)
91 | assert os.path.exists(tmp_path / "secondary" / interface2.uuid)
92 |
93 | assert not local.find(interface2)
94 | with local:
95 | downloads = storage.download(interface2.uuid, related=False)
96 | assert len(downloads) == 1
97 | assert os.path.exists(local.local_directory(interface2.uuid))
98 | assert not os.path.exists(local.local_directory(interface1.uuid))
99 | assert local.find(interface2)
100 | assert not local.find(interface1)
101 |
102 | downloads = storage.download(interface2.uuid, related=True)
103 | assert len(downloads) == 2
104 | assert os.path.exists(local.local_directory(interface1.uuid))
105 | assert local.find(interface1)
106 |
107 | project.__exit__()
108 | i.__exit__()
109 |
--------------------------------------------------------------------------------
/docs/guide/element.md:
--------------------------------------------------------------------------------
1 | # Element
2 |
3 | At a basic level, machinable projects are regular Python projects consisting of *elements* - configurable classes that encapsulate parts of your code.
4 | For example, an element implementation of a dataset might look like this:
5 |
6 | ```python
7 | from machinable import Element # Element base class
8 |
9 | class MnistData(Element):
10 | """A dataset of handwritten characters"""
11 | Config = {
12 | "batch_size": 8,
13 | "name": "mnist"
14 | }
15 | ```
16 |
17 | When inheriting from