├── example ├── site.yaml ├── projects │ └── example.yaml ├── bench │ ├── test.yaml │ ├── testbench.yaml │ ├── bench.py │ └── Makefile ├── design │ └── top │ │ ├── design.yaml │ │ ├── top.py │ │ ├── counter.sv │ │ ├── adder.sv │ │ └── top.sv.mako ├── infra │ ├── tools │ │ ├── pythonsite.txt │ │ ├── wave_viewers.py │ │ ├── simulators.py │ │ ├── misc.py │ │ └── objstore.py │ ├── interfaces │ │ └── interfaces.py │ ├── transforms │ │ ├── lint.py │ │ ├── examples.py │ │ └── templating.py │ ├── config │ │ └── config.py │ ├── caches.py │ └── workflows.py ├── .caches.yaml ├── .bw.yaml └── README.md ├── docs ├── assets │ ├── logo.png │ ├── mascot_b_black_e_white.png │ ├── mascot_white.svg │ └── foundation │ │ └── anchors-spec-to-gds.svg ├── cli │ ├── bootstrap.md │ ├── info.md │ ├── tools.md │ ├── tool.md │ ├── shell.md │ ├── introduction.md │ └── exec.md ├── config │ ├── bw_yaml.md │ └── caching.md ├── tech │ ├── state.md │ └── caching.md ├── index.md └── syntax │ └── bootstrap.md ├── blockwork ├── tools │ ├── tools │ │ ├── __init__.py │ │ └── shell.py │ └── __init__.py ├── transforms │ ├── transforms │ │ ├── __init__.py │ │ └── file.py │ └── __init__.py ├── containerfiles │ └── foundation │ │ ├── launch.sh │ │ ├── README.md │ │ ├── Containerfile_arm │ │ ├── Containerfile_x86 │ │ └── forwarder │ │ └── blockwork ├── __init__.py ├── common │ ├── __init__.py │ ├── yaml │ │ └── __init__.py │ ├── complexnamespaces.py │ ├── singleton.py │ ├── inithooks.py │ ├── scopes.py │ ├── registry.py │ └── checkeddataclasses.py ├── build │ ├── __init__.py │ └── file.py ├── workflows │ └── __init__.py ├── config │ ├── __init__.py │ ├── caching.py │ ├── blockwork.py │ ├── base.py │ └── scheduler.py ├── containers │ └── __init__.py ├── bootstrap │ ├── __init__.py │ ├── containers.py │ ├── tools.py │ └── bootstrap.py ├── activities │ ├── __init__.py │ ├── shell.py │ ├── bootstrap.py │ ├── exec.py │ ├── info.py │ ├── workflow.py │ ├── tools.py │ ├── common.py │ └── cache.py ├── state.py └── __main__.py ├── .pre-commit-config.yaml ├── tests ├── __init__.py ├── conftest.py ├── test_scheduler.py ├── test_config.py ├── test_context.py └── test_state.py ├── .github ├── composites │ └── setup │ │ └── action.yaml └── workflows │ └── ci.yml ├── mkdocs.yml ├── pyproject.toml ├── .gitignore └── README.md /example/site.yaml: -------------------------------------------------------------------------------- 1 | !Site 2 | projects: 3 | ex: projects/example.yaml 4 | -------------------------------------------------------------------------------- /example/projects/example.yaml: -------------------------------------------------------------------------------- 1 | !Project 2 | units: 3 | top: design/top 4 | bench: bench 5 | -------------------------------------------------------------------------------- /docs/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blockwork-eda/blockwork/HEAD/docs/assets/logo.png -------------------------------------------------------------------------------- /example/bench/test.yaml: -------------------------------------------------------------------------------- 1 | !Test 2 | tests: 3 | - !Build 4 | target: !Testbench bench 5 | match: mako 6 | -------------------------------------------------------------------------------- /example/bench/testbench.yaml: -------------------------------------------------------------------------------- 1 | !Testbench 2 | design: !Design top 3 | bench_python: bench/bench.py 4 | bench_make: bench/Makefile 5 | -------------------------------------------------------------------------------- /docs/assets/mascot_b_black_e_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blockwork-eda/blockwork/HEAD/docs/assets/mascot_b_black_e_white.png -------------------------------------------------------------------------------- /blockwork/tools/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # Expose various definitions 2 | from .shell import Bash 3 | 4 | # Unused import lint guards 5 | assert all((Bash,)) 6 | -------------------------------------------------------------------------------- /blockwork/transforms/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | # Expose various definitions 2 | from .file import Copy 3 | 4 | # Unused import lint guards 5 | assert all((Copy,)) 6 | -------------------------------------------------------------------------------- /example/design/top/design.yaml: -------------------------------------------------------------------------------- 1 | !Design 2 | top: top 3 | transforms: 4 | - !Mako 5 | template: top.sv.mako 6 | output: top.sv 7 | sources: 8 | - adder.sv 9 | - counter.sv 10 | - top.sv 11 | -------------------------------------------------------------------------------- /example/infra/tools/pythonsite.txt: -------------------------------------------------------------------------------- 1 | cocotb==1.8.0 2 | cocotb-bus==0.2.1 3 | cocotb-coverage==1.1.0 4 | find-libpython==0.3.0 5 | Mako==1.2.4 6 | MarkupSafe==2.1.3 7 | python-constraint==1.4.0 8 | PyYAML==6.0.1 9 | -------------------------------------------------------------------------------- /example/.caches.yaml: -------------------------------------------------------------------------------- 1 | !Caching 2 | targets: False 3 | enabled: True 4 | caches: 5 | - !Cache 6 | name: local-cache 7 | path: infra.caches.BasicFileCache 8 | fetch_condition: True 9 | store_condition: True 10 | max_size: 5GB 11 | -------------------------------------------------------------------------------- /blockwork/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | # Expose various definitions 2 | from . import transforms 3 | from .transform import IN, OUT, EnvPolicy, IEnv, IFace, IPath, Transform, TransformResult 4 | 5 | # Unused import lint guards 6 | assert all((EnvPolicy, IEnv, IFace, IN, IPath, OUT, Transform, TransformResult, transforms)) 7 | -------------------------------------------------------------------------------- /example/design/top/top.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import ClassVar 3 | 4 | from blockwork.build import Entity 5 | 6 | 7 | @Entity.register() 8 | class Top(Entity): 9 | files: ClassVar[list[Path]] = [ 10 | Entity.ROOT / "adder.sv", 11 | Entity.ROOT / "counter.sv", 12 | Entity.ROOT / "top.sv.mako", 13 | ] 14 | -------------------------------------------------------------------------------- /example/.bw.yaml: -------------------------------------------------------------------------------- 1 | !Blockwork 2 | project: example 3 | site: ./site.yaml 4 | default_cache_config: ./caches.yaml 5 | tooldefs: 6 | - infra.tools.misc 7 | - infra.tools.compilers 8 | - infra.tools.simulators 9 | - infra.tools.wave_viewers 10 | workflows: 11 | - infra.workflows 12 | caches: 13 | - infra.caches.BasicFileCache 14 | config: 15 | - infra.config.config 16 | -------------------------------------------------------------------------------- /blockwork/containerfiles/foundation/launch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # If BLOCKWORK_XTOKEN provided, append it to Xauthority 4 | if [[ ! -z "$BLOCKWORK_XTOKEN" ]]; then 5 | export XAUTHORITY="/tmp/.Xauthority" 6 | touch $XAUTHORITY 7 | xauth add $DISPLAY MIT-MAGIC-COOKIE-1 $BLOCKWORK_XTOKEN 8 | fi 9 | 10 | # Execute whatever command was passed in 11 | /bin/bash $BLOCKWORK_CMD 12 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # Example Project 2 | 3 | ## Running Testbench 4 | 5 | ```bash 6 | $> bw exec -- make -f bench/Makefile run_cocotb 7 | ``` 8 | 9 | ## Viewing Waves 10 | 11 | ```bash 12 | $> bw tool gtkwave ../example.scratch/waves.lxt 13 | ``` 14 | 15 | ## Running workflows 16 | ```bash 17 | $> bw wf build -p ex -t top:design 18 | $> bw wf test -p ex -t top:design 19 | $> bw wf lint -p ex -t top:design 20 | ``` 21 | -------------------------------------------------------------------------------- /blockwork/containerfiles/foundation/README.md: -------------------------------------------------------------------------------- 1 | # Foundation Container 2 | 3 | This directory contains the definition of the foundation container. 4 | 5 | ## Building 6 | 7 | ```bash 8 | $> cd containers/foundation 9 | $> podman build --file=Containerfile --format=oci --rm=true --tag=foundation . 10 | ``` 11 | 12 | ## Saving 13 | 14 | ```bash 15 | $> podman save foundation --format=oci-archive --output=foundation.tar 16 | ``` 17 | -------------------------------------------------------------------------------- /blockwork/transforms/transforms/file.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from ...context import Context 4 | from ...tools import tools 5 | from ..transform import Transform 6 | 7 | 8 | class Copy(Transform): 9 | frm: Path = Transform.IN() 10 | to: Path = Transform.OUT(init=True, default=...) 11 | bash: tools.Bash = Transform.TOOL() 12 | 13 | def execute(self, ctx: Context): 14 | yield self.bash.cp(ctx, frm=self.frm, to=self.to) 15 | -------------------------------------------------------------------------------- /docs/cli/bootstrap.md: -------------------------------------------------------------------------------- 1 | The `bootstrap` command runs all known [bootstrapping](../syntax/bootstrap.md) 2 | steps. 3 | 4 | It has no options or sub-commands. 5 | 6 | ## Usage Example 7 | 8 | ```bash 9 | $> bw bootstrap 10 | [20:33:20] INFO Importing 1 bootstrapping paths 11 | INFO Invoking 1 bootstrap methods 12 | INFO Ran bootstrap step 'infra.bootstrap.tool_a.setup_tool_a' 13 | INFO Bootstrap complete 14 | ``` 15 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | # Basic checks 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v4.5.0 5 | hooks: 6 | - id: trailing-whitespace 7 | - id: check-merge-conflict 8 | - id: end-of-file-fixer 9 | - id: no-commit-to-branch 10 | args: [--branch, main] 11 | # Ruff Python linting and formatting 12 | - repo: https://github.com/astral-sh/ruff-pre-commit 13 | rev: v0.2.1 14 | hooks: 15 | - id: ruff 16 | args: [--fix] 17 | - id: ruff-format 18 | -------------------------------------------------------------------------------- /docs/cli/info.md: -------------------------------------------------------------------------------- 1 | The `info` command tabulates various details about the active Blockwork project. 2 | 3 | It has no options or sub-commands. 4 | 5 | ## Usage Example 6 | 7 | ```bash 8 | $> bw info 9 | ┌────────────────────┬──────────────────────────────┐ 10 | │ Project │ example │ 11 | │ Root Directory │ /path/to/my/project │ 12 | │ Configuration File │ /path/to/my/project/.bw.yaml │ 13 | └────────────────────┴──────────────────────────────┘ 14 | ``` 15 | -------------------------------------------------------------------------------- /docs/cli/tools.md: -------------------------------------------------------------------------------- 1 | The `tools` command tabulates all of the available tools from all known vendors, 2 | listing each version and identifying which is the default. 3 | 4 | It has no options or sub-commands. 5 | 6 | ## Usage Example 7 | 8 | ```bash 9 | $> bw tools 10 | ┏━━━━━━━━┳━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ 11 | ┃ Vendor ┃ Tool ┃ Version ┃ Default ┃ 12 | ┡━━━━━━━━╇━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ 13 | │ N/A │ gcc │ 13.1.0 │ ✔ │ 14 | │ N/A │ make │ 4.4 │ ✔ │ 15 | │ │ │ 4.3 │ │ 16 | └────────┴──────┴─────────┴─────────┘ 17 | ``` 18 | -------------------------------------------------------------------------------- /example/design/top/counter.sv: -------------------------------------------------------------------------------- 1 | module counter #( 2 | parameter WIDTH = 32 3 | ) ( 4 | input logic i_clk 5 | , input logic i_rst 6 | , output logic [WIDTH-1:0] o_count 7 | ); 8 | 9 | logic [WIDTH-1:0] count, count_q; 10 | 11 | always_comb begin : comb_count 12 | count = count_q + 'd1; 13 | end 14 | 15 | always_ff @(posedge i_clk, posedge i_rst) begin : ff_count 16 | if (i_rst) begin 17 | count_q <= 'd0; 18 | end else begin 19 | count_q <= count; 20 | end 21 | end 22 | 23 | assign o_count = count_q; 24 | 25 | endmodule : counter 26 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /blockwork/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /blockwork/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /example/bench/bench.py: -------------------------------------------------------------------------------- 1 | import cocotb 2 | from cocotb.clock import Clock 3 | from cocotb.triggers import ClockCycles 4 | 5 | 6 | @cocotb.test() 7 | async def smoke(dut): 8 | # Start a clock 9 | dut._log.info("Starting clock") 10 | cocotb.start_soon(Clock(dut.i_clk, 1, units="ns").start()) 11 | # Drive reset high 12 | dut._log.info("Driving reset high") 13 | dut.i_rst.value = 1 14 | # Wait for 50 cycles 15 | dut._log.info("Waiting for 50 cycles") 16 | await ClockCycles(dut.i_clk, 50) 17 | # Drive the reset low 18 | dut._log.info("Driving reset high") 19 | dut.i_rst.value = 0 20 | # Run for 1000 cycles 21 | dut._log.info("Running for 1000 cycles") 22 | await ClockCycles(dut.i_clk, 1000) 23 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable 2 | from datetime import datetime 3 | from pathlib import Path 4 | 5 | import pytest 6 | 7 | from blockwork.bootstrap import build_foundation 8 | from blockwork.config.api import ConfigApi 9 | from blockwork.context import Context 10 | 11 | 12 | @pytest.fixture(name="api") 13 | def api(tmp_path: Path) -> Iterable["ConfigApi"]: 14 | "Fixture to create a basic api object from dummy bw config" 15 | bw_yaml = tmp_path / ".bw.yaml" 16 | with bw_yaml.open("w", encoding="utf-8") as fh: 17 | fh.write("!Blockwork\nproject: test\n") 18 | ctx = Context(tmp_path) 19 | build_foundation(ctx, datetime.min) 20 | with ConfigApi(Context(tmp_path)) as api: 21 | yield api 22 | -------------------------------------------------------------------------------- /docs/cli/tool.md: -------------------------------------------------------------------------------- 1 | The `tool` command launches an action offered by a [Tool](../syntax/tools.md) 2 | declaration (see the [Tool](../syntax/tools.md) documentation on how actions are 3 | declared). These can run interactively and with X11 display forwarding, as 4 | specified by the action. 5 | 6 | Similarly to [exec](exec.md), the exit code of the contained process will be 7 | forwarded to the host. 8 | 9 | The `tool` command has no sub-commands or options of its own so all positional 10 | arguments, and those after a `--` delimiter, are forwarded to the action. 11 | 12 | ## Usage Example 13 | 14 | As with [exec](exec.md), using the explicit argument delimiter is recommended: 15 | 16 | ```bash 17 | $> bw tool gtkwave.view -- a/waves.vcd b/waves.gtkw 18 | ``` 19 | -------------------------------------------------------------------------------- /.github/composites/setup/action.yaml: -------------------------------------------------------------------------------- 1 | name: Project Setup 2 | 3 | inputs: 4 | python-version: 5 | description: Python version to install 6 | required: true 7 | default: "3.11" 8 | 9 | runs: 10 | using: composite 11 | steps: 12 | # 1. Setup Python 13 | - name: Set up Python 14 | uses: actions/setup-python@v5 15 | with: 16 | python-version: ${{ inputs.python-version }} 17 | # 2. Install Poetry 18 | - name: Install Poetry 19 | shell: bash 20 | run: | 21 | python -m pip install --upgrade pip 22 | python -m pip install poetry poethepoet 23 | # 3. Install development dependencies 24 | - name: Setup environment 25 | shell: bash 26 | run: | 27 | poetry install --with dev 28 | -------------------------------------------------------------------------------- /blockwork/build/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .file import FileType 16 | from .caching import Cache 17 | 18 | assert all((FileType, Cache)) 19 | -------------------------------------------------------------------------------- /blockwork/tools/tools/shell.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from blockwork.context import Context 4 | 5 | from ..tool import Invocation, Tool, Version 6 | 7 | 8 | @Tool.register() 9 | class Bash(Tool): 10 | versions = ( 11 | Version( 12 | location=Tool.HOST_ROOT / "bash" / "1.0", 13 | version="1.0", 14 | default=True, 15 | ), 16 | ) 17 | 18 | @Tool.action(default=True) 19 | def script(self, ctx: Context, *script: str) -> Invocation: 20 | return Invocation(tool=self, execute="bash", args=["-c", " && ".join(script)]) 21 | 22 | @Tool.action() 23 | def cp(self, ctx: Context, frm: str | Path, to: str | Path) -> Invocation: 24 | return Invocation(tool=self, execute="cp", args=["-r", frm, to]) 25 | -------------------------------------------------------------------------------- /blockwork/workflows/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Expose various definitions 16 | from .workflow import Workflow 17 | 18 | # Unused import lint guards 19 | assert all((Workflow,)) 20 | -------------------------------------------------------------------------------- /blockwork/config/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .blockwork import Blockwork, BlockworkParser 16 | from .caching import CacheConfig, CachingConfig, CachingParser 17 | 18 | assert all((Blockwork, BlockworkParser, CacheConfig, CachingConfig, CachingParser)) 19 | -------------------------------------------------------------------------------- /example/design/top/adder.sv: -------------------------------------------------------------------------------- 1 | module adder #( 2 | parameter WIDTH = 32 3 | ) ( 4 | input logic i_clk 5 | , input logic i_rst 6 | , input logic [WIDTH-1:0] i_value_a 7 | , input logic [WIDTH-1:0] i_value_b 8 | , output logic [WIDTH-1:0] o_sum 9 | , output logic o_overflow 10 | ); 11 | 12 | logic [WIDTH-1:0] sum, sum_q; 13 | logic overflow, overflow_q; 14 | 15 | always_comb begin : comb_add 16 | {overflow, sum} = i_value_a + i_value_b; 17 | end 18 | 19 | always_ff @(posedge i_clk, posedge i_rst) begin : ff_add 20 | if (i_rst) begin 21 | sum_q <= 'd0; 22 | overflow_q <= 'd0; 23 | end else begin 24 | sum_q <= sum; 25 | overflow_q <= overflow; 26 | end 27 | end 28 | 29 | assign o_sum = sum_q; 30 | assign o_overflow = overflow_q; 31 | 32 | endmodule : adder 33 | -------------------------------------------------------------------------------- /blockwork/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Expose various definitions 16 | from . import tools 17 | from .tool import Invocation, Require, Tool, ToolError, ToolMode, Version 18 | 19 | # Unused import lint guards 20 | assert all((Invocation, Require, Tool, ToolError, ToolMode, Version, tools)) 21 | -------------------------------------------------------------------------------- /example/infra/interfaces/interfaces.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from blockwork.transforms import IFace 18 | 19 | 20 | class DesignInterface(IFace): 21 | sources: list[Path] = IFace.FIELD(default_factory=list) 22 | headers: list[Path] = IFace.FIELD(default_factory=list) 23 | -------------------------------------------------------------------------------- /blockwork/containers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Expose various definitions 16 | from .container import Container, ContainerBindError, ContainerError, ContainerResult 17 | from .runtime import Runtime 18 | 19 | # Unused import lint guards 20 | assert all((Container, ContainerError, ContainerBindError, ContainerResult, Runtime)) 21 | -------------------------------------------------------------------------------- /blockwork/bootstrap/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Bootstrap infrastructure 16 | from .bootstrap import Bootstrap, BwBootstrapMode 17 | 18 | # Built-in bootstrapping rules 19 | from .containers import build_foundation 20 | from .tools import install_tools 21 | 22 | # Lint guards 23 | assert all((Bootstrap, BwBootstrapMode)) 24 | assert all((build_foundation, install_tools)) 25 | -------------------------------------------------------------------------------- /blockwork/activities/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Expose activities 16 | from .bootstrap import bootstrap 17 | from .cache import cache 18 | from .exec import exec 19 | from .info import info 20 | from .shell import shell 21 | from .tools import tool, tools 22 | from .workflow import wf, wf_step 23 | 24 | # List all activities 25 | activities = (bootstrap, cache, info, exec, shell, tool, tools, wf, wf_step) 26 | 27 | # Lint guard 28 | assert activities 29 | -------------------------------------------------------------------------------- /blockwork/containerfiles/foundation/Containerfile_arm: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/arm64 docker.io/library/rockylinux:9.1 2 | ADD forwarder/blockwork /usr/bin/blockwork 3 | ADD launch.sh /usr/bin/launch.sh 4 | RUN \ 5 | ln -s /usr/bin/blockwork /usr/bin/bw && \ 6 | dnf update -y && \ 7 | dnf install -y epel-release && \ 8 | dnf install -y which htop nano gtk2 gtk3 gtk3-devel wget xz cpio iputils xterm \ 9 | gcc perl bzip2 bzip2-devel tcl tcl-devel tk-devel libnsl \ 10 | openssl-devel diffutils libedit-devel readline-devel \ 11 | glibc-langpack-en tcsh libX11 libXext procps-ng libXft \ 12 | xcb-util xcb-util-wm xcb-util-image xcb-util-keysyms \ 13 | xcb-util-renderutil libxkbcommon-x11 mesa-dri-drivers \ 14 | xorg-x11-utils xorg-x11-xauth libXScrnSaver numactl-libs \ 15 | mesa-libGL time bc && \ 16 | ln -s /usr/lib64/libbz2.so.1 /usr/lib64/libbz2.so.1.0 17 | # Set locale 18 | ENV LANG=en_GB.utf8 19 | # Ensure launch shell is executable 20 | RUN chmod +x /usr/bin/launch.sh 21 | -------------------------------------------------------------------------------- /example/infra/transforms/lint.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from blockwork.transforms import Transform 17 | 18 | from ..interfaces.interfaces import DesignInterface 19 | from ..tools.simulators import Verilator 20 | 21 | 22 | class VerilatorLintTransform(Transform): 23 | verilator: Verilator = Transform.TOOL() 24 | design: DesignInterface = Transform.IN() 25 | 26 | def execute(self, ctx): 27 | yield self.verilator.run(ctx, "--lint-only", "-Wall", *self.design["sources"]) 28 | -------------------------------------------------------------------------------- /blockwork/build/file.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from ..common.singleton import ParameterisedSingleton 18 | 19 | class FileType(metaclass=ParameterisedSingleton): 20 | 21 | def __init__(self, extension : str) -> None: 22 | self.extension = extension 23 | 24 | def __repr__(self) -> str: 25 | return f"" 26 | 27 | @classmethod 28 | def from_path(cls, path : Path) -> "FileType": 29 | return FileType("".join(path.suffixes)) 30 | -------------------------------------------------------------------------------- /blockwork/common/yaml/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .converters import ( 16 | ConverterRegistry, 17 | DataclassConverter, 18 | YamlConversionError, 19 | YamlExtraFieldsError, 20 | YamlFieldError, 21 | YamlMissingFieldsError, 22 | ) 23 | from .parsers import Parser, SimpleParser 24 | 25 | assert all( 26 | ( 27 | ConverterRegistry, 28 | DataclassConverter, 29 | YamlConversionError, 30 | YamlFieldError, 31 | YamlMissingFieldsError, 32 | YamlExtraFieldsError, 33 | Parser, 34 | SimpleParser, 35 | ) 36 | ) 37 | -------------------------------------------------------------------------------- /blockwork/bootstrap/containers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from datetime import datetime 16 | 17 | from rich.console import Console 18 | 19 | from ..context import Context 20 | from ..foundation import Foundation 21 | from .bootstrap import Bootstrap 22 | 23 | 24 | @Bootstrap.register() 25 | def build_foundation(context: Context, last_run: datetime) -> bool: 26 | """ 27 | Built-in bootstrap action that builds the foundation container using the 28 | active runtime. 29 | """ 30 | with Console().status("Building container...", spinner="arc"): 31 | Foundation(context).build() 32 | return True 33 | -------------------------------------------------------------------------------- /blockwork/containerfiles/foundation/Containerfile_x86: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 docker.io/library/rockylinux:9.1 2 | ADD forwarder/blockwork /usr/bin/blockwork 3 | ADD launch.sh /usr/bin/launch.sh 4 | RUN \ 5 | ln -s /usr/bin/blockwork /usr/bin/bw && \ 6 | dnf update -y && \ 7 | dnf install -y epel-release && \ 8 | dnf install -y which htop nano gtk2 gtk3 gtk3-devel wget xz cpio iputils \ 9 | xterm gcc perl bzip2 bzip2-devel tcl tcl-devel tk-devel \ 10 | libnsl openssl-devel diffutils libedit-devel gmp-devel \ 11 | readline-devel glibc-langpack-en tcsh glibc.i686 \ 12 | glibc-devel.i686 libgcc.i686 libX11 libX11.i686 libXext \ 13 | libXext.i686 procps-ng libXft libXft.i686 xcb-util \ 14 | xcb-util.i686 xcb-util-wm xcb-util-image xcb-util-keysyms \ 15 | xcb-util-renderutil libxkbcommon-x11 mesa-dri-drivers \ 16 | xorg-x11-utils xorg-x11-xauth libXScrnSaver numactl-libs \ 17 | mesa-libGL time bc && \ 18 | ln -s /usr/lib64/libbz2.so.1 /usr/lib64/libbz2.so.1.0 19 | # Set locale 20 | ENV LANG=en_GB.utf8 21 | # Ensure launch shell is executable 22 | RUN chmod +x /usr/bin/launch.sh 23 | -------------------------------------------------------------------------------- /example/design/top/top.sv.mako: -------------------------------------------------------------------------------- 1 | module top #( 2 | parameter WIDTH = 32 3 | ) ( 4 | input logic i_clk 5 | , input logic i_rst 6 | , output logic [WIDTH-1:0] o_result 7 | , output logic o_overflow 8 | ); 9 | 10 | logic [WIDTH-1:0] count_a, count_b; 11 | 12 | %for sfx in ("a", "b"): 13 | counter #( 14 | .WIDTH ( WIDTH ) 15 | ) u_count_${sfx} ( 16 | .i_clk ( i_clk ) 17 | , .i_rst ( i_rst ) 18 | , .o_count ( count_${sfx} ) 19 | ); 20 | %endfor 21 | 22 | adder #( 23 | .WIDTH ( WIDTH ) 24 | ) u_adder ( 25 | .i_clk ( i_clk ) 26 | , .i_rst ( i_rst ) 27 | , .i_value_a ( count_a ) 28 | , .i_value_b ( count_b ) 29 | , .o_sum ( o_result ) 30 | , .o_overflow ( o_overflow ) 31 | ); 32 | 33 | initial begin : init_waves 34 | string f_name; 35 | $timeformat(-9, 2, " ns", 20); 36 | if ($value$plusargs("WAVE_FILE=%s", f_name)) begin 37 | $display("%0t: Capturing wave file %s", $time, f_name); 38 | $dumpfile(f_name); 39 | $dumpvars(0, top); 40 | end else begin 41 | $display("%0t: No filename provided - disabling wave capture", $time); 42 | end 43 | end 44 | 45 | endmodule : top 46 | -------------------------------------------------------------------------------- /example/infra/transforms/examples.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from infra.tools.misc import PythonSite 18 | 19 | from blockwork.context import Context 20 | from blockwork.transforms import Transform 21 | 22 | 23 | class CapturedTransform(Transform): 24 | "Transform with stdout captured to a file interface" 25 | 26 | pythonsite: PythonSite = Transform.TOOL() 27 | output: Path = Transform.OUT(init=True, default=...) 28 | 29 | def execute(self, ctx: Context): 30 | output = ctx.map_to_host(self.output) 31 | with output.open(mode="w", encoding="utf-8") as stdout: 32 | inv = self.pythonsite.run(ctx, "-c", "print('hello interface')") 33 | inv.stdout = stdout 34 | yield inv 35 | -------------------------------------------------------------------------------- /example/infra/transforms/templating.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from blockwork.context import Context 18 | from blockwork.transforms import Transform 19 | 20 | from ..tools.misc import PythonSite 21 | 22 | 23 | class MakoTransform(Transform): 24 | pythonsite: PythonSite = Transform.TOOL() 25 | template: Path = Transform.IN() 26 | output: Path = Transform.OUT(init=True, default=...) 27 | 28 | def execute(self, ctx: Context): 29 | cmd = "from mako.template import Template;" 30 | cmd += f"fh = open('{self.output}', 'w');" 31 | cmd += f"fh.write(Template(filename='{self.template}').render());" 32 | cmd += "fh.flush();" 33 | cmd += "fh.close()" 34 | yield self.pythonsite.run(ctx, "-c", cmd) 35 | -------------------------------------------------------------------------------- /blockwork/activities/shell.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | import click 18 | 19 | from ..context import Context 20 | from ..foundation import Foundation 21 | from ..tools import ToolMode 22 | from .common import BwExecCommand 23 | 24 | 25 | @click.command(cls=BwExecCommand) 26 | @click.pass_obj 27 | def shell(ctx: Context, tool: list[str], no_tools: bool, tool_mode: str): 28 | """Launch a shell within the container environment""" 29 | container = Foundation(ctx, hostname=f"{ctx.config.project}_shell") 30 | container.bind(ctx.host_root, ctx.container_root, False) 31 | BwExecCommand.bind_tools(container, no_tools, tool, ToolMode(tool_mode)) 32 | # Launch the shell and forward the exit code 33 | sys.exit(container.shell(workdir=ctx.container_root, show_detach=False)) 34 | -------------------------------------------------------------------------------- /docs/cli/shell.md: -------------------------------------------------------------------------------- 1 | The `shell` command opens a bash shell within the development environment, with 2 | the entire project and all tools mapped in. The environment can be customised 3 | using options to the command. 4 | 5 | It has no sub-commands. 6 | 7 | ## Options 8 | 9 | * `--tool ` / `-t ` - bind a tool into the container - this must use 10 | one of the following forms: 11 | 12 | * `:=` - full syntax that explicitly identifies the 13 | vendor, tool name, and version; 14 | * `:` - identifies vendor and tool name, implicitly selecting 15 | the default version; 16 | * `=` - identifies tool name and version where there is no 17 | vendor specified for the tool; 18 | * `` - identifies only the tool name, where there is no vendor and the 19 | default version is implicitly selected. 20 | 21 | * `--no-tools` - disable automatic binding of all known tools into the container. 22 | 23 | !!!note 24 | 25 | If neither `--tool` or `--no-tools` options are provided, then Blockwork will 26 | automatically bind the default version of all known tools into the container. 27 | If a single `--tool` is specified, then automatic tool binds will be disabled. 28 | 29 | ## Usage Example 30 | 31 | ```bash 32 | $> bw shell --tool GNU:make=4.4 33 | [12:05:26] INFO Binding tool make from N/A version 4.4 into shell 34 | [root@example_shell project]# make -v 35 | GNU Make 4.4 36 | ``` 37 | -------------------------------------------------------------------------------- /blockwork/config/caching.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ..common.checkeddataclasses import dataclass, field 16 | from ..common.yaml import ConverterRegistry, DataclassConverter, Parser 17 | 18 | _registry = ConverterRegistry() 19 | 20 | 21 | @_registry.register(DataclassConverter, tag="!Cache") 22 | @dataclass 23 | class CacheConfig: 24 | name: str 25 | path: str 26 | max_size: str | None = None 27 | fetch_condition: bool | str = False 28 | store_condition: bool | str = False 29 | check_determinism: bool = True 30 | 31 | 32 | @_registry.register(DataclassConverter, tag="!Caching") 33 | @dataclass 34 | class CachingConfig: 35 | enabled: bool = True 36 | targets: bool = False 37 | expect: bool = False 38 | trace: bool = False 39 | caches: list[CacheConfig] = field(default_factory=list) 40 | 41 | 42 | CachingParser = Parser(_registry)(CachingConfig) 43 | -------------------------------------------------------------------------------- /docs/cli/introduction.md: -------------------------------------------------------------------------------- 1 | The command line interface (CLI) is the main way to interact with Blockwork. It 2 | adopts a style similar to `git` with stacked sub-commands and scoped options. 3 | It may be invoked using the `blockwork` command, or by using the short-hand 4 | version of just `bw`. 5 | 6 | ## Top-Level Options 7 | 8 | * `--help` - displays the built-in help, listing the top-level options and the 9 | first level of subcommands; 10 | 11 | * `--cwd ` / `-C ` - by default Blockwork will expect the current 12 | working directory or one of its parents to contain a `.bw.yaml` configuration 13 | file, if commands are being run from outside of this hierarchy then the 14 | working directory can be overridden using this option. 15 | 16 | ## Commands 17 | 18 | * [bootstrap](bootstrap.md) - runs all known bootstrapping stages; 19 | * [exec](exec.md) - executes a command within the contained environment; 20 | * [info](info.md) - displays information about the current project; 21 | * [shell](shell.md) - opens a shell within the contained environment; 22 | * [tool](tool.md) - invoke a specific action of a selected tool; 23 | * [tools](tools.md) - lists all available [tools](../syntax/tools.md). 24 | 25 | ## Usage Example 26 | 27 | ```bash 28 | $> bw --help 29 | Usage: bw [OPTIONS] COMMAND [ARGS]... 30 | 31 | Options: 32 | -C, --cwd DIRECTORY Override the working directory 33 | --help Show this message and exit. 34 | 35 | Commands: 36 | info List information about the project 37 | shell 38 | tools Tabulate all of the available tools 39 | ``` 40 | -------------------------------------------------------------------------------- /example/bench/Makefile: -------------------------------------------------------------------------------- 1 | PARENT_DIRX := $(dir $(lastword $(MAKEFILE_LIST))) 2 | 3 | all: run_cocotb 4 | 5 | clean:: clean_cocotb 6 | 7 | # ============================================================================== 8 | # Icarus Verilog Setup 9 | # ============================================================================== 10 | 11 | export IVERILOG_DUMPER := lxt 12 | 13 | # ============================================================================== 14 | # cocotb 15 | # ============================================================================== 16 | 17 | SIM ?= verilator 18 | TOPLEVEL_LANG ?= verilog 19 | VERILOG_SOURCES += $(wildcard $(PARENT_DIRX)/../design/top/*.sv) /scratch/top.sv 20 | SIM_BUILD ?= /scratch/sim_build 21 | COCOTB_RESULTS_FILE ?= /scratch/results.xml 22 | SIM_WAVES ?= /scratch/waves.lxt 23 | TOPLEVEL ?= top 24 | MODULE ?= bench 25 | # COMPILE_ARGS += -D sim_icarus 26 | PLUSARGS += +WAVE_FILE=$(SIM_WAVES) 27 | 28 | export PYTHONPATH := $(PYTHONPATH):$(PARENT_DIRX) 29 | 30 | include $(shell cocotb-config --makefiles)/Makefile.sim 31 | 32 | /scratch/top.sv: /project/design/top.sv.mako 33 | @python3 -c "from mako.template import Template; \ 34 | fh = open('$@', 'w'); \ 35 | fh.write(Template(filename='$<').render()); \ 36 | fh.flush(); \ 37 | fh.close() " 38 | 39 | .PHONY: run_cocotb $(COCOTB_RESULTS_FILE) 40 | run_cocotb: $(COCOTB_RESULTS_FILE) 41 | 42 | .PHONY: clean_cocotb 43 | clean_cocotb: 44 | rm -rf sim_build $(COCOTB_RESULTS_FILE) 45 | -------------------------------------------------------------------------------- /blockwork/activities/bootstrap.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | import click 18 | 19 | from ..bootstrap import Bootstrap, BwBootstrapMode 20 | from ..context import Context 21 | 22 | 23 | @click.command() 24 | @click.option( 25 | "--mode", 26 | type=click.Choice(BwBootstrapMode, case_sensitive=False), 27 | default="default", 28 | help="""Set the bootstrap mode. 29 | default: Rebuild out of date steps 30 | force: Rebuild all steps 31 | """, 32 | ) 33 | @click.pass_obj 34 | def bootstrap(ctx: Context, mode: str) -> None: 35 | """Run all bootstrapping actions""" 36 | mode: BwBootstrapMode = getattr(BwBootstrapMode, mode) 37 | logging.info(f"Importing {len(ctx.config.bootstrap)} bootstrapping paths") 38 | Bootstrap.setup(ctx.host_root, ctx.config.bootstrap) 39 | logging.info(f"Invoking {len(Bootstrap.get_all())} bootstrap methods") 40 | Bootstrap.evaluate_all(ctx, mode=mode) 41 | logging.info("Bootstrap complete") 42 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Blockwork 2 | repo_name: intuity/blockwork 3 | repo_url: https://github.com/intuity/Blockwork 4 | theme: 5 | name: material 6 | logo: assets/mascot_white.svg 7 | palette: 8 | scheme: slate 9 | primary: indigo 10 | features: 11 | - search.suggest 12 | - search.highlight 13 | plugins: 14 | - search 15 | - mkdocstrings: 16 | handlers: 17 | python: 18 | options: 19 | docstring_style: sphinx 20 | markdown_extensions: 21 | - attr_list 22 | - pymdownx.highlight: 23 | anchor_linenums: true 24 | line_spans: __span 25 | pygments_lang_class: true 26 | - pymdownx.inlinehilite 27 | - pymdownx.snippets 28 | - pymdownx.superfences 29 | - admonition 30 | - pymdownx.details 31 | - pymdownx.tabbed: 32 | alternate_style: true 33 | - pymdownx.emoji: 34 | emoji_index: !!python/name:material.extensions.emoji.twemoji 35 | emoji_generator: !!python/name:material.extensions.emoji.to_svg 36 | nav: 37 | - Welcome: "index.md" 38 | - Foundations: "foundations.md" 39 | - Technologies: 40 | - Containers: "tech/containers.md" 41 | - State: "tech/state.md" 42 | - Caching: "tech/caching.md" 43 | - Configuration: 44 | - ".bw.yaml": "config/bw_yaml.md" 45 | - "caching": "config/caching.md" 46 | - Syntax: 47 | - Bootstrap: "syntax/bootstrap.md" 48 | - Tools: "syntax/tools.md" 49 | - Transforms: "syntax/transforms.md" 50 | - "Command Line Interface": 51 | - Introduction: "cli/introduction.md" 52 | - bootstrap: "cli/bootstrap.md" 53 | - info: "cli/info.md" 54 | - exec: "cli/exec.md" 55 | - shell: "cli/shell.md" 56 | - tool: "cli/tool.md" 57 | - tools: "cli/tools.md" 58 | -------------------------------------------------------------------------------- /blockwork/config/blockwork.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ..common.checkeddataclasses import dataclass, field 16 | from ..common.yaml import DataclassConverter, SimpleParser 17 | 18 | 19 | @dataclass 20 | class Blockwork: 21 | project: str 22 | site: str = "./site.yaml" 23 | root: str = field(default="/project") 24 | scratch: str = field(default="/scratch") 25 | tools: str = field(default="/tools") 26 | host_scratch: str = "../{project}.scratch" 27 | host_state: str = "../{project}.state" 28 | host_tools: str = "../{project}.tools" 29 | hub_url: str | None = None 30 | config: list[str] = field(default_factory=list) 31 | bootstrap: list[str] = field(default_factory=list) 32 | tooldefs: list[str] = field(default_factory=list) 33 | workflows: list[str] = field(default_factory=list) 34 | default_cache_config: str | None = None 35 | 36 | @root.check 37 | @scratch.check 38 | @staticmethod 39 | def abs_path(_field, value): 40 | if not value.startswith("/"): 41 | raise TypeError(f"Expected absolute path, but got {value}") 42 | 43 | 44 | BlockworkParser = SimpleParser(Blockwork, DataclassConverter) 45 | -------------------------------------------------------------------------------- /docs/assets/mascot_white.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /example/infra/config/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from collections.abc import Iterable 16 | 17 | from blockwork.common.checkeddataclasses import field 18 | from blockwork.config import base 19 | from blockwork.transforms import Transform 20 | 21 | from ..transforms.lint import DesignInterface, VerilatorLintTransform 22 | from ..transforms.templating import MakoTransform 23 | 24 | 25 | class Site(base.Site): 26 | pass 27 | 28 | 29 | class Project(base.Project): 30 | pass 31 | 32 | 33 | class Mako(base.Config): 34 | template: str 35 | output: str 36 | 37 | def iter_transforms(self): 38 | yield MakoTransform( 39 | template=self.api.path(self.template), 40 | output=self.api.path(self.output), 41 | ) 42 | 43 | 44 | class Design(base.Config): 45 | top: str 46 | sources: list[str] 47 | transforms: list[Mako] = field(default_factory=list) 48 | 49 | def iter_config(self): 50 | yield from self.transforms 51 | 52 | def iter_transforms(self) -> Iterable[Transform]: 53 | idesign = DesignInterface(sources=list(map(self.api.path, self.sources)), headers=[]) 54 | yield VerilatorLintTransform(design=idesign) 55 | 56 | 57 | class Testbench(base.Config): 58 | design: Design 59 | bench_python: str 60 | bench_make: str 61 | 62 | def iter_config(self): 63 | yield self.design 64 | -------------------------------------------------------------------------------- /example/infra/caches.py: -------------------------------------------------------------------------------- 1 | import os 2 | from collections.abc import Iterable 3 | from pathlib import Path 4 | from shutil import copy, copytree, rmtree 5 | 6 | from blockwork.build.caching import Cache 7 | from blockwork.context import Context 8 | 9 | 10 | class BasicFileCache(Cache): 11 | def __init__(self, ctx: Context) -> None: 12 | self.cache_root = ctx.host_scratch / "cache" 13 | self.content_store = self.cache_root / "store" 14 | self.cache_root.mkdir(exist_ok=True) 15 | self.content_store.mkdir(exist_ok=True) 16 | 17 | def store_item(self, key: str, frm: Path) -> bool: 18 | to = self.content_store / key 19 | if to.exists(): 20 | return True 21 | if frm.is_dir(): 22 | copytree(frm, to) 23 | else: 24 | copy(frm, to) 25 | return True 26 | 27 | def drop_item(self, key: str) -> bool: 28 | path = self.content_store / key 29 | if path.exists(): 30 | if path.is_dir(): 31 | rmtree(path) 32 | else: 33 | path.unlink() 34 | return True 35 | 36 | def fetch_item(self, key: str, to: Path) -> bool: 37 | to.parent.mkdir(exist_ok=True, parents=True) 38 | frm = self.content_store / key 39 | if not frm.exists(): 40 | return False 41 | try: 42 | to.symlink_to(frm, target_is_directory=frm.is_dir()) 43 | except FileNotFoundError: 44 | return False 45 | return True 46 | 47 | def get_last_fetch_utc(self, key: str) -> float: 48 | frm = self.content_store / key 49 | try: 50 | return frm.stat().st_mtime 51 | except OSError: 52 | return 0 53 | 54 | def set_last_fetch_utc(self, key: str): 55 | frm = self.content_store / key 56 | if frm.exists(): 57 | frm.touch(exist_ok=True) 58 | 59 | def iter_keys(self) -> Iterable[str]: 60 | if not self.content_store.exists(): 61 | yield from [] 62 | yield from os.listdir(self.content_store) 63 | -------------------------------------------------------------------------------- /blockwork/activities/exec.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | from pathlib import Path 17 | 18 | import click 19 | 20 | from ..context import Context 21 | from ..foundation import Foundation 22 | from ..tools import ToolMode 23 | from .common import BwExecCommand 24 | 25 | 26 | @click.command(cls=BwExecCommand) 27 | @click.option( 28 | "--interactive", 29 | "-i", 30 | is_flag=True, 31 | default=False, 32 | help="Make the shell interactive (attaches a TTY)", 33 | ) 34 | @click.option( 35 | "--cwd", 36 | type=str, 37 | default=None, 38 | help="Set the working directory within the container", 39 | ) 40 | @click.argument("runargs", nargs=-1, type=click.UNPROCESSED) 41 | @click.pass_obj 42 | def exec( # noqa: A001 43 | ctx: Context, 44 | tool: list[str], 45 | no_tools: bool, 46 | tool_mode: str, 47 | interactive: bool, 48 | cwd: str, 49 | runargs: list[str], 50 | ) -> None: 51 | """Run a command within the container environment""" 52 | container = Foundation(ctx, hostname=f"{ctx.config.project}_run") 53 | container.bind(ctx.host_root, ctx.container_root, False) 54 | BwExecCommand.bind_tools(container, no_tools, tool, ToolMode(tool_mode)) 55 | # Execute and forward the exit code 56 | sys.exit( 57 | container.launch( 58 | *runargs, 59 | workdir=Path(cwd) if cwd else ctx.container_root, 60 | interactive=interactive, 61 | display=True, 62 | show_detach=False, 63 | ).exit_code 64 | ) 65 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["poetry"] 3 | build-backend = "poetry.core.masonry.api" 4 | 5 | [tool.poetry] 6 | name = "blockwork" 7 | version = "1.0" 8 | description = "An opionated EDA flow" 9 | authors = [] 10 | license = "Apache-2.0" 11 | readme = "README.md" 12 | 13 | [tool.poetry.dependencies] 14 | python = "^3.9" 15 | docker = "6.1.2" 16 | click = "8.1.7" 17 | pyyaml = "6.0.1" 18 | typeguard = "4.2.1" 19 | rich = "13.7.1" 20 | poethepoet = "0.20.0" 21 | ordered-set = "4.1.0" 22 | filelock = "3.14.0" 23 | pytz = "2024.1" 24 | requests = "2.31.0" 25 | gator-eda = { git = "https://github.com/Intuity/gator.git", rev = "81670cb0c6d91473a773db54c2b1da782a4fdeb5" } 26 | boto3 = "1.34.103" 27 | humanfriendly = "^10.0" 28 | 29 | [tool.poetry.group.dev.dependencies] 30 | pytest = "7.3.1" 31 | pytest-cov = "4.1.0" 32 | pytest-mock = "3.14.0" 33 | ruff = "0.2.2" 34 | mkdocs = "1.5.3" 35 | mkdocs-material = "9.5.17" 36 | mkdocstrings = { extras = ["python"], version = "^0.26.1" } 37 | pre-commit = "3.7.0" 38 | 39 | [tool.poetry.scripts] 40 | bw = "blockwork:__main__.main" 41 | blockwork = "blockwork:__main__.main" 42 | 43 | [tool.pytest.ini_options] 44 | minversion = "6.0" 45 | addopts = "--no-cov-on-fail --cov=blockwork --cov-branch --cov-report html --cov-report term -x" 46 | testpaths = ["tests"] 47 | 48 | [tool.poe.tasks.test] 49 | shell = "poetry run pytest" 50 | 51 | [tool.poe.tasks.lint] 52 | shell = "poetry run ruff check blockwork" 53 | 54 | [tool.poe.tasks.fmt] 55 | shell = "poetry run ruff format blockwork && poetry run ruff check --fix blockwork" 56 | 57 | [tool.poe.tasks.docs] 58 | shell = "poetry run mkdocs build" 59 | 60 | [tool.poe.tasks.docs_serve] 61 | shell = "poetry run mkdocs serve" 62 | 63 | [tool.ruff] 64 | line-length = 100 65 | indent-width = 4 66 | 67 | # Assume Python 3.11 68 | target-version = "py311" 69 | 70 | [tool.ruff.lint] 71 | select = ["E", "F", "B", "UP", "N", "W", "I", "A", "C4", "PTH", "RUF"] 72 | ignore = [] 73 | fixable = ["ALL"] 74 | unfixable = [] 75 | 76 | [tool.ruff.format] 77 | quote-style = "double" 78 | indent-style = "space" 79 | skip-magic-trailing-comma = false 80 | line-ending = "auto" 81 | docstring-code-format = true 82 | docstring-code-line-length = "dynamic" 83 | -------------------------------------------------------------------------------- /docs/cli/exec.md: -------------------------------------------------------------------------------- 1 | The `exec` command executes a shell command within the contained environment, 2 | binding in as many tools as requested. It can be made to run interactively (i.e. 3 | the TTY forwards both STDIN and STDOUT) if required. 4 | 5 | The exit code of the contained process will be forwarded to the host, so that it 6 | can be used in a script's control flow. 7 | 8 | The `exec` command has no sub-commands. 9 | 10 | ## Options 11 | 12 | * `--tool ` / `-t ` - bind a tool into the container - this must use 13 | one of the following forms: 14 | 15 | * `:=` - full syntax that explicitly identifies the 16 | vendor, tool name, and version; 17 | * `:` - identifies vendor and tool name, implicitly selecting 18 | the default version; 19 | * `=` - identifies tool name and version where there is no 20 | vendor specified for the tool; 21 | * `` - identifies only the tool name, where there is no vendor and the 22 | default version is implicitly selected. 23 | 24 | * `--no-tools` - disable automatic binding of all known tools into the container; 25 | * `--interactive` / `-i` - attaches a TTY onto the shell and interactively 26 | forwards STDIN and STDOUT; 27 | * `--cwd ` - optionally set the working directory, defaults to using the 28 | container project root. 29 | 30 | ## Arguments 31 | 32 | All positional and unrecognised arguments will be taken as being part of the 33 | command to execute. The `--` delimiter may be used to explicitly mark the end of 34 | the arguments to Blockwork and the start of the command to execute. 35 | 36 | ## Usage Example 37 | 38 | With implicit separation between Blockwork arguments and the command: 39 | 40 | ```bash 41 | $> bw exec --tool pythonsite python3 my_script.py 42 | ``` 43 | 44 | However, if the command needs to take any `-` or `--` arguments then explicit 45 | separation must be used. For example: 46 | 47 | ```bash 48 | $ bw exec --tool pythonsite python3 -c "print('hello')" 49 | Usage: bw exec [OPTIONS] [RUNARGS]... 50 | Try 'bw exec --help' for help. 51 | 52 | Error: No such option: -c 53 | ``` 54 | 55 | So instead the `--` delimiter should be used: 56 | 57 | ```bash 58 | $> bw exec --tool pythonsite -- python3 -c "print('hello')" 59 | hello 60 | ``` 61 | -------------------------------------------------------------------------------- /blockwork/activities/info.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | import click 18 | from rich.console import Console 19 | from rich.table import Table 20 | 21 | import blockwork 22 | 23 | from ..context import Context 24 | 25 | 26 | @click.command() 27 | @click.argument("query", nargs=-1, type=str) 28 | @click.pass_obj 29 | def info(ctx: Context, query: list[str]): 30 | """ 31 | List information about the project, an optional list of keys may be provided 32 | to only print select information - for example 'bw info host_tools' will just 33 | show the host tools' root directory path and nothing else. 34 | """ 35 | info = { 36 | "Project": ctx.config.project, 37 | "Configuration File": ctx.config_path.as_posix(), 38 | "Blockwork Install": Path(blockwork.__file__).parent.as_posix(), 39 | "Site": ctx.site.as_posix(), 40 | "Host Root": ctx.host_root.as_posix(), 41 | "Host Tools": ctx.host_tools.as_posix(), 42 | "Host Scratch": ctx.host_scratch.as_posix(), 43 | "Host State": ctx.host_state.as_posix(), 44 | "Container Root": ctx.container_root.as_posix(), 45 | "Container Tools": ctx.container_tools.as_posix(), 46 | "Container Scratch": ctx.container_scratch.as_posix(), 47 | } 48 | if query: 49 | for partial in query: 50 | for name, value in info.items(): 51 | if name.lower().replace(" ", "_").startswith(partial): 52 | print(value) 53 | else: 54 | table = Table(show_header=False) 55 | for name, value in info.items(): 56 | table.add_row(name, value) 57 | Console().print(table) 58 | -------------------------------------------------------------------------------- /example/infra/tools/wave_viewers.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import ClassVar 3 | 4 | from blockwork.context import Context 5 | from blockwork.tools import Invocation, Require, Tool, Version 6 | 7 | from .compilers import GCC, Automake, GPerf 8 | 9 | 10 | @Tool.register() 11 | class GTKWave(Tool): 12 | versions: ClassVar[list[Version]] = [ 13 | Version( 14 | location=Tool.HOST_ROOT / "gtkwave" / "3.3.116", 15 | version="3.3.116", 16 | requires=[ 17 | Require(Automake, "1.16.5"), 18 | Require(GCC, "13.1.0"), 19 | Require(GPerf, "3.1"), 20 | ], 21 | paths={"PATH": [Tool.CNTR_ROOT / "bin"]}, 22 | default=True, 23 | ), 24 | ] 25 | 26 | @Tool.action(default=True) 27 | def view(self, ctx: Context, wavefile: str, *args: list[str]) -> Invocation: 28 | path = Path(wavefile).absolute() 29 | return Invocation( 30 | tool=self, 31 | execute="gtkwave", 32 | args=[path, *args], 33 | display=True, 34 | binds=[path.parent], 35 | ) 36 | 37 | @Tool.action() 38 | def gtk_version(self, ctx: Context, *args: list[str]) -> Invocation: 39 | return Invocation( 40 | tool=self, 41 | execute="gtkwave", 42 | args=["--version", *args], 43 | display=True, 44 | ) 45 | 46 | @Tool.installer() 47 | def install(self, ctx: Context, *args: list[str]) -> Invocation: 48 | vernum = self.vernum 49 | tool_dir = Path("/tools") / self.location.relative_to(Tool.HOST_ROOT) 50 | script = [ 51 | f"wget --quiet https://github.com/gtkwave/gtkwave/archive/refs/tags/v{vernum}.tar.gz", 52 | f"tar -xf v{vernum}.tar.gz", 53 | f"cd gtkwave-{vernum}/gtkwave3-gtk3", 54 | f"./configure --prefix={tool_dir.as_posix()} --enable-gtk3", 55 | "make -j4", 56 | "make install", 57 | "cd ../..", 58 | f"rm -rf gtkwave-{vernum} ./*.tar.*", 59 | ] 60 | return Invocation( 61 | tool=self, 62 | execute="bash", 63 | args=["-c", " && ".join(script)], 64 | workdir=tool_dir, 65 | interactive=True, 66 | ) 67 | -------------------------------------------------------------------------------- /blockwork/containerfiles/foundation/forwarder/blockwork: -------------------------------------------------------------------------------- 1 | #!/bin/env python3 2 | 3 | # ============================================================================== 4 | # This is a simple socket interface that relies only on a basic Python 3 install 5 | # without any additional packages. It forwards the arguments and the working 6 | # directory to the Blockwork host process and waits for a response. Data passing 7 | # over the socket interface is encoded as JSON, with a simple header that carries 8 | # the data length as a 4-byte integer. 9 | # ============================================================================== 10 | 11 | import json 12 | import os 13 | import select 14 | import socket 15 | import sys 16 | 17 | # Determine the location of the blockwork host 18 | host_port = os.environ.get("BLOCKWORK_FWD", None) 19 | if host_port is None: 20 | print("ERROR: The BLOCKWORK_FWD environment variable has not been set") 21 | sys.exit(1) 22 | host, port, *_ = host_port.split(":") 23 | 24 | # Read data from STDIN if it's immediately available 25 | if select.select([sys.stdin], [], [], 0.0)[0]: 26 | stdin = sys.stdin.read() 27 | else: 28 | stdin = "" 29 | 30 | # Connect to forwarding host 31 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 32 | s.connect((host, int(port))) 33 | # Encode request as JSON 34 | encoded = json.dumps({ "args" : sys.argv[1:], 35 | "cwd" : os.getcwd(), 36 | "stdin": stdin }).encode("utf-8") 37 | # Send the total encoded data size 38 | s.sendall(bytearray(((len(encoded) >> (x * 8)) & 0xFF) for x in range(4))) 39 | # Send the encoded data 40 | s.sendall(encoded) 41 | # Receive the response data size 42 | raw_size = s.recv(4) 43 | size = sum([(int(x) << (i * 8)) for i, x in enumerate(raw_size)]) 44 | # Receive the response data 45 | raw_data = s.recv(size) 46 | # Decode JSON 47 | try: 48 | data = json.loads(raw_data) 49 | except json.JSONDecodeError: 50 | print("Error occurred while decoding response") 51 | sys.exit(255) 52 | # Log the response STDOUT and STDERR 53 | if (stdout := data.get("stdout", None)) is not None: 54 | sys.stdout.write(stdout) 55 | if (stderr := data.get("stderr", None)) is not None: 56 | sys.stderr.write(stderr) 57 | # Exit with the right code 58 | sys.exit(data.get("exitcode", 0)) 59 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [ "main" ] 7 | 8 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 9 | permissions: 10 | contents: read 11 | pages: write 12 | id-token: write 13 | 14 | # Allow only one concurrent deployment to GitHub pages, but don't cancel running 15 | # builds and wait for completion 16 | concurrency: 17 | group: "pages" 18 | cancel-in-progress: false 19 | 20 | jobs: 21 | lint: 22 | runs-on: ubuntu-latest 23 | timeout-minutes: 15 24 | steps: 25 | - uses: actions/checkout@v4 26 | - uses: ./.github/composites/setup 27 | with: 28 | python-version: 3.11 29 | - name: Run Ruff formatting check 30 | run: poetry run ruff format --check . 31 | - name: Run Ruff linting check 32 | run: poetry run ruff check . 33 | - name: Run precommit checks 34 | env: 35 | SKIP: no-commit-to-branch 36 | run: poetry run pre-commit run --all-files 37 | 38 | build_docs: 39 | runs-on: ubuntu-latest 40 | timeout-minutes: 15 41 | steps: 42 | - uses: actions/checkout@v4 43 | - uses: ./.github/composites/setup 44 | with: 45 | python-version: 3.11 46 | - name: Build documentation 47 | run: | 48 | poe docs 49 | - name: Upload documentation artifact 50 | uses: actions/upload-pages-artifact@v3 51 | with: 52 | path: ./site 53 | 54 | deploy_docs: 55 | runs-on: ubuntu-latest 56 | if: github.event_name != 'pull_request' 57 | timeout-minutes: 15 58 | needs: build_docs 59 | steps: 60 | - name: Deploy to GitHub pages 61 | id: deployment 62 | uses: actions/deploy-pages@v4 63 | 64 | unit_tests: 65 | runs-on: ubuntu-22.04 66 | timeout-minutes: 15 67 | strategy: 68 | fail-fast: false 69 | matrix: 70 | # TODO: Run against other versions "3.8", "3.9", "3.10" 71 | python-version: ["3.11"] 72 | 73 | steps: 74 | - uses: actions/checkout@v4 75 | - uses: ./.github/composites/setup 76 | with: 77 | python-version: ${{ matrix.python-version }} 78 | - name: Run tests 79 | timeout-minutes: 5 80 | run: poe test 81 | env: 82 | TMPDIR: ${{ runner.temp }} 83 | - name: Archive code coverage results 84 | uses: actions/upload-artifact@v4 85 | with: 86 | name: pytest-cov 87 | path: htmlcov 88 | -------------------------------------------------------------------------------- /blockwork/common/complexnamespaces.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Generic, TypeVar 16 | 17 | _NamespaceValue = TypeVar("_NamespaceValue") 18 | 19 | 20 | class ComplexNamespace(Generic[_NamespaceValue]): 21 | """ 22 | Typed version of types.SimpleNamespace with read and modify only options. 23 | """ 24 | 25 | def __init__( 26 | self, 27 | *, 28 | _RO: bool = False, # noqa: N803 29 | _MO: bool = False, # noqa: N803 30 | **kwargs: _NamespaceValue, 31 | ): 32 | self.__dict__["RO"] = _RO 33 | self.__dict__["MO"] = _MO 34 | self.__dict__["ns"] = kwargs 35 | 36 | def __getattr__(self, name) -> _NamespaceValue: 37 | return self.ns[name] 38 | 39 | def __getitem__(self, name) -> _NamespaceValue: 40 | return self.ns[name] 41 | 42 | def __setattr__(self, name: str, value: _NamespaceValue) -> None: 43 | if self.RO: 44 | raise RuntimeError("Namespace is read only") 45 | if self.MO and name not in self.ns: 46 | raise RuntimeError(f"Namespace is modify only and {name} is not already present") 47 | self.ns[name] = value 48 | 49 | def __repr__(self): 50 | return f"namespace({', '.join(f'{k}={v}' for k,v in self.ns.items())})" 51 | 52 | def keys(self): 53 | yield from self.ns.keys() 54 | 55 | def values(self): 56 | yield from self.ns.values() 57 | 58 | def items(self): 59 | yield from self.ns.items() 60 | 61 | 62 | class ReadonlyNamespace(ComplexNamespace[_NamespaceValue]): 63 | """ 64 | Namespace where attributes cannot be added or modified post initialisation 65 | """ 66 | 67 | def __init__(self, **kwargs: _NamespaceValue): 68 | self.__dict__["RO"] = True 69 | self.__dict__["MO"] = False 70 | self.__dict__["ns"] = kwargs 71 | -------------------------------------------------------------------------------- /docs/assets/foundation/anchors-spec-to-gds.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 |
Specification
Specification
GDS
GDSText is not SVG - cannot display 4 | -------------------------------------------------------------------------------- /example/infra/workflows.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from collections.abc import Iterable 16 | 17 | import click 18 | 19 | from blockwork.config.base import Config 20 | from blockwork.transforms import Transform 21 | from blockwork.workflows.workflow import Workflow 22 | 23 | from .transforms.examples import CapturedTransform 24 | from .transforms.lint import VerilatorLintTransform 25 | 26 | 27 | class Build(Config): 28 | target: Config 29 | match: str | None 30 | 31 | @Workflow("build").with_target() 32 | @click.option("--match", type=str, default=None) 33 | @staticmethod 34 | def from_command(ctx, project, target, match): 35 | return Build(target=target, match=match) 36 | 37 | def iter_config(self) -> Iterable[Config]: 38 | yield self.target 39 | 40 | def transform_filter(self, transform: Transform, config: Config) -> bool: 41 | return self.match is None or self.match.lower() in transform.__class__.__name__.lower() 42 | 43 | 44 | class Test(Config): 45 | tests: list[Config] 46 | 47 | @Workflow("test").with_target() 48 | @staticmethod 49 | def from_command(ctx, project, target): 50 | tests = [Build(target=target, match="mako")] 51 | return Test(tests=tests) 52 | 53 | def iter_config(self) -> Iterable[Config]: 54 | yield from self.tests 55 | 56 | def config_filter(self, config: Config): 57 | return config in self.tests 58 | 59 | def iter_transforms(self) -> Iterable[Transform]: 60 | yield CapturedTransform(output=self.api.path("./captured_stdout")) 61 | 62 | 63 | class Lint(Config): 64 | target: Config 65 | 66 | @Workflow("lint").with_target() 67 | @staticmethod 68 | def from_command(ctx, project, target): 69 | return Lint(target=target) 70 | 71 | def transform_filter(self, transform: Transform, config: Config) -> bool: 72 | return isinstance(transform, VerilatorLintTransform) 73 | 74 | def iter_config(self) -> Iterable[Config]: 75 | yield self.target 76 | -------------------------------------------------------------------------------- /blockwork/common/singleton.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from collections.abc import Callable, Hashable 16 | from typing import Any, ClassVar, Self, cast 17 | 18 | # NOTE: Credit to Andriy Ivaneyko for the singleton pattern used below 19 | # https://stackoverflow.com/questions/51896862/how-to-create-singleton-class-with-arguments-in-python 20 | 21 | 22 | class Singleton(type): 23 | INSTANCES: ClassVar[dict[type, Any]] = {} 24 | 25 | def __call__(cls, *args, **kwds) -> Any: 26 | if cls not in Singleton.INSTANCES: 27 | Singleton.INSTANCES[cls] = super().__call__(*args, **kwds) 28 | return Singleton.INSTANCES[cls] 29 | 30 | 31 | class ParameterisedSingleton(type): 32 | INSTANCES: ClassVar[dict[tuple[type, str | int], Any]] = {} 33 | 34 | def __call__(cls, *args, **kwds) -> Any: 35 | uniq_key = (cls, *args, *(f"{k}={v}" for k, v in kwds.items())) 36 | if uniq_key not in ParameterisedSingleton.INSTANCES: 37 | ParameterisedSingleton.INSTANCES[uniq_key] = super().__call__(*args, **kwds) 38 | return ParameterisedSingleton.INSTANCES[uniq_key] 39 | 40 | 41 | def keyed_singleton( 42 | arg_key: Callable[..., Hashable] | None = None, 43 | inst_key: Callable[[Any], Hashable] | None = None, 44 | ): 45 | "Factory for singletons which use a key function to uniquify/share" 46 | 47 | if (arg_key is None) == (inst_key is None): 48 | raise RuntimeError("Must specify arg_key or inst_key (but not both)") 49 | 50 | class FactoriedSingleton(type): 51 | INSTANCES: ClassVar[dict[str, Self]] = {} 52 | 53 | def __call__(cls, *args, **kwds): 54 | if inst_key: 55 | inst = super().__call__(*args, **kwds) 56 | k = inst_key(inst) 57 | else: 58 | inst = None 59 | k = cast(Callable, arg_key)(*args, **kwds) 60 | if k not in cls.INSTANCES: 61 | cls.INSTANCES[k] = inst or cls(*args, **kwds) 62 | return cls.INSTANCES[k] 63 | 64 | return FactoriedSingleton 65 | -------------------------------------------------------------------------------- /blockwork/activities/workflow.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | import logging 17 | from pathlib import Path 18 | 19 | import click 20 | 21 | from ..build.caching import BWFrozenHash, Cache 22 | from ..context import Context 23 | from ..transforms.transform import SerialTransform, Transform 24 | 25 | 26 | @click.group(name="wf") 27 | def wf() -> None: 28 | """ 29 | Workflow argument group. 30 | 31 | In the future we may want to add common options such as --dryrun here 32 | """ 33 | pass 34 | 35 | 36 | @click.command(name="_wf_step", hidden=True) 37 | @click.argument("spec_path", type=click.Path(dir_okay=False, exists=True, path_type=Path)) 38 | @click.argument("input_hash", type=click.STRING) 39 | @click.option("--target", is_flag=True, default=False) 40 | @click.pass_obj 41 | def wf_step(ctx: Context, spec_path: Path, input_hash: str, target: bool): 42 | """ 43 | Loads a serialised transform specification from a provided file path, then 44 | resolves the transform class and executes it. This should NOT be called 45 | directly but instead as part of a wider workflow. 46 | """ 47 | # TODO @intuity: We should consider making wf_step part of non-parallel 48 | # executions so that there is a single execution path 49 | # Reload the serialised workflow step specification 50 | spec: SerialTransform = json.loads(spec_path.read_text(encoding="utf-8")) 51 | # Load the relevant transform 52 | transform = Transform.deserialize(spec, BWFrozenHash(spec["name"], bytes.fromhex(input_hash))) 53 | 54 | is_caching = Cache.enabled(ctx) 55 | 56 | if ( 57 | is_caching 58 | and (ctx.cache_targets or not target) 59 | and Cache.fetch_transform_from_any(ctx, transform) 60 | ): 61 | logging.info("Fetched transform from cache: %s (late)", transform) 62 | else: 63 | logging.info("Running transform: %s", transform) 64 | result = transform.run(ctx) 65 | 66 | # Whether a cache is in place 67 | if is_caching and Cache.store_transform_to_any(ctx, transform, result.run_time): 68 | logging.info("Stored transform to cache: %s", transform) 69 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | /build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | poetry.lock 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .nox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | *.py,cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # pipenv 89 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 90 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 91 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 92 | # install all needed dependencies. 93 | #Pipfile.lock 94 | 95 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 96 | __pypackages__/ 97 | 98 | # Celery stuff 99 | celerybeat-schedule 100 | celerybeat.pid 101 | 102 | # SageMath parsed files 103 | *.sage.py 104 | 105 | # Environments 106 | .env 107 | .venv 108 | env/ 109 | venv/ 110 | ENV/ 111 | env.bak/ 112 | venv.bak/ 113 | 114 | # Spyder project settings 115 | .spyderproject 116 | .spyproject 117 | 118 | # Rope project settings 119 | .ropeproject 120 | 121 | # mkdocs documentation 122 | /site 123 | 124 | # mypy 125 | .mypy_cache/ 126 | .dmypy.json 127 | dmypy.json 128 | 129 | # Pyre type checker 130 | .pyre/ 131 | 132 | # macOS 133 | .DS_Store 134 | 135 | # VSCode 136 | .vscode 137 | 138 | # Test directories 139 | tracking 140 | 141 | # Ruff 142 | .ruff_cache 143 | 144 | # Drawio 145 | *.drawio.bkp 146 | *.drawio.dtmp 147 | 148 | # Blockwork 149 | *.scratch/ 150 | *.state/ 151 | *.tools/ 152 | -------------------------------------------------------------------------------- /blockwork/common/inithooks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from inspect import getmembers_static 15 | 16 | 17 | class InitHooks: 18 | """ 19 | Provides methods for registering pre and post init hooks that 20 | will run on subclasses even if the subclasses don't call 21 | `super().__init__(...)`. For example:: 22 | 23 | @InitHooks() 24 | class MyNumberClass: 25 | @InitHooks.pre 26 | def init_numbers(): 27 | ... 28 | """ 29 | 30 | PRE_ATTR = "_init_hooks_pre_init" 31 | POST_ATTR = "_init_hooks_post_init" 32 | 33 | def __call__(self, cls_): 34 | return InitHooks._wrap_cls(cls_) 35 | 36 | @staticmethod 37 | def _wrap_cls(cls_): 38 | # Find the pre and post hooks 39 | pre_hooks = [] 40 | post_hooks = [] 41 | for _name, value in getmembers_static(cls_): 42 | if hasattr(value, InitHooks.PRE_ATTR): 43 | pre_hooks.append(value) 44 | if hasattr(value, InitHooks.POST_ATTR): 45 | post_hooks.append(value) 46 | 47 | # Replace the init_subclass method with one that wraps 48 | # the subclasses init method. 49 | orig_init_subclass = cls_.__init_subclass__ 50 | 51 | def _custom_init_subclass(subcls_, *args, **kwargs): 52 | orig_init_subclass(*args, **kwargs) 53 | InitHooks._wrap_subcls(subcls_, pre_hooks, post_hooks) 54 | 55 | cls_.__init_subclass__ = classmethod(_custom_init_subclass) 56 | # Finally wrap the cls itself 57 | return InitHooks._wrap_subcls(cls_, pre_hooks, post_hooks) 58 | 59 | @staticmethod 60 | def _wrap_subcls(subcls_, pre_hooks, post_hooks): 61 | # Wrap the subclasses init method with one that 62 | # runs our hooks. 63 | orig_init = subcls_.__init__ 64 | 65 | def _custom_init(self, *args, **kwargs): 66 | if self.__class__ is subcls_: 67 | for hook in pre_hooks: 68 | hook(self) 69 | orig_init(self, *args, **kwargs) 70 | if self.__class__ is subcls_: 71 | for hook in post_hooks: 72 | hook(self) 73 | 74 | subcls_.__init__ = _custom_init 75 | return subcls_ 76 | 77 | @staticmethod 78 | def pre(fn): 79 | "Run this function before `__init__` (class must be decorated with `InitHooks`)" 80 | setattr(fn, InitHooks.PRE_ATTR, True) 81 | return fn 82 | 83 | @staticmethod 84 | def post(fn): 85 | "Run this function after `__init__` (class must be decorated with `InitHooks`)" 86 | setattr(fn, InitHooks.POST_ATTR, True) 87 | return fn 88 | -------------------------------------------------------------------------------- /docs/config/bw_yaml.md: -------------------------------------------------------------------------------- 1 | A Blockwork workspace is configured through a `.bw.yaml` file located in the root 2 | folder of the project. It identifies the project, locates tool definitions, and 3 | defines various other behaviours. 4 | 5 | The file must use a `!Blockwork` tag as its root element, as per the example below: 6 | 7 | ```yaml linenums="1" 8 | !Blockwork 9 | project : example 10 | root : /project 11 | scratch : /scratch 12 | tools : /tools 13 | host_scratch: ../{project}.scratch 14 | host_state : ../{project}.state 15 | host_tools : ../{project}.tools 16 | bootstrap : 17 | - infra.bootstrap.setup 18 | tooldefs : 19 | - infra.tools.linters 20 | - infra.tools.compilers 21 | ``` 22 | 23 | The fields of the `!Blockwork` tag are: 24 | 25 | | Field | Required | Default | Description | 26 | |----------------------|:----------------:|------------------------|----------------------------------------------------------------------------| 27 | | project | :material-check: | | Sets the project's name | 28 | | root | | `/project` | Location to map the project's root directory inside the container | 29 | | scratch | | `/scratch` | Location to map the scratch area inside the container | 30 | | tools | | `/tools` | Location to map the tools inside the container | 31 | | host_scratch | | `../{project}.scratch` | Directory to store build objects and other artefacts | 32 | | host_state | | `../{project}.state` | Directory to store Blockwork's state information for the project | 33 | | host_tools | | `../{project}.tools` | Directory containing tool installations on the host | 34 | | default_cache_config | | | Path to the default cache configuration, see [Caching](../tech/caching.md) | 35 | | bootstrap | | | Python paths containing [Bootstrap](../syntax/bootstrap.md) definitions | 36 | | tooldefs | | | Python paths containing [Tool](../syntax/tools.md) definitions | 37 | 38 | !!!note 39 | 40 | The `host_scratch`, `host_state`, and `host_tools` directories are resolved 41 | relative to the project's root directory on the host, and the `{project}` 42 | keyword will be substituted for the projects name (taken from the `project` 43 | field). 44 | 45 | ## Variable Substitutions 46 | 47 | Some configuration fields support variable substitution into values, these are 48 | summarised in the table below: 49 | 50 | | Variable | Supported By | Description | 51 | |--------------|-------------------|----------------------------------------------------------------| 52 | | `{project}` | `root`, `scratch` | Name of the project (from the `project` field) | 53 | | `{root_dir}` | `root`, `scratch` | Name of the directory that's an immediate parent to `.bw.yaml` | 54 | -------------------------------------------------------------------------------- /blockwork/common/scopes.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from collections.abc import Callable 16 | from typing import ClassVar, Generic, ParamSpec, Self, TypeVar 17 | 18 | 19 | class ScopeError(RuntimeError): 20 | "Attempted to access scoped data outside of any scope" 21 | 22 | 23 | _ScopedData = TypeVar("_ScopedData") 24 | 25 | 26 | class _ScopeWrap(Generic[_ScopedData]): 27 | _stack: ClassVar[list[_ScopedData]] = [] 28 | 29 | def __init__(self, data: _ScopedData): 30 | self._data = data 31 | 32 | def __enter__(self): 33 | self._stack.append(self._data) 34 | return self._data 35 | 36 | def __exit__(self, exc_type, exc_val, exc_tb): 37 | self._stack.pop() 38 | 39 | @classmethod 40 | @property 41 | def current(cls) -> _ScopedData: 42 | try: 43 | return cls._stack[-1] 44 | except IndexError as e: 45 | raise ScopeError from e 46 | 47 | 48 | class Scope: 49 | """ 50 | Mixin class to provide scoping via a context manager 51 | stack for data that may otherwise end up global. See example:: 52 | 53 | @dataclass 54 | class Verbosity(Scope): 55 | VERBOSE: bool 56 | 57 | def do_something(): 58 | if Verbosity.current().VERBOSE: 59 | ... 60 | else 61 | ... 62 | 63 | with Verbosity(VERBOSE=True): 64 | do_something() 65 | """ 66 | 67 | _stack: ClassVar[list[Self]] = [] 68 | 69 | def __enter__(self): 70 | self._stack.append(self) 71 | return self 72 | 73 | def __exit__(self, exc_type, exc_val, exc_tb): 74 | self._stack.pop() 75 | 76 | @classmethod 77 | @property 78 | def current(cls) -> Self: 79 | try: 80 | return cls._stack[-1] 81 | except IndexError as e: 82 | raise ScopeError from e 83 | 84 | 85 | _Param = ParamSpec("_Param") 86 | _Return = TypeVar("_Return") 87 | 88 | 89 | def scope(wrapee: Callable[_Param, _Return]) -> Callable[_Param, _ScopeWrap[_Return]]: 90 | """ 91 | Decorator intended to provide scoping via a context manager 92 | stack for data that may otherwise end up global. See example:: 93 | 94 | @scope 95 | @dataclass 96 | class Verbosity: 97 | VERBOSE: bool 98 | 99 | def do_something(): 100 | if Verbosity.current().VERBOSE: 101 | ... 102 | else 103 | ... 104 | 105 | with Verbosity(VERBOSE=True): 106 | do_something() 107 | """ 108 | 109 | class Scoped(_ScopeWrap[wrapee]): 110 | def __init__(self, *args: _Param.args, **kwargs: _Param.kwargs): 111 | self._data = wrapee(*args, **kwargs) 112 | 113 | return Scoped 114 | -------------------------------------------------------------------------------- /example/infra/tools/simulators.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import ClassVar 3 | 4 | from blockwork.context import Context 5 | from blockwork.tools import Invocation, Require, Tool, Version 6 | 7 | from .compilers import GCC, Autoconf, Bison, CCache, Flex, GPerf, Help2Man 8 | 9 | 10 | @Tool.register() 11 | class IVerilog(Tool): 12 | versions: ClassVar[list[Version]] = [ 13 | Version( 14 | location=Tool.HOST_ROOT / "iverilog" / "12.0", 15 | version="12.0", 16 | requires=[ 17 | Require(Autoconf, "2.71"), 18 | Require(Bison, "3.8"), 19 | Require(GCC, "13.1.0"), 20 | Require(Flex, "2.6.4"), 21 | Require(GPerf, "3.1"), 22 | ], 23 | paths={"PATH": [Tool.CNTR_ROOT / "bin"]}, 24 | default=True, 25 | ), 26 | ] 27 | 28 | @Tool.installer() 29 | def install(self, ctx: Context, *args: list[str]) -> Invocation: 30 | vernum = self.vernum.replace(".", "_") 31 | tool_dir = Path("/tools") / self.location.relative_to(Tool.HOST_ROOT) 32 | script = [ 33 | f"wget --quiet https://github.com/steveicarus/iverilog/archive/refs/tags/v{vernum}.tar.gz", 34 | f"tar -xf v{vernum}.tar.gz", 35 | f"cd iverilog-{vernum}", 36 | "autoconf", 37 | f"./configure --prefix={tool_dir.as_posix()}", 38 | "make -j4", 39 | "make install", 40 | "cd ..", 41 | f"rm -rf iverilog-{vernum} ./*.tar.*", 42 | ] 43 | return Invocation( 44 | tool=self, 45 | execute="bash", 46 | args=["-c", " && ".join(script)], 47 | workdir=tool_dir, 48 | ) 49 | 50 | 51 | @Tool.register() 52 | class Verilator(Tool): 53 | versions: ClassVar[list[Version]] = [ 54 | Version( 55 | location=Tool.HOST_ROOT / "verilator" / "5.014", 56 | version="5.014", 57 | env={ 58 | "VERILATOR_BIN": "../../bin/verilator_bin", 59 | "VERILATOR_ROOT": Tool.CNTR_ROOT / "share" / "verilator", 60 | }, 61 | paths={"PATH": [Tool.CNTR_ROOT / "bin"]}, 62 | requires=[ 63 | Require(Autoconf, "2.71"), 64 | Require(Bison, "3.8"), 65 | Require(CCache, "4.8.2"), 66 | Require(GCC, "13.1.0"), 67 | Require(Flex, "2.6.4"), 68 | Require(Help2Man, "1.49.3"), 69 | ], 70 | default=True, 71 | ), 72 | ] 73 | 74 | @Tool.action() 75 | def run(self, ctx: Context, *args: list[str]) -> Invocation: 76 | return Invocation(version=self, execute="verilator", args=args) 77 | 78 | @Tool.installer() 79 | def install(self, ctx: Context, *args: list[str]) -> Invocation: 80 | vernum = self.vernum 81 | tool_dir = Path("/tools") / self.location.relative_to(Tool.HOST_ROOT) 82 | script = [ 83 | f"wget --quiet https://github.com/verilator/verilator/archive/refs/tags/v{vernum}.tar.gz", 84 | f"tar -xf v{vernum}.tar.gz", 85 | f"cd verilator-{vernum}", 86 | "autoconf", 87 | f"./configure --prefix={tool_dir.as_posix()}", 88 | "make -j4", 89 | "make install", 90 | f"rm -rf verilator-{vernum} ./*.tar.*", 91 | ] 92 | return Invocation( 93 | tool=self, 94 | execute="bash", 95 | args=["-c", " && ".join(script)], 96 | workdir=tool_dir, 97 | ) 98 | -------------------------------------------------------------------------------- /tests/test_scheduler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Any 16 | 17 | import pytest 18 | 19 | from blockwork.config.scheduler import CyclicError, Scheduler 20 | 21 | 22 | class StateRecord: 23 | def __init__(self): 24 | self.states = [] 25 | self.keys = { 26 | "leaves", 27 | "schedulable", 28 | "blocked", 29 | "unscheduled", 30 | "scheduled", 31 | "incomplete", 32 | "complete", 33 | } 34 | 35 | def record(self, scheduler): 36 | self.states.append({key: getattr(scheduler, key) for key in self.keys}) 37 | 38 | def __getattr__(self, __name: str) -> Any: 39 | if __name in self.keys: 40 | return tuple(state.get(__name) for state in self.states) 41 | return super().__getattribute__(__name) 42 | 43 | 44 | class TestScheduler: 45 | def test_basic(self): 46 | "Tests a basic chain" 47 | dependency_map = {"y": {"x"}, "z": {"y"}} 48 | scheduler = Scheduler(dependency_map) 49 | 50 | states = StateRecord() 51 | 52 | while scheduler.incomplete: 53 | states.record(scheduler) 54 | for item in scheduler.schedulable: 55 | scheduler.schedule(item) 56 | scheduler.finish(item) 57 | 58 | assert states.schedulable == ({"x"}, {"y"}, {"z"}) 59 | assert states.blocked == ({"y", "z"}, {"z"}, set()) 60 | assert states.incomplete == ({"x", "y", "z"}, {"y", "z"}, {"z"}) 61 | assert states.complete == (set(), {"x"}, {"x", "y"}) 62 | 63 | def test_cycle(self): 64 | "Tests that cycles are detected" 65 | dependency_map = {"y": {"x"}, "z": {"y"}, "x": {"z"}} 66 | scheduler = Scheduler(dependency_map) 67 | 68 | with pytest.raises(CyclicError): 69 | _ = scheduler.schedulable 70 | 71 | def test_complex(self) -> None: 72 | "Tests a more complex tree is scheduled correctly" 73 | dependency_map = { 74 | "b": {"a"}, 75 | "c": {"b"}, 76 | "d": {"b"}, 77 | "e": {"c"}, 78 | "f": {"d", "e", "g"}, 79 | } 80 | scheduler = Scheduler(dependency_map) 81 | 82 | states = StateRecord() 83 | while scheduler.incomplete: 84 | states.record(scheduler) 85 | for item in scheduler.schedulable: 86 | scheduler.schedule(item) 87 | scheduler.finish(item) 88 | 89 | assert states.schedulable == ({"a", "g"}, {"b"}, {"c", "d"}, {"e"}, {"f"}) 90 | 91 | # Do it again, but this time with a specific target 92 | scheduler = Scheduler(dependency_map, targets=["e"]) 93 | states = StateRecord() 94 | while scheduler.incomplete: 95 | states.record(scheduler) 96 | for item in scheduler.schedulable: 97 | scheduler.schedule(item) 98 | scheduler.finish(item) 99 | assert states.schedulable == ({"a"}, {"b"}, {"c"}, {"e"}) 100 | -------------------------------------------------------------------------------- /docs/config/caching.md: -------------------------------------------------------------------------------- 1 | Caching is configured through `.yaml` files. The `.bw.yaml` 2 | (see [bw_yaml](../config/bw_yaml.md)) defines the default caching configuration 3 | but this can be overriden on the command line with the `--cache-config` option. 4 | 5 | Each caching configuration must use a `!Caching` tag as its root element, as 6 | per the example below: 7 | 8 | ```yaml linenums="1" 9 | !Caching 10 | enabled: True 11 | targets: False 12 | trace: False 13 | caches: 14 | - !Cache 15 | name: local-cache 16 | path: infra.caches.Local 17 | fetch_condition: True 18 | store_condition: True 19 | max_size: 5GB 20 | - !Cache 21 | name: remote-cache 22 | path: infra.caches.FileStore 23 | fetch_condition: 10 MB/s 24 | store_condition: False 25 | 26 | ``` 27 | 28 | The fields of the `!Caching` tag are: 29 | 30 | | Field | Default | Description | 31 | |----------------------|-------- |--------------------------------------------------------------------| 32 | | enabled | `True` | Whether to enable caching by default (overridable on command line) | 33 | | targets | `False` | Whether to pull targetted1 transforms from the cache | 34 | | trace | `False` | Whether to enable (computationally intensive) debug tracing. | 35 | | caches | `[]` | A list of `!Cache` configurations (see below) | 36 | 37 | 38 | 1 Targetted transforms are those selected to be run by a workflow, 39 | in contrast to those which are only dependencies of targetted transforms. 40 | 41 | 42 | The fields of the `!Cache` tag are: 43 | 44 | | Field | Required | Default | Description | 45 | |-------------------|:----------------:|-------- |-----------------------------------------------------------------| 46 | | name | :material-check: | | The name of the cache (used in logging etc) | 47 | | path | :material-check: | | The python import path to the cache implementation2 | 48 | | max_size | | `None` | The maximum cache size, which the cache will self-prune down to | 49 | | store_condition | | `False` | The condition for storing to the cache3 | 50 | | fetch_condition | | `False` | The condition for fetching from the cache3 | 51 | | check_determinism | | `True` | Whether to check object determinism4 | 52 | 53 | 2 Specified as `..` 54 | 55 | 3 Specified as: 56 | 57 | - `True`: Always store-to or fetch-from this cache 58 | - `False`: Never store-to or fetch-from this cache 59 | - `B/s`: Only store-to or fetch-from this cache if the transforms byte-rate 60 | is below the provided value. This is useful when a cache is networked, and it 61 | may be more efficient to just re-compute quick-to-run, high-output transforms 62 | than pull them down. Some example values are: 63 | - `1B/s`: > 1 second to create each byte. 64 | - `1GB/h`: > 1 hour to create each Gigabyte. 65 | - `5MB/4m` > 4 minutes to create each 5 Megabytes. 66 | 67 | Note: The rate-specification may be removed in the future, in favour of a 68 | dynamic scheme. 69 | 70 | 4 When enabled, if a transform hash exists in the cache and the 71 | transform is re-run, check that both produced the same output hashes. It is 72 | recommended this is left on, but it may be desirable to turn this off if 73 | cache lookups are expensive for a particular cache. Note, this will result 74 | in fetches of the key-data even when fetch_condition is `False`. 75 | -------------------------------------------------------------------------------- /docs/tech/state.md: -------------------------------------------------------------------------------- 1 | Some operations related to the project may be stateful - for example any setup 2 | performed by [bootstrapping operations](../syntax/bootstrap.md) only needs to be 3 | executed on a fresh checkout or whenever the setup process is changed. Blockwork 4 | offers a mechanism to persist variables that can then be retrieved in later 5 | invocations. 6 | 7 | Persisted variables are separated into different namespaces, and different tools 8 | can register their own namespaces if required. For example, the 9 | [bootstrap](../syntax/bootstrap.md) state is preserved into a namespace called 10 | 'bootstrap'. 11 | 12 | Each namespace is serialised to JSON, and reloaded lazily - avoiding unnecessary 13 | delays when invoking the Blockwork command as only the state which is required 14 | will be read from disk. Serialised data will be stored into the directory set by 15 | [`state_dir` in the !Blockwork configuration](../config/bw_yaml.md). 16 | 17 | For most usecases, state will be access via the context object that is passed 18 | into different routines. For example, the [bootstrap](../syntax/bootstrap.md) 19 | stage shown in the example below accesses the state dictionary via the `context` 20 | argument: 21 | 22 | ```python linenums="1" title="infra/bootstrap/tool_a.py" 23 | import logging 24 | import os 25 | import pwd 26 | from datetime import datetime 27 | from pathlib import Path 28 | 29 | from blockwork.bootstrap import Bootstrap 30 | from blockwork.context import Context 31 | 32 | @Bootstrap.register() 33 | def setup_tool_a(context : Context, last_run : datetime) -> bool: 34 | # Log any previous install 35 | if (prev_url := context.state.tool_a.url) is not None: 36 | logging.info(f"Previous version of tool A was installed from {prev_url}") 37 | # URL to download the tool from 38 | url = "http://example.com/tool_a.zip" 39 | # ...some actions to download the tool and install it... 40 | # Remember some details about the install 41 | context.state.tool_a.url = url 42 | context.state.tool_a.install_date = datetime.now().timestamp() 43 | context.state.tool_a.installed_by = pwd.getpwuid(os.getuid()).pw_name 44 | # See bootstrapping section to explain the return value 45 | return False 46 | ``` 47 | 48 | The instance of the `State` class is accessed via `context.state` - this is the 49 | correct way to manage state in most cases (as opposed to manually creating an 50 | instance of `State`). Different state namespaces are accessed by using the `.` 51 | operator - for example `context.state.tool_a` opens a namespace called `tool_a`, 52 | automatically creating it if it doesn't already exist. Namespaces can alternatively 53 | be accessed using the `get` method - for example `context.state.get("tool_a")`. 54 | 55 | Each namespace is an instance of the `StateNamespace` class, and variables can 56 | be similiarly set and retrieved using the `.` operator or the `set` and `get` 57 | methods - examples of both methods are shown below: 58 | 59 | ```python title="Using the '.' operator" 60 | # Reading a value 61 | existing = context.state.my_tool.some_var 62 | if existing is not None: 63 | print(f"An existing value was already set: {existing}") 64 | # Writing a value 65 | context.state.my_tool.some_var = 123 66 | ``` 67 | 68 | ```python title="Using the 'set' and 'get' methods" 69 | # Reading a value 70 | existing = context.state.my_tool.get("some_var") 71 | if existing is not None: 72 | print(f"An existing value was already set: {existing}") 73 | # Writing a value 74 | context.state.my_tool.some_var.set("123") 75 | ``` 76 | 77 | !!!warning 78 | 79 | State variables may only have primitive types such as string, integer, float, 80 | and boolean - using any other type will raise an exception. Namespaces are 81 | also shallow, so do not support deep variable hierarchies (i.e. only 82 | `context.state.my_tool.some_var` is supported and not 83 | `context.state.my_tool.lvl_one.lvl_two.some_var`). 84 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Blockwork](docs/assets/mascot_b_black_e_white.png) 2 | 3 | **NOTE** Blockwork is currently in active development and is not yet suitable 4 | for production environments. It is missing many features and does not yet fulfill 5 | its stated aims. 6 | 7 | # Getting Started 8 | 9 | ## Recommended Pre-requisites for macOS 10 | 11 | On macOS we recommend the following: 12 | 13 | * [XQuartz](https://www.xquartz.org) - to support X11 forwarding from applications 14 | running in the contained environment. 15 | * [Docker](http://docker.com) or [Orbstack](https://orbstack.dev) as the container 16 | runtime. [Podman](https://podman-desktop.io) is supported but it exhibits poor 17 | filesystem performance. 18 | * Python 3.11 installed through [pyenv](https://github.com/pyenv/pyenv) to 19 | protect your OS's default install from contamination. 20 | * [Poetry](https://python-poetry.org) installed through Python's package manager 21 | i.e. `python -m pip install poetry`. 22 | 23 | ## Recommended Pre-requisites for Linux 24 | 25 | On Linux we recommend the following: 26 | 27 | * [Docker](http://docker.com) as the container runtime. [Podman](https://podman-desktop.io) 28 | is supported but it exhibits poor filesystem performance (there are some notes 29 | to improve this in the [troubleshooting section](#troubleshooting)). 30 | * Python 3.11 installed through [pyenv](https://github.com/pyenv/pyenv) to 31 | protect your OS's default install from contamination. 32 | * [Poetry](https://python-poetry.org) installed through Python's package manager 33 | i.e. `python -m pip install poetry`. 34 | 35 | ## Installing Blockwork 36 | 37 | To install the bleeding edge version of Blockwork, use the following command: 38 | 39 | ```bash 40 | $> python3 -m pip install git+https://github.com/blockwork-eda/blockwork 41 | ``` 42 | 43 | ## Setting up a Development Environment 44 | 45 | Follow these steps to get a development environment: 46 | 47 | ```bash 48 | # Clone the repository 49 | $> git clone git@github.com:blockwork-eda/blockwork.git 50 | $> cd blockwork 51 | # Activate a poetry shell 52 | $> poetry shell 53 | # Install all dependencies (including those just for development) 54 | $> poetry install --with=dev 55 | # Bootstrap the example project 56 | $> bw -C example bootstrap 57 | # Run a test command 58 | $> bw -C example exec -- echo "hi" 59 | ``` 60 | 61 | # Troubleshooting 62 | 63 | ## macOS 64 | 65 | ### X11 Forwarding 66 | 67 | * Ensure [XQuartz](https://www.xquartz.org) is installed 68 | * Tick "Allow connections from network clients" in XQuartz preferences 69 | * Quit and re-open XQuartz 70 | * Execute `xhost +` 71 | 72 | 73 | **NOTE** The `DISPLAY` environment variable must be set to `host.internal:0` for 74 | Docker/Orbstack or `host.containers.internal:0` for Podman, this should be setup 75 | automatically by the framework. 76 | 77 | ## Linux 78 | 79 | ### Podman Socket 80 | 81 | To start the user-space socket service execute: 82 | 83 | ```bash 84 | $> systemctl --user status podman.socket 85 | ``` 86 | 87 | **NOTE** Do not use `sudo` as the service needs to run in user-space. 88 | 89 | ### Slow Podman Performance 90 | 91 | Ensure that you are using the overlay filesystem (`fuse-overlayfs`), as the 92 | default `vfs` is very slow! 93 | 94 | ```bash 95 | $> sudo apt install -y fuse-overlayfs 96 | $> podman system reset 97 | $> podman info --debug | grep graphDriverName 98 | ``` 99 | 100 | If the `graphDriverName` is not reported as `overlay`, then you can try forcing 101 | it by editing `~/.config/containers/storage.conf` to contain: 102 | 103 | ```toml 104 | [storage] 105 | driver = "overlay" 106 | ``` 107 | 108 | Then execute `podman system reset` again, and perform the same check for the 109 | graph driver. 110 | 111 | After changing the filesystem driver, you will need to rebuild the foundation 112 | container as it is deleted by the reset command. 113 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | ## Getting Started 2 | 3 | Blockwork is compatible with Python version 3.8 through 3.11 and is developed using 4 | [the Poetry packaging tool](https://python-poetry.org). To install Blockwork, check 5 | out the repository from GitHub and install it with Poetry: 6 | 7 | ### Recommended Pre-requisites for macOS 8 | 9 | On macOS we recommend the following: 10 | 11 | * [XQuartz](https://www.xquartz.org) - to support X11 forwarding from applications 12 | running in the contained environment. 13 | * [Docker](http://docker.com) or [Orbstack](https://orbstack.dev) as the container 14 | runtime. [Podman](https://podman-desktop.io) is supported but it exhibits poor 15 | filesystem performance. 16 | * Python 3.11 installed through [pyenv](https://github.com/pyenv/pyenv) to 17 | protect your OS's default install from contamination. 18 | * [Poetry](https://python-poetry.org) installed through Python's package manager 19 | i.e. `python -m pip install poetry`. 20 | 21 | ### Recommended Pre-requisites for Linux 22 | 23 | On Linux we recommend the following: 24 | 25 | * [Docker](http://docker.com) as the container runtime. [Podman](https://podman-desktop.io) 26 | is supported but it exhibits poor filesystem performance (there are some notes 27 | to improve this in the [troubleshooting section](#troubleshooting)). 28 | * Python 3.11 installed through [pyenv](https://github.com/pyenv/pyenv) to 29 | protect your OS's default install from contamination. 30 | * [Poetry](https://python-poetry.org) installed through Python's package manager 31 | i.e. `python -m pip install poetry`. 32 | 33 | ### Installing Blockwork 34 | 35 | To install the bleeding edge version of Blockwork, use the following command: 36 | 37 | ```bash 38 | $> python3 -m pip install git+https://github.com/blockwork-eda/blockwork 39 | ``` 40 | 41 | ### Setting up a Development Environment 42 | 43 | Follow these steps to get a development environment: 44 | 45 | ```bash 46 | # Clone the repository 47 | $> git clone git@github.com:blockwork-eda/blockwork.git 48 | $> cd blockwork 49 | # Activate a poetry shell 50 | $> poetry shell 51 | # Install all dependencies (including those just for development) 52 | $> poetry install --with=dev 53 | # Bootstrap the example project 54 | $> bw -C example bootstrap 55 | # Run a test command 56 | $> bw -C example exec -- echo "hi" 57 | ``` 58 | 59 | ## Troubleshooting 60 | 61 | ### macOS 62 | 63 | #### X11 Forwarding 64 | 65 | * Ensure [XQuartz](https://www.xquartz.org) is installed 66 | * Tick "Allow connections from network clients" in XQuartz preferences 67 | * Quit and re-open XQuartz 68 | * Execute `xhost +` 69 | 70 | 71 | **NOTE** The `DISPLAY` environment variable must be set to `host.internal:0` for 72 | Docker/Orbstack or `host.containers.internal:0` for Podman, this should be setup 73 | automatically by the framework. 74 | 75 | ### Linux 76 | 77 | #### Podman Socket 78 | 79 | To start the user-space socket service execute: 80 | 81 | ```bash 82 | $> systemctl --user status podman.socket 83 | ``` 84 | 85 | **NOTE** Do not use `sudo` as the service needs to run in user-space. 86 | 87 | #### Slow Podman Performance 88 | 89 | Ensure that you are using the overlay filesystem (`fuse-overlayfs`), as the 90 | default `vfs` is very slow! 91 | 92 | ```bash 93 | $> sudo apt install -y fuse-overlayfs 94 | $> podman system reset 95 | $> podman info --debug | grep graphDriverName 96 | ``` 97 | 98 | If the `graphDriverName` is not reported as `overlay`, then you can try forcing 99 | it by editing `~/.config/containers/storage.conf` to contain: 100 | 101 | ```toml 102 | [storage] 103 | driver = "overlay" 104 | ``` 105 | 106 | Then execute `podman system reset` again, and perform the same check for the 107 | graph driver. 108 | 109 | After changing the filesystem driver, you will need to rebuild the foundation 110 | container as it is deleted by the reset command. 111 | -------------------------------------------------------------------------------- /example/infra/tools/misc.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import ClassVar 3 | 4 | from blockwork.context import Context 5 | from blockwork.tools import Invocation, Require, Tool, Version 6 | 7 | from .compilers import GCC 8 | 9 | 10 | @Tool.register() 11 | class Python(Tool): 12 | versions: ClassVar[list[Version]] = [ 13 | Version( 14 | location=Tool.HOST_ROOT / "python" / "3.11.4", 15 | version="3.11.4", 16 | requires=[Require(GCC, "13.1.0")], 17 | paths={ 18 | "PATH": [Tool.CNTR_ROOT / "bin"], 19 | "LD_LIBRARY_PATH": [Tool.CNTR_ROOT / "lib"], 20 | }, 21 | default=True, 22 | ), 23 | ] 24 | 25 | @Tool.installer() 26 | def install(self, ctx: Context, *args: list[str]) -> Invocation: 27 | vernum = self.vernum 28 | tool_dir = Path("/tools") / self.location.relative_to(Tool.HOST_ROOT) 29 | script = [ 30 | f"wget --quiet https://www.python.org/ftp/python/{vernum}/Python-{vernum}.tgz", 31 | f"tar -xf Python-{vernum}.tgz", 32 | f"cd Python-{vernum}", 33 | f"./configure --enable-optimizations --with-ensurepip=install " 34 | f"--enable-shared --prefix={tool_dir.as_posix()}", 35 | "make -j4", 36 | "make install", 37 | "cd ..", 38 | f"rm -rf Python-{vernum} ./*.tgz*", 39 | ] 40 | return Invocation( 41 | tool=self, 42 | execute="bash", 43 | args=["-c", " && ".join(script)], 44 | workdir=tool_dir, 45 | ) 46 | 47 | 48 | @Tool.register() 49 | class PythonSite(Tool): 50 | versions: ClassVar[list[Version]] = [ 51 | Version( 52 | location=Tool.HOST_ROOT / "python-site" / "3.11.4", 53 | version="3.11.4", 54 | env={"PYTHONUSERBASE": Tool.CNTR_ROOT}, 55 | paths={ 56 | "PATH": [Tool.CNTR_ROOT / "bin"], 57 | "PYTHONPATH": [Tool.CNTR_ROOT / "lib" / "python3.11" / "site-packages"], 58 | }, 59 | requires=[Require(Python, "3.11.4")], 60 | default=True, 61 | ), 62 | ] 63 | 64 | @Tool.action() 65 | def run(self, ctx: Context, *args: list[str]) -> Invocation: 66 | return Invocation(tool=self, execute="python3", args=args) 67 | 68 | @Tool.installer() 69 | def install(self, ctx: Context, *args: list[str]) -> Invocation: 70 | return Invocation( 71 | tool=self, 72 | execute="python3", 73 | args=[ 74 | "-m", 75 | "pip", 76 | "--no-cache-dir", 77 | "install", 78 | "-r", 79 | ctx.container_root / "infra" / "tools" / "pythonsite.txt", 80 | ], 81 | interactive=True, 82 | ) 83 | 84 | 85 | @Tool.register() 86 | class Make(Tool): 87 | versions: ClassVar[list[Version]] = [ 88 | Version( 89 | location=Tool.HOST_ROOT / "make" / "4.4.1", 90 | version="4.4.1", 91 | paths={"PATH": [Tool.CNTR_ROOT / "bin"]}, 92 | default=True, 93 | ), 94 | ] 95 | 96 | @Tool.action(default=True) 97 | def run(self, ctx: Context, *args: list[str]) -> Invocation: 98 | return Invocation(tool=self, execute="make", args=args) 99 | 100 | @Tool.installer() 101 | def install(self, ctx: Context, *args: list[str]) -> Invocation: 102 | vernum = self.vernum 103 | tool_dir = Path("/tools") / self.location.relative_to(Tool.HOST_ROOT) 104 | script = [ 105 | f"wget --quiet https://ftp.gnu.org/gnu/make/make-{vernum}.tar.gz", 106 | f"tar -xf make-{vernum}.tar.gz", 107 | f"cd make-{vernum}", 108 | f"./configure --prefix={tool_dir.as_posix()}", 109 | "make -j4", 110 | "make install", 111 | "cd ..", 112 | f"rm -rf make-{vernum} ./*.tar.*", 113 | ] 114 | return Invocation( 115 | tool=self, 116 | execute="bash", 117 | args=["-c", " && ".join(script)], 118 | workdir=tool_dir, 119 | ) 120 | -------------------------------------------------------------------------------- /blockwork/bootstrap/tools.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import inspect 16 | import logging 17 | from datetime import datetime 18 | from pathlib import Path 19 | 20 | from ordered_set import OrderedSet as OSet 21 | 22 | from ..context import Context 23 | from ..foundation import Foundation 24 | from ..tools import Tool, ToolError 25 | from ..tools.tool import ToolActionError 26 | from .bootstrap import Bootstrap 27 | 28 | 29 | @Bootstrap.register() 30 | def install_tools(context: Context, last_run: datetime) -> bool: 31 | """ 32 | Run the install action for all known tools 33 | """ 34 | # Get instances of all of the tools and install all specified versions 35 | all_tools = OSet([]) 36 | for tool in Tool.get_all().values(): 37 | for version in tool.versions: 38 | all_tools.add(version) 39 | 40 | # Order by requirements 41 | resolved = [] 42 | last_len = len(all_tools) 43 | logging.debug(f"Ordering {len(all_tools)} tools based on requirements:") 44 | while all_tools: 45 | for tool in all_tools: 46 | if len(set(tool.resolve_requirements()).difference(resolved)) == 0: 47 | logging.debug(f" - {len(resolved)}: {' '.join(tool.id_tuple)}") 48 | resolved.append(tool) 49 | all_tools = all_tools - set(resolved) 50 | if len(all_tools) == last_len: 51 | raise ToolError("Deadlock detected resolving tool requirements") 52 | last_len = len(all_tools) 53 | 54 | # Install in order 55 | logging.info(f"Installing {len(resolved)} tools:") 56 | for idx, tool in enumerate(resolved): 57 | tool_id = " ".join(tool.id_tuple) 58 | tool_file = Path(inspect.getfile(type(tool.tool))) 59 | host_loc = tool.tool.get_host_path(context, absolute=False) 60 | # Ensure the parent of the tool's folder exists 61 | host_loc.parent.mkdir(exist_ok=True, parents=True) 62 | # Select a touch file location, this is used to determine if the tool 63 | # installation is up to date 64 | touch_file = context.host_state / "tools" / tool.tool.name / tool.version / Tool.TOUCH_FILE 65 | touch_file.parent.mkdir(exist_ok=True, parents=True) 66 | # If the touch file exists and install has been run more recently than 67 | # the definition file was updated, then skip 68 | if touch_file.exists(): 69 | tch_date = datetime.fromtimestamp(touch_file.stat().st_mtime) 70 | def_date = datetime.fromtimestamp(tool_file.stat().st_mtime) 71 | if tch_date >= def_date: 72 | logging.debug(f" - {idx}: Tool {tool_id} is already installed") 73 | continue 74 | # Attempt to install 75 | try: 76 | act_def = tool.get_action("installer") 77 | except ToolActionError: 78 | logging.debug(f" - {idx}: Tool {tool_id} does not define an install action") 79 | else: 80 | logging.info(f" - {idx}: Launching installation of {tool_id}") 81 | invk = act_def(context) 82 | if invk is not None: 83 | container = Foundation( 84 | context, hostname=f"{context.config.project}_install_{tool.id}" 85 | ) 86 | exit_code = container.invoke(context, act_def(context), readonly=False).exit_code 87 | if exit_code != 0: 88 | raise ToolError(f"Installation of {tool_id} failed") 89 | else: 90 | logging.debug(f" - {idx}: Installation of {tool_id} produced a null invocation") 91 | logging.debug(f" - {idx}: Installation of {tool_id} succeeded") 92 | # Touch the install folder to ensure its datetime is updated 93 | try: 94 | touch_file.touch() 95 | except PermissionError as e: 96 | logging.debug(f" - Could not update modified time of {touch_file}: {e}") 97 | pass 98 | -------------------------------------------------------------------------------- /blockwork/activities/tools.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | from collections.abc import Sequence 17 | 18 | import click 19 | from rich.console import Console 20 | from rich.table import Table 21 | 22 | from ..context import Context 23 | from ..foundation import Foundation 24 | from ..tools import Tool 25 | from ..tools.tool import ToolActionError 26 | from .common import BwExecCommand, ToolMode 27 | 28 | 29 | @click.command() 30 | @click.pass_obj 31 | def tools(ctx: Context): 32 | """ 33 | Tabulate all of the available tools including vendor name, tool name, version, 34 | which version is default, and a list of supported actions. The default action 35 | will be marked with an asterisk ('*'). 36 | """ 37 | table = Table() 38 | table.add_column("Vendor") 39 | table.add_column("Tool") 40 | table.add_column("Version") 41 | table.add_column("Default", justify="center") 42 | table.add_column("Actions") 43 | for tool_def in Tool.get_all().values(): 44 | tool = tool_def() 45 | t_acts = Tool.ACTIONS.get(tool.name, {}) 46 | actions = [(x, y) for x, y in t_acts.items() if x != "default"] 47 | default = t_acts.get("default", None) 48 | act_str = ", ".join(f"{x}{'*' if y is default else ''}" for x, y in actions) 49 | for idx, version in enumerate(tool): 50 | table.add_row( 51 | tool.vendor if idx == 0 else "", 52 | tool.name if idx == 0 else "", 53 | version.version, 54 | ["", ":heavy_check_mark:"][version.default], 55 | act_str if idx == 0 else "", 56 | ) 57 | Console().print(table) 58 | 59 | 60 | @click.command() 61 | @click.option("--version", "-v", type=str, default=None, help="Set the tool version to use.") 62 | @click.option( 63 | "--tool-mode", 64 | type=click.Choice(ToolMode, case_sensitive=False), 65 | default="readonly", 66 | help="Set the file mode used when binding tools " 67 | "to enable write access. Legal values are " 68 | "either 'readonly' or 'readwrite', defaults " 69 | "to 'readonly'.", 70 | ) 71 | @click.argument("tool_action", type=str) 72 | @click.argument("runargs", nargs=-1, type=click.UNPROCESSED) 73 | @click.pass_obj 74 | def tool( 75 | ctx: Context, 76 | version: str | None, 77 | tool_action: str, 78 | tool_mode: str, 79 | runargs: Sequence[str], 80 | ) -> None: 81 | """ 82 | Run an action defined by a specific tool. The tool and action is selected by 83 | the first argument either using the form . or just 84 | where the default action is acceptable. 85 | """ 86 | # Split . or into parts 87 | base_tool, action, *_ = (tool_action + ".default").split(".") 88 | # Find the tool 89 | tool = f"{base_tool}={version}" if version else base_tool 90 | vendor, name, version = BwExecCommand.decode_tool(tool) 91 | if (tool_ver := Tool.get(vendor, name, version)) is None: 92 | raise Exception(f"Cannot locate tool for {tool}") 93 | # See if there is an action registered 94 | try: 95 | act_def = tool_ver.get_action(action) 96 | except ToolActionError: 97 | raise Exception(f"No action known for '{action}' on tool {tool}") from None 98 | # Run the action and forward the exit code 99 | container = Foundation(ctx, hostname=f"{ctx.config.project}_{tool}_{action}") 100 | runargs = container.bind_and_map_args(ctx, runargs) 101 | invocation = act_def(ctx, *runargs) 102 | # Actions may sometimes return null invocations if they have no work to do 103 | if invocation is None: 104 | return 105 | # Launch the invocation 106 | sys.exit( 107 | container.invoke( 108 | ctx, invocation, readonly=(ToolMode(tool_mode) == ToolMode.READONLY) 109 | ).exit_code 110 | ) 111 | -------------------------------------------------------------------------------- /blockwork/activities/common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | import click 18 | from click.core import Command, Option 19 | 20 | from ..foundation import Foundation 21 | from ..tools import Tool, ToolMode 22 | 23 | 24 | class BwExecCommand(Command): 25 | """Standard argument handling for commands that launch a container""" 26 | 27 | def __init__(self, *args, **kwargs): 28 | super().__init__(*args, **kwargs) 29 | self.params.insert( 30 | 0, 31 | Option( 32 | ("--tool", "-t"), 33 | type=str, 34 | multiple=True, 35 | default=[], 36 | help="Bind specific tools into the shell, if " 37 | "omitted then all known tools will be " 38 | "bound. Either use the form " 39 | "'--tool ' or '--tool =' " 40 | "where a specific version other than the " 41 | "default is desired. To specify a vendor use " 42 | "the form '--tool :(=)'.", 43 | ), 44 | ) 45 | self.params.insert( 46 | 0, 47 | Option( 48 | ("--no-tools",), 49 | is_flag=True, 50 | default=False, 51 | help="Do not bind any tools by default", 52 | ), 53 | ) 54 | self.params.insert( 55 | 0, 56 | Option( 57 | ("--tool-mode",), 58 | type=click.Choice(ToolMode, case_sensitive=False), 59 | default="readonly", 60 | help="Set the file mode used when binding tools " 61 | "to enable write access. Legal values are " 62 | "either 'readonly' or 'readwrite', defaults " 63 | "to 'readonly'.", 64 | ), 65 | ) 66 | self.params.insert( 67 | 0, 68 | Option( 69 | ("--no-tools",), 70 | is_flag=True, 71 | default=False, 72 | help="Do not bind any tools by default", 73 | ), 74 | ) 75 | 76 | @staticmethod 77 | def decode_tool(fullname: str) -> tuple[str, str, str | None]: 78 | """ 79 | Decode a tool vendor, name, and version from a string - in one of the 80 | forms :=, =, :, or 81 | just . Where a vendor is not provided, NO_VENDOR is assumed. Where 82 | no version is provided None is returned to select the default variant 83 | 84 | :param fullname: Encoded tool using one of the forms described. 85 | :returns: Tuple of vendor, name, and version 86 | """ 87 | fullname, version, *_ = (fullname + "=").split("=") 88 | vendor, name = (Tool.NO_VENDOR + ":" + fullname).split(":")[-2:] 89 | return vendor, name, (version or None) 90 | 91 | @staticmethod 92 | def bind_tools( 93 | container: Foundation, no_tools: bool, tools: list[str], tool_mode: ToolMode 94 | ) -> None: 95 | readonly = tool_mode == ToolMode.READONLY 96 | specified_tools = set() 97 | 98 | # If tools are provided, process them for default version overrides 99 | for vendor, name, version in map(BwExecCommand.decode_tool, tools): 100 | matched: Tool = Tool.get(vendor, name, version or None) 101 | if not matched: 102 | raise Exception(f"Failed to identify tool '{vendor}:{name}={version}'") 103 | logging.info(f"Binding tool {name} from {vendor} version {version} into shell") 104 | container.add_tool(matched, readonly=readonly) 105 | specified_tools.add(matched.base_id) 106 | 107 | # If auto-binding allowed bind default versions of remaining tools 108 | if not no_tools: 109 | logging.info("Binding all tools into shell") 110 | for tool in Tool.get_all().values(): 111 | if tool.base_id not in specified_tools: 112 | container.add_tool(tool, readonly=readonly) 113 | -------------------------------------------------------------------------------- /blockwork/common/registry.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import importlib 16 | import sys 17 | from collections import defaultdict 18 | from collections.abc import Callable 19 | from contextlib import contextmanager 20 | from pathlib import Path 21 | from typing import Any, ClassVar, TypeVar 22 | 23 | _RegObj = TypeVar("_RegObj") 24 | 25 | 26 | class RegistryError(Exception): 27 | pass 28 | 29 | 30 | class Registry: 31 | LOOKUP_BY_NAME: ClassVar[dict[type, dict[str, "RegisteredMethod"]]] = defaultdict(lambda: {}) 32 | LOOKUP_BY_OBJ: ClassVar[dict[type, dict[Callable, "RegisteredMethod"]]] = defaultdict( 33 | lambda: {} 34 | ) 35 | 36 | @staticmethod 37 | def setup(root: Path, paths: list[str]) -> None: 38 | """ 39 | Import Python modules that register objects of this type from a list of 40 | module paths that are either system wide or relative to a given root path. 41 | 42 | :param root: Root path under which Python modules are defined, this is 43 | added to the PYTHONPATH prior to discovery 44 | :param paths: Python module names to import from 45 | """ 46 | if root.absolute().as_posix() not in sys.path: 47 | sys.path.append(root.absolute().as_posix()) 48 | for path in paths: 49 | importlib.import_module(path) 50 | 51 | @classmethod 52 | def wrap(cls, obj: Any) -> Any: 53 | del obj 54 | raise NotImplementedError( 55 | "The 'wrap' method must be implemented by an " "inheriting registry type" 56 | ) 57 | 58 | @classmethod 59 | def register(cls, *_args, **_kwds) -> Callable[[_RegObj], _RegObj]: 60 | def _inner(obj: _RegObj) -> _RegObj: 61 | cls.wrap(obj) 62 | return obj 63 | 64 | return _inner 65 | 66 | @classmethod 67 | def get_all(cls) -> dict[str, "RegisteredMethod"]: 68 | return RegisteredMethod.LOOKUP_BY_NAME[cls] 69 | 70 | @classmethod 71 | def get_by_name(cls, name: str) -> "RegisteredMethod": 72 | base = RegisteredMethod.LOOKUP_BY_NAME[cls] 73 | if name not in base: 74 | raise RegistryError(f"Unknown {cls.__name__.lower()} for '{name}'") 75 | return base[name] 76 | 77 | @classmethod 78 | @contextmanager 79 | def temp_registry(cls): 80 | """Context managed temporary registry for use in tests""" 81 | lookup_by_name = RegisteredMethod.LOOKUP_BY_NAME[cls] 82 | lookup_by_obj = RegisteredMethod.LOOKUP_BY_OBJ[cls] 83 | RegisteredMethod.LOOKUP_BY_NAME[cls] = defaultdict(lambda: {}) 84 | RegisteredMethod.LOOKUP_BY_OBJ[cls] = defaultdict(lambda: {}) 85 | try: 86 | yield None 87 | finally: 88 | RegisteredMethod.LOOKUP_BY_NAME[cls] = lookup_by_name 89 | RegisteredMethod.LOOKUP_BY_OBJ[cls] = lookup_by_obj 90 | 91 | @classmethod 92 | def clear_registry(cls) -> None: 93 | """Clear all existing registrations for this registry""" 94 | RegisteredMethod.LOOKUP_BY_NAME[cls] = {} 95 | RegisteredMethod.LOOKUP_BY_OBJ[cls] = {} 96 | 97 | 98 | class RegisteredMethod(Registry): 99 | """ 100 | Provides registry behaviours for an object type. A decorator is provided 101 | `@RegisteredMethod.register()` to associate an object with the registry. 102 | """ 103 | 104 | @classmethod 105 | def wrap(cls, obj: Callable) -> Callable: 106 | if obj in Registry.LOOKUP_BY_OBJ[cls]: 107 | return Registry.LOOKUP_BY_OBJ[cls][obj] 108 | else: 109 | wrp = cls(obj) 110 | Registry.LOOKUP_BY_NAME[cls][obj.__name__] = wrp 111 | Registry.LOOKUP_BY_OBJ[cls][obj] = wrp 112 | return wrp 113 | 114 | 115 | class RegisteredClass(Registry): 116 | """ 117 | Provides registry behaviours for an object type. A decorator is provided 118 | `@RegisteredClass.register()` to associate an object with the registry. 119 | """ 120 | 121 | @classmethod 122 | def wrap(cls, obj: type) -> type: 123 | if obj in RegisteredClass.LOOKUP_BY_OBJ[cls]: 124 | return obj 125 | else: 126 | RegisteredClass.LOOKUP_BY_NAME[cls][obj.__name__] = obj 127 | RegisteredClass.LOOKUP_BY_OBJ[cls][obj] = obj 128 | return obj 129 | -------------------------------------------------------------------------------- /blockwork/bootstrap/bootstrap.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | from collections.abc import Callable 17 | from datetime import datetime 18 | from enum import StrEnum, auto 19 | from pathlib import Path 20 | 21 | from ..common.registry import RegisteredMethod 22 | from ..context import Context 23 | 24 | 25 | class BwBootstrapMode(StrEnum): 26 | default = auto() 27 | "Default behaviour" 28 | force = auto() 29 | "Rerun steps even when they are in-date" 30 | 31 | 32 | class Bootstrap(RegisteredMethod): 33 | """ 34 | Defines a single bootstrapping step to perform when setting up the workspace 35 | """ 36 | 37 | def __init__(self, method: Callable) -> None: 38 | self.method = method 39 | self.full_path = method.__module__ + "." + method.__qualname__ 40 | self.checkpoints: list[Path] = [] 41 | 42 | def add_checkpoint(self, path: Path) -> None: 43 | self.checkpoints.append(path) 44 | 45 | @property 46 | def id(self) -> str: 47 | return self.full_path.replace(".", "__") 48 | 49 | def __call__(self, context: Context, mode: BwBootstrapMode = BwBootstrapMode.default) -> None: 50 | """ 51 | Wrap the call to the bootstrapping method with handling to track when 52 | the step was last run, and whether it needs to be re-run based on its 53 | checkpoint paths alone. 54 | 55 | :param context: The context object 56 | :param mode: Modifier for when build steps should be considered invalid 57 | """ 58 | # When 'forcing' bootstrap to re-run, set the datestamp to the earliest ever date 59 | if mode == BwBootstrapMode.force: 60 | last_run = datetime.min 61 | # Otherwise, attempt to read the date back from the context 62 | else: 63 | raw = context.state.bootstrap.get(self.id, 0) 64 | last_run = datetime.fromisoformat(raw) if raw else datetime.min 65 | # Evaluate checkpoints 66 | if self.checkpoints: 67 | expired = False 68 | for chk in self.checkpoints: 69 | chk_path = context.host_root / chk 70 | if ( 71 | chk_path.exists() 72 | and datetime.fromtimestamp(chk_path.stat().st_mtime) <= last_run 73 | ): 74 | logging.debug( 75 | f"Bootstrap step '{self.full_path}' checkpoint " 76 | f"'{chk_path}' is up-to-date" 77 | ) 78 | else: 79 | logging.debug( 80 | f"Bootstrap step '{self.full_path}' checkpoint " 81 | f"'{chk_path}' has been updated" 82 | ) 83 | expired = True 84 | if not expired: 85 | logging.info( 86 | f"Bootstrap step '{self.full_path}' is already up " 87 | f"to date (based on checkpoints)" 88 | ) 89 | return 90 | # Run the bootstrapping function 91 | logging.debug(f"Evaluating bootstrap step '{self.full_path}'") 92 | if self.method(context=context, last_run=last_run) is True: 93 | logging.info( 94 | f"Bootstrap step '{self.full_path}' is already up " f"to date (based on method)" 95 | ) 96 | else: 97 | logging.info(f"Ran bootstrap step '{self.full_path}'") 98 | context.state.bootstrap.set(self.id, datetime.now().isoformat()) 99 | 100 | @classmethod 101 | def checkpoint(cls, path: Path) -> Callable: 102 | def _inner(func: Callable) -> Callable: 103 | boot = cls.wrap(func) 104 | boot.add_checkpoint(path) 105 | return func 106 | 107 | return _inner 108 | 109 | @classmethod 110 | def evaluate_all( 111 | cls, context: Context, mode: BwBootstrapMode = BwBootstrapMode.default 112 | ) -> None: 113 | """ 114 | Evaluate all of the registered bootstrap methods, checking to see whether 115 | they are out-of-date based on their 'check_point' before executing them. 116 | 117 | :param context: The context object of the current session 118 | :param mode: Modifier for when build steps should be considered invalid 119 | """ 120 | for step in cls.get_all().values(): 121 | step(context, mode) 122 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | from blockwork.common.yaml import ( 18 | DataclassConverter, 19 | SimpleParser, 20 | YamlFieldError, 21 | YamlMissingFieldsError, 22 | ) 23 | from blockwork.config import Blockwork 24 | 25 | BlockworkConfig = SimpleParser(Blockwork, DataclassConverter) 26 | 27 | 28 | class TestConfig: 29 | def test_config(self) -> None: 30 | """Custom project configuration""" 31 | cfg = BlockworkConfig.parse_str( 32 | "!Blockwork\n" 33 | "project: test_project\n" 34 | "root: /my_root\n" 35 | "scratch: /my_scratch\n" 36 | "host_state: ../my_{project}_state\n" 37 | "host_scratch: ../my_{project}_scratch\n" 38 | "bootstrap:\n" 39 | " - infra.bootstrap.step_a\n" 40 | " - infra.bootstrap.step_b\n" 41 | "tooldefs:\n" 42 | " - infra.tools.set_a\n" 43 | " - infra.tools.set_b\n" 44 | ) 45 | assert isinstance(cfg, Blockwork) 46 | assert cfg.project == "test_project" 47 | assert cfg.root == "/my_root" 48 | assert cfg.scratch == "/my_scratch" 49 | assert cfg.host_state == "../my_{project}_state" 50 | assert cfg.host_scratch == "../my_{project}_scratch" 51 | assert cfg.bootstrap == ["infra.bootstrap.step_a", "infra.bootstrap.step_b"] 52 | assert cfg.tooldefs == ["infra.tools.set_a", "infra.tools.set_b"] 53 | 54 | def test_config_default(self) -> None: 55 | """Simple project configuration using mostly default values""" 56 | cfg = BlockworkConfig.parse_str("!Blockwork\n" "project: test_project\n") 57 | assert isinstance(cfg, Blockwork) 58 | assert cfg.project == "test_project" 59 | assert cfg.root == "/project" 60 | assert cfg.scratch == "/scratch" 61 | assert cfg.host_state == "../{project}.state" 62 | assert cfg.host_scratch == "../{project}.scratch" 63 | assert cfg.bootstrap == [] 64 | assert cfg.tooldefs == [] 65 | 66 | def test_config_error(self) -> None: 67 | """Different syntax errors""" 68 | # Missing project name 69 | with pytest.raises(YamlMissingFieldsError) as exc: 70 | BlockworkConfig.parse_str("!Blockwork\n" "tooldefs: [a, b, c]\n") 71 | assert "project" in exc.value.fields 72 | # Bad root directory (integer) 73 | with pytest.raises(YamlFieldError) as exc: 74 | BlockworkConfig.parse_str("!Blockwork\n" "project: test\n" "root: 123\n") 75 | assert exc.value.field == "root" 76 | assert isinstance(exc.value.orig_ex, TypeError) 77 | # Bad root directory (relative path) 78 | with pytest.raises(YamlFieldError) as exc: 79 | BlockworkConfig.parse_str("!Blockwork\n" "project: test\n" "root: a/b\n") 80 | assert exc.value.field == "root" 81 | # Bad scratch directory (integer) 82 | with pytest.raises(YamlFieldError) as exc: 83 | BlockworkConfig.parse_str("!Blockwork\n" "project: test\n" "scratch: 123\n") 84 | assert exc.value.field == "scratch" 85 | # Bad scratch directory (relative path) 86 | with pytest.raises(YamlFieldError) as exc: 87 | BlockworkConfig.parse_str("!Blockwork\n" "project: test\n" "scratch: a/b\n") 88 | assert exc.value.field == "scratch" 89 | # Bad scratch directory (integer) 90 | with pytest.raises(YamlFieldError) as exc: 91 | BlockworkConfig.parse_str("!Blockwork\n" "project: test\n" "host_scratch: 123\n") 92 | assert exc.value.field == "host_scratch" 93 | # Bad state directory (integer) 94 | with pytest.raises(YamlFieldError) as exc: 95 | BlockworkConfig.parse_str("!Blockwork\n" "project: test\n" "host_state: 123\n") 96 | assert exc.value.field == "host_state" 97 | # Bootstrap and tool definitions 98 | for key, _name in (("bootstrap", "Bootstrap"), ("tooldefs", "Tool")): 99 | # Definitions not a list 100 | with pytest.raises(YamlFieldError) as exc: 101 | BlockworkConfig.parse_str("!Blockwork\n" "project: test\n" f"{key}: abcd\n") 102 | assert exc.value.field == key 103 | # Definitions not a list of strings 104 | with pytest.raises(YamlFieldError) as exc: 105 | BlockworkConfig.parse_str("!Blockwork\n" "project: test\n" f"{key}: [1, 2, 3]\n") 106 | assert exc.value.field == key 107 | -------------------------------------------------------------------------------- /blockwork/state.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import atexit 16 | import json 17 | import logging 18 | from pathlib import Path 19 | from typing import Any 20 | 21 | 22 | class StateError(Exception): 23 | pass 24 | 25 | 26 | class StateNamespace: 27 | """ 28 | A wrapper around a state tracking file, which is simply a JSON dictionary 29 | written to disk. The wrapper allows arbitrary keys to be set and retrieved, 30 | with values serialised to disk when the tool exits. 31 | 32 | :param name: Name of the state object 33 | :param path: Path to the JSON file where data is serialised 34 | """ 35 | 36 | def __init__(self, name: str, path: Path) -> None: 37 | self.__name = name 38 | self.__path = path 39 | self.__data = {} 40 | self.__altered = False 41 | self.load() 42 | 43 | def load(self) -> None: 44 | """Load state from disk if the file exists""" 45 | if self.__path.exists(): 46 | with self.__path.open("r", encoding="utf-8") as fh: 47 | self.__data = json.load(fh) 48 | 49 | def store(self) -> None: 50 | """Write out state to disk if any values have been changed""" 51 | # Check the alterations flag, return immediately if nothing has changed 52 | if not self.__altered: 53 | return 54 | # Write out the updated data 55 | logging.debug(f"Saving updated state for {self.__name} to {self.__path}") 56 | with self.__path.open("w", encoding="utf-8") as fh: 57 | json.dump(self.__data, fh, indent=4) 58 | # Clear the alterations flag 59 | self.__altered = False 60 | 61 | def __getattr__(self, name: str) -> Any: 62 | try: 63 | return super().__getattr__(name) 64 | except Exception: 65 | return self.get(name) 66 | 67 | def __setattr__(self, name: str, value: str | int | float | bool) -> None: 68 | if name in ("get", "set") or name.startswith("_"): 69 | super().__setattr__(name, value) 70 | else: 71 | self.set(name, value) 72 | 73 | def get(self, name: str, default: Any = None) -> str | int | float | bool | None: 74 | """ 75 | Retrieve a value from the stored data, returning a default value if the 76 | key has not been set. 77 | 78 | :param name: Name of the attribute to retrieve 79 | :param default: Default value to return if no previous value set 80 | :returns: Value or the default value if not defined 81 | """ 82 | return self.__data.get(name, default) 83 | 84 | def set(self, name: str, value: str | int | float | bool) -> None: 85 | """ 86 | Set a value into the stored data, this must be of a primitive type such 87 | as string, integer, float, or boolean (so that it can be serialised) 88 | 89 | :param name: Name of the attribute to set 90 | :param value: Value to set 91 | """ 92 | if not isinstance(value, str | int | float | bool): 93 | raise StateError(f"Value of type {type(value).__name__} is not supported") 94 | if not self.__altered: 95 | self.__altered = value != self.__data.get(name, None) 96 | self.__data[name] = value 97 | 98 | 99 | class State: 100 | """ 101 | Manages the state tracking folder for the project 102 | 103 | :param location: Absolute path to the state folder 104 | """ 105 | 106 | def __init__(self, location: Path) -> None: 107 | self.__location = location 108 | self.__files: dict[str, StateNamespace] = {} 109 | # When the program exits, ensure all modifications are saved to disk 110 | atexit.register(self.save_all) 111 | 112 | def save_all(self) -> None: 113 | """Iterate through all open state objects and store any modifications""" 114 | self.__location.mkdir(parents=True, exist_ok=True) 115 | for file in self.__files.values(): 116 | file.store() 117 | 118 | def __getattr__(self, name: str) -> Any: 119 | try: 120 | return super().__getattr__(name) 121 | except Exception: 122 | return self.get(name) 123 | 124 | def get(self, name: str) -> StateNamespace: 125 | """ 126 | Retrieve a state file wrapper for a given name, generating a new wrapper 127 | on the fly if one has never been retrieved before. 128 | 129 | :param name: Name of the state file 130 | :returns: Instance of StateNamespace 131 | """ 132 | if name not in self.__files: 133 | self.__files[name] = StateNamespace(name, self.__location / f"{name}.json") 134 | return self.__files[name] 135 | -------------------------------------------------------------------------------- /example/infra/tools/objstore.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import pprint 17 | import shutil 18 | from pathlib import Path 19 | from typing import ClassVar 20 | from zipfile import ZipFile 21 | 22 | import boto3 23 | import botocore 24 | 25 | from blockwork.common.singleton import Singleton 26 | from blockwork.context import Context 27 | from blockwork.tools import Invocation, Tool, Version 28 | 29 | 30 | @Tool.register() 31 | class ObjStore(Tool, metaclass=Singleton): 32 | versions: ClassVar[list[Version]] = [ 33 | Version( 34 | location=Tool.HOST_ROOT / "objstore", 35 | version="latest", 36 | default=True, 37 | ), 38 | ] 39 | 40 | def __init__(self, *args, **kwds) -> None: 41 | super().__init__(*args, **kwds) 42 | self._is_setup = False 43 | 44 | def setup(self, ctx: Context) -> "ObjStore": 45 | if self._is_setup: 46 | return self 47 | self._is_setup = True 48 | # Check if the object store has been configured 49 | if None in ( 50 | ctx.state.objstore.endpoint, 51 | ctx.state.objstore.access_key, 52 | ctx.state.objstore.secret_key, 53 | ctx.state.objstore.bucket, 54 | ): 55 | logging.warning("Object store details need to be configured") 56 | ctx.state.objstore.endpoint = input("Endpoint URL: ") 57 | ctx.state.objstore.access_key = input("Access Key : ") 58 | ctx.state.objstore.secret_key = input("Secret Key : ") 59 | ctx.state.objstore.bucket = input("Bucket : ") 60 | logging.info("Object store configured") 61 | # Create a client 62 | self.store = boto3.client( 63 | service_name="s3", 64 | endpoint_url=ctx.state.objstore.endpoint, 65 | aws_access_key_id=ctx.state.objstore.access_key, 66 | aws_secret_access_key=ctx.state.objstore.secret_key, 67 | ) 68 | # Locally cache the bucket 69 | self.bucket = ctx.state.objstore.bucket 70 | # Return self to allow chaining 71 | return self 72 | 73 | def get_info(self, path: str) -> dict[str, str] | None: 74 | try: 75 | return self.store.head_object(Bucket=self.bucket, Key=path) 76 | except botocore.exceptions.ClientError: 77 | return None 78 | 79 | def download(self, path: str, target: Path) -> None: 80 | if not self.get_info(path): 81 | raise Exception(f"Unknown object: {path}") 82 | with target.open("wb") as fh: 83 | self.store.download_fileobj(self.bucket, path, fh) 84 | 85 | @Tool.action() 86 | def authenticate(self, ctx: Context, *args: list[str]) -> Invocation: 87 | if len(args) == 4: 88 | ctx.state.objstore.endpoint = args[0] 89 | ctx.state.objstore.access_key = args[1] 90 | ctx.state.objstore.secret_key = args[2] 91 | ctx.state.objstore.bucket = args[3] 92 | elif len(args) > 0: 93 | raise Exception( 94 | "SYNTAX: bw tool objstore.authenticate " 95 | " " 96 | ) 97 | ObjStore().setup(ctx) 98 | return None 99 | 100 | @Tool.action() 101 | def lookup(self, ctx: Context, path: str) -> Invocation: 102 | ObjStore().setup(ctx) 103 | if lkp := ObjStore().get_info(path): 104 | logging.info( 105 | pprint.pformat( 106 | lkp, indent=4, compact=True, width=shutil.get_terminal_size().columns - 20 107 | ) 108 | ) 109 | else: 110 | raise Exception(f"No object known for: {path}") 111 | 112 | 113 | def from_objstore(func): 114 | def _mock(tool: Tool, ctx: Context, version: Version, *args: list[str]) -> Invocation: 115 | objname = f"{tool.name.lower()}_{version.version.replace('.', '_')}.zip" 116 | store = ObjStore().setup(ctx) 117 | # If the object store doesn't offer this tool, defer to the next installer 118 | if not store.get_info(objname): 119 | logging.warning(f"Object store doesn't offer {tool.name} @ {version.version}") 120 | return func(tool, ctx, version, *args) 121 | # Download the object 122 | logging.debug(f"Downloading tool {tool.name} from object store") 123 | store.download(objname, target := ctx.host_tools / objname) 124 | # Unpack archive 125 | logging.debug(f"Unpacking tool {tool.name} into {ctx.host_tools}") 126 | with ZipFile(target, "r") as zh: 127 | zh.extractall(ctx.host_tools) 128 | # Clean-up 129 | logging.debug("Removing zip archive") 130 | target.unlink() 131 | 132 | return _mock 133 | -------------------------------------------------------------------------------- /blockwork/common/checkeddataclasses.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import dataclasses 16 | import warnings 17 | from collections.abc import Callable 18 | from typing import Any, Generic, Literal, Self, TypeVar, cast, dataclass_transform 19 | 20 | import typeguard 21 | 22 | 23 | class FieldError(TypeError): 24 | "Dataclass field validation error" 25 | 26 | def __init__(self, msg: str, field: str): 27 | self.msg = msg 28 | self.field = field 29 | 30 | def __str__(self): 31 | return self.msg 32 | 33 | 34 | FunctionType = TypeVar("FunctionType", bound=Callable[..., Any]) 35 | 36 | 37 | class CopySignature(Generic[FunctionType]): 38 | def __init__(self, target: FunctionType) -> None: 39 | ... 40 | 41 | def __call__(self, wrapped: Callable[..., Any]) -> FunctionType: 42 | return cast(FunctionType, wrapped) 43 | 44 | 45 | DCLS = TypeVar("DCLS") 46 | 47 | 48 | def _dataclass_inner(cls: DCLS) -> DCLS: 49 | "Subclasses a dataclass, adding checking after initialisation." 50 | orig_init = cls.__init__ 51 | 52 | # Replacement init function calls original, then runs checks 53 | def _dc_init(self, *args, **kwargs): 54 | orig_init(self, *args, **kwargs) 55 | 56 | # Check each field has the expected type 57 | for field in dataclasses.fields(cls): 58 | value = getattr(self, field.name) 59 | with warnings.catch_warnings(): 60 | # Catches a warning when typegaurd can't resolve a string type 61 | # definition to an actual type meaning it can't check the type. 62 | # This isn't ideal, but as far as @ed.kotarski can tell there 63 | # is no way round this limitation in user code meaning the 64 | # warning is just noise. 65 | warnings.simplefilter("ignore", category=typeguard.TypeHintWarning) 66 | try: 67 | typeguard.check_type(value, field.type) 68 | except typeguard.TypeCheckError as ex: 69 | raise FieldError(str(ex), field.name) from None 70 | if isinstance(field, Field): 71 | field.run_checks(value) 72 | 73 | cls.__init__ = _dc_init 74 | return cls 75 | 76 | 77 | class Field(dataclasses.Field): 78 | "Checked version of Field. See field." 79 | 80 | checkers: list[Callable[[Self, Any], None]] 81 | 82 | def check(self, checker: Callable[[Any], None]): 83 | """ 84 | Register a checking function for this field. 85 | Intended for use as a decorator. 86 | Returns the checker function so it is chainable. 87 | """ 88 | self.checkers = getattr(self, "checkers", []) 89 | self.checkers.append(checker) 90 | return checker 91 | 92 | def run_checks(self, value): 93 | for checker in getattr(self, "checkers", []): 94 | try: 95 | checker(self, value) 96 | except TypeError as ex: 97 | raise FieldError(str(ex), self.name) from None 98 | 99 | 100 | T_Field = TypeVar("T_Field") 101 | 102 | 103 | def field( 104 | *, 105 | default: T_Field | Literal[dataclasses.MISSING] = dataclasses.MISSING, 106 | default_factory: Callable[[], T_Field] | Literal[dataclasses.MISSING] = dataclasses.MISSING, 107 | init=True, 108 | repr=True, # noqa: A002 109 | hash=None, # noqa: A002 110 | compare=True, 111 | metadata=None, 112 | kw_only=dataclasses.MISSING, 113 | ) -> T_Field: 114 | """ 115 | Checked version of field which allows addictional checking functions to be 116 | registered to a field. Checking functions should raise type errors if the 117 | field value is not valid. For example: 118 | 119 | @dataclass 120 | class Location: 121 | path: str = field() 122 | column: int 123 | line: int 124 | 125 | @path.check 126 | def absPath(value): 127 | if not value.startswith("/"): 128 | raise TypeError("Expected absolute path") 129 | 130 | """ 131 | if default is not dataclasses.MISSING and default_factory is not dataclasses.MISSING: 132 | raise ValueError("cannot specify both default and default_factory") 133 | return cast( 134 | T_Field, Field(default, default_factory, init, repr, hash, compare, metadata, kw_only) 135 | ) 136 | 137 | 138 | @CopySignature(dataclasses.dataclass) 139 | @dataclass_transform( 140 | kw_only_default=True, frozen_default=True, eq_default=False, field_specifiers=(field,) 141 | ) 142 | def dataclass(__cls=None, /, **kwargs): 143 | "Checked version of the dataclass decorator which adds runtime type checking." 144 | if __cls is None: 145 | 146 | def wrap(cls): 147 | dc = dataclasses.dataclass(**kwargs)(cls) 148 | return _dataclass_inner(dc) 149 | 150 | return wrap 151 | else: 152 | dc = dataclasses.dataclass()(__cls) 153 | return _dataclass_inner(dc) 154 | -------------------------------------------------------------------------------- /blockwork/config/base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import typing 16 | from collections.abc import Iterable 17 | from typing import Protocol, dataclass_transform 18 | 19 | import yaml 20 | 21 | from ..common.checkeddataclasses import dataclass, field 22 | from ..common.singleton import keyed_singleton 23 | from ..common.yaml import DataclassConverter 24 | from ..common.yaml.parsers import Parser 25 | from ..transforms import Transform 26 | from .api import ConfigApi 27 | 28 | 29 | class ConfigConverter(DataclassConverter["ConfigProtocol", "Parser"]): 30 | def construct_scalar(self, loader: yaml.Loader, node: yaml.ScalarNode) -> "ConfigProtocol": 31 | # Allow elements to be indirected with a path e.g. `! [.]` 32 | target = loader.construct_scalar(node) 33 | if not isinstance(target, str): 34 | raise RuntimeError 35 | with ConfigApi.current.with_target(target, self.typ) as api: 36 | return api.target.config 37 | 38 | def construct_mapping(self, loader: yaml.Loader, node: yaml.MappingNode) -> "ConfigProtocol": 39 | with ConfigApi.current.with_node(node): 40 | return super().construct_mapping(loader, node) 41 | 42 | 43 | @typing.runtime_checkable 44 | class ConfigProtocol(Protocol): 45 | "Protocol for Config objects" 46 | 47 | def iter_config(self) -> Iterable["ConfigProtocol"]: 48 | ... 49 | 50 | def iter_transforms(self) -> Iterable[Transform]: 51 | ... 52 | 53 | def config_filter(self, config: "ConfigProtocol") -> bool: 54 | ... 55 | 56 | def transform_filter(self, transform: Transform, config: "ConfigProtocol") -> bool: 57 | ... 58 | 59 | 60 | @dataclass_transform( 61 | kw_only_default=True, frozen_default=True, eq_default=False, field_specifiers=(field,) 62 | ) 63 | class Config(metaclass=keyed_singleton(inst_key=lambda i: hash(i))): 64 | """ 65 | Base class for all config. 66 | All-caps keys are reserved. 67 | """ 68 | 69 | "Defines which YAML registry the config belongs to i.e. site/project/element" 70 | # _CONVERTER: type[DataclassConverter] = DataclassConverter 71 | _CONVERTER = ConfigConverter 72 | "Defines how to convert the YAML tag into a Python object" 73 | YAML_TAG: str | None = None 74 | "The ! to represent this document in YAML" 75 | FILE_NAME: str | None = None 76 | "The api object for this config" 77 | api: ConfigApi 78 | "The parser for this config" 79 | parser: Parser = Parser() 80 | """The implicit file name to use when one isn't provided, 81 | defaults to YAML_TAG if provided, else class name""" 82 | 83 | def __init_subclass__(cls, *args, **kwargs): 84 | super().__init_subclass__(*args, **kwargs) 85 | cls.api = field(default_factory=lambda: ConfigApi.current) 86 | # Ensure that even if no annotations existed before, that the class is 87 | # well behaved as this caused a bug under Python 3.11.7 88 | if not hasattr(cls, "__annotations__"): 89 | cls.__annotations__ = {} 90 | if "__annotations__" not in cls.__dict__: 91 | setattr(cls, "__annotations__", cls.__annotations__) # noqa: B010 92 | # Force an annotation for 'api' 93 | cls.__annotations__["api"] = ConfigApi 94 | dataclass(kw_only=True, frozen=True, eq=False, repr=False)(cls) 95 | cls.parser.register(cls._CONVERTER, tag=cls.YAML_TAG)(cls) 96 | 97 | def __init__(self, *args, **kwargs): 98 | ... 99 | 100 | def __hash__(self): 101 | return self.api.node_id() or id(self) 102 | 103 | def __eq__(self, other): 104 | return hash(self) == hash(other) 105 | 106 | def iter_config(self) -> Iterable["ConfigProtocol"]: 107 | """ 108 | Yields any sub-config which is used as part of this one. 109 | 110 | Implementation notes: 111 | - This function must be implemented when sub-elements are used. 112 | """ 113 | yield from [] 114 | 115 | def iter_transforms(self) -> Iterable[Transform]: 116 | """ 117 | Yields any transforms from this element. 118 | """ 119 | yield from [] 120 | 121 | def config_filter(self, config: "ConfigProtocol") -> bool: 122 | """ 123 | Filter configs underneath this which are "interesting". 124 | 125 | For uninteresting configs, we use our own transform filter 126 | for transforms underneath them, for interesting configs we 127 | use theirs. 128 | """ 129 | return False 130 | 131 | def transform_filter(self, transform: Transform, config: "ConfigProtocol") -> bool: 132 | """ 133 | Filter transforms underneath this which are "interesting". 134 | 135 | Interesting transforms will be the run targets, uninteresting 136 | transforms which are dependencies will also get run. 137 | """ 138 | return True 139 | 140 | 141 | class Site(Config): 142 | "Base class for site configuration" 143 | 144 | projects: dict[str, str] 145 | 146 | 147 | class Project(Config): 148 | "Base class for project configuration" 149 | 150 | units: dict[str, str] 151 | -------------------------------------------------------------------------------- /docs/tech/caching.md: -------------------------------------------------------------------------------- 1 | Caching allows build outputs to be re-used across workflow runs. In Blockwork 2 | caching is used to: 3 | 4 | - Save compute: Builds don't need to be repeated 5 | - Save disk-space: Identical objects can be de-duplicated 6 | - Save developer-time: Re-building only changed items 7 | - Ensure build determinism: Checking the same inputs always produce the same 8 | output 9 | 10 | See [Caching](../config/caching.md) for details on how to configure caches, or 11 | read on for details on how to implement your own Blockwork caches, or how the 12 | caching scheme works. 13 | 14 | ## Implementing a Cache 15 | 16 | A custom cache can be implemented by inheriting from the base Cache class and 17 | implementing a minimal set of methods. A minimal implementation which stores 18 | into a local directory is: 19 | 20 | ```python 21 | from collections.abc import Iterable 22 | from filelock import FileLock 23 | from pathlib import Path 24 | from shutil import copy, copytree 25 | 26 | from blockwork.build.caching import Cache 27 | from blockwork.config import CacheConfig 28 | from blockwork.context import Context 29 | 30 | 31 | class BasicFileCache(Cache): 32 | def __init__(self, ctx: Context, cfg: CacheConfig) -> None: 33 | super().__init__(ctx, cfg) # Super must be called! 34 | self.cache_root: Path = ctx.host_scratch / "my-cache" 35 | self.cache_root.mkdir(exist_ok=True) 36 | self.lock = FileLock(self.cache_root.with_suffix(".lock")) 37 | 38 | def store_item(self, key: str, frm: Path) -> bool: 39 | with self.lock: 40 | to = self.cache_root / key 41 | if to.exists(): 42 | # Two different key hashes resulted in the same content hash, 43 | # item has already been stored 44 | return True 45 | to.parent.mkdir(exist_ok=True, parents=True) 46 | if frm.is_dir(): 47 | copytree(frm, to) 48 | else: 49 | copy(frm, to) 50 | return True 51 | 52 | def fetch_item(self, key: str, to: Path) -> bool: 53 | to.parent.mkdir(exist_ok=True, parents=True) 54 | frm = self.cache_root / key 55 | if not frm.exists(): 56 | return False 57 | try: 58 | to.symlink_to(frm, target_is_directory=frm.is_dir()) 59 | except FileNotFoundError: 60 | return False 61 | return True 62 | ``` 63 | 64 | This implementation will give you a local cache which is safe for running in 65 | parallel (the lock could be removed if only running serially). 66 | 67 | However, without implementing some of the optional methods this cache will grow 68 | indefinitely. To allow the cache to self prune, some extra methods must be 69 | implemented: 70 | 71 | ```python 72 | def drop_item(self, key: str) -> bool: 73 | with self.lock: 74 | path = self.cache_root / key 75 | if path.exists(): 76 | if path.is_dir(): 77 | rmtree(path) 78 | else: 79 | path.unlink() 80 | return True 81 | 82 | def iter_keys(self) -> Iterable[str]: 83 | if not self.cache_root.exists(): 84 | yield from [] 85 | yield from self.cache_root.iterdir() 86 | ``` 87 | 88 | This allows the cache to prune itself down to the maximum size as expressed in 89 | the cache configuration. It will prune itself at the end of each workflow 90 | by calculating a score for each item based on `time-to-create / size` and 91 | removing items from lowest to highest score until the max-size is reached. 92 | 93 | A final enhancement enables intelligent pruning based on how recently an item 94 | was used by including a `time-since-last-use` term. This can be enabled by 95 | implementing a final pair of methods as follows: 96 | 97 | ```python 98 | def get_last_fetch_utc(self, key: str) -> float: 99 | frm = self.cache_root / key 100 | try: 101 | return frm.stat().st_mtime 102 | except OSError: 103 | return 0 104 | 105 | def set_last_fetch_utc(self, key: str): 106 | frm = self.cache_root / key 107 | if frm.exists(): 108 | frm.touch(exist_ok=True) 109 | ``` 110 | 111 | ## Caching Scheme 112 | 113 | Blockwork's cache scheme calculates two types of hash: 114 | 115 | - Transform hashes: Calculated by hashing a transform's definition with the 116 | definition of its dependencies recursively. This can be calculated before 117 | any transforms have been run. 118 | - File hashes: Calculated by hashing a transform file-output, which can only 119 | be done after the transform has run. 120 | 121 | After a transform is run, the output-files are stored in the cache according 122 | to their hash, and a key-file containing each output hash is stored according 123 | to the transform hash. 124 | 125 | This two level scheme allows many transforms to refer to the same cached file, 126 | preventing unnecessary copies being stored. 127 | 128 | ### Process 129 | 130 | Stages: 131 | 132 | - Initial: 133 | - Hash the content of static input interfaces 134 | - Use these to compute transform hashkeys for nodes with no dependencies 135 | - Use input interface names along with the hashkeys of transforms that 136 | output them to get the transform hashkeys for nodes with dependencies 137 | 138 | - Pre-run: 139 | - Go through the transforms in reverse order 140 | - Try and pull all output interfaces from caches - if successful mark that 141 | transform as fetched. 142 | - If all dependents of a transform are fetched, mark a transform as skipped 143 | 144 | - During-run: 145 | - Go through transforms in dependency order as usual 146 | - Skip transforms marked as fetched or skipped 147 | - Push output interfaces to all caches that allow it 148 | -------------------------------------------------------------------------------- /tests/test_context.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | import pytest 18 | 19 | from blockwork.common.yaml import YamlConversionError 20 | from blockwork.config import Blockwork 21 | from blockwork.context import Context 22 | from blockwork.state import State 23 | 24 | 25 | class TestContext: 26 | def test_context(self, tmp_path: Path) -> None: 27 | """Context should recognise the .bw.yaml file""" 28 | root = tmp_path / "project" 29 | bw_yaml = root / ".bw.yaml" 30 | infra = root / "infra" 31 | infra.mkdir(parents=True) 32 | # Create a tool definition 33 | with (infra / "tools.py").open("w", encoding="utf-8") as fh: 34 | fh.write( 35 | "from pathlib import Path\n" 36 | "from blockwork.tools import Tool, Version\n" 37 | "class ToolA(Tool):\n" 38 | f" versions = [Version('1.1', Path('{infra}'))]\n" 39 | ) 40 | # Create a configuration file 41 | with bw_yaml.open("w", encoding="utf-8") as fh: 42 | fh.write("!Blockwork\nproject: test_project\nroot: /a/b\ntooldefs:\n - infra.tools\n") 43 | # Create a context 44 | ctx = Context(root) 45 | assert ctx.host_root == root 46 | assert ctx.host_scratch.samefile(tmp_path / "test_project.scratch") 47 | assert ctx.host_state.samefile(tmp_path / "test_project.state") 48 | assert ctx.host_scratch.exists() 49 | assert ctx.host_state.exists() 50 | assert ctx.container_root == Path("/a/b") 51 | assert ctx.container_scratch == Path("/scratch") 52 | assert ctx.file == Path(".bw.yaml") 53 | assert ctx.config_path == bw_yaml 54 | assert isinstance(ctx.config, Blockwork) 55 | assert ctx.config.project == "test_project" 56 | 57 | def test_context_dig(self, tmp_path: Path) -> None: 58 | """Context should recognise the .bw.yaml file in a parent layer""" 59 | bw_yaml = tmp_path / ".bw.yaml" 60 | infra = tmp_path / "infra" 61 | infra.mkdir() 62 | # Create a tool definition 63 | with (infra / "tools.py").open("w", encoding="utf-8") as fh: 64 | fh.write( 65 | "from pathlib import Path\n" 66 | "from blockwork.tools import Tool, Version\n" 67 | "class ToolA(Tool):\n" 68 | f" versions = [Version('1.1', Path('{infra}'))]\n" 69 | ) 70 | # Create a configuration file 71 | with bw_yaml.open("w", encoding="utf-8") as fh: 72 | fh.write("!Blockwork\nproject: test_project\nroot: /a/b\ntooldefs:\n - infra.tools\n") 73 | # Create a context in a sub-path 74 | sub_path = tmp_path / "a" / "b" / "c" 75 | sub_path.mkdir(parents=True) 76 | ctx = Context(sub_path) 77 | assert ctx.host_root == tmp_path 78 | assert ctx.container_root == Path("/a/b") 79 | assert ctx.file == Path(".bw.yaml") 80 | assert ctx.config_path == bw_yaml 81 | assert isinstance(ctx.config, Blockwork) 82 | assert ctx.config.project == "test_project" 83 | 84 | def test_context_bad_path(self, tmp_path: Path) -> None: 85 | """A bad root should raise an exception""" 86 | with pytest.raises(Exception) as exc: 87 | Context(tmp_path) 88 | assert str(exc.value) == f"Could not identify work area in parents of {tmp_path}" 89 | 90 | def test_context_bad_config(self, tmp_path: Path) -> None: 91 | """A malformed configuration should raise an exception""" 92 | bw_yaml = tmp_path / ".bw.yaml" 93 | with bw_yaml.open("w", encoding="utf-8") as fh: 94 | fh.write("blargh\n") 95 | with pytest.raises(YamlConversionError): 96 | _ = Context(tmp_path).config 97 | 98 | def test_context_state(self, tmp_path: Path) -> None: 99 | """Check that a state object is created at the right path""" 100 | bw_yaml = tmp_path / ".bw.yaml" 101 | with bw_yaml.open("w", encoding="utf-8") as fh: 102 | fh.write("!Blockwork\nproject: test\nhost_state: .my_{project}_state\n") 103 | ctx = Context(tmp_path) 104 | assert isinstance(ctx.state, State) 105 | assert ctx.state._State__location == tmp_path / ".my_test_state" 106 | 107 | def test_context_root_dir(self, tmp_path: Path) -> None: 108 | """Test that {root_dir} substitution works for scratch and state directories""" 109 | root = tmp_path / "other_dir" 110 | root.mkdir(exist_ok=True) 111 | bw_yaml = root / ".bw.yaml" 112 | # Create a configuration file 113 | with bw_yaml.open("w", encoding="utf-8") as fh: 114 | fh.write( 115 | "!Blockwork\n" 116 | "project: test_project\n" 117 | "host_scratch: ../{root_dir}.scratch\n" 118 | "host_state: ../{root_dir}.state\n" 119 | ) 120 | # Create a context 121 | ctx = Context(root) 122 | assert ctx.host_root == root 123 | assert ctx.host_scratch.samefile(tmp_path / "other_dir.scratch") 124 | assert ctx.host_state.samefile(tmp_path / "other_dir.state") 125 | assert ctx.host_scratch.exists() 126 | assert ctx.host_state.exists() 127 | -------------------------------------------------------------------------------- /blockwork/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import sys 17 | from pathlib import Path 18 | 19 | import click 20 | from rich.console import Console 21 | from rich.logging import RichHandler 22 | 23 | from .activities import activities 24 | from .bootstrap import Bootstrap 25 | from .common.registry import Registry 26 | from .containers.runtime import Runtime 27 | from .context import Context, DebugScope, HostArchitecture 28 | from .tools import Tool 29 | 30 | logging.basicConfig( 31 | level=logging.INFO, 32 | format="%(message)s", 33 | datefmt="[%X]", 34 | handlers=[RichHandler(rich_tracebacks=True, tracebacks_show_locals=True, show_path=False)], 35 | ) 36 | 37 | 38 | @click.group() 39 | @click.pass_context 40 | @click.option( 41 | "--cwd", 42 | "-C", 43 | type=click.Path(exists=True, file_okay=False), 44 | default=None, 45 | help="Override the working directory", 46 | ) 47 | @click.option( 48 | "--verbose", 49 | "-v", 50 | is_flag=True, 51 | default=False, 52 | help="Raise the verbosity of messages to debug", 53 | ) 54 | @click.option( 55 | "--verbose-locals", 56 | is_flag=True, 57 | default=False, 58 | help="Print local variables in an exception traceback", 59 | ) 60 | @click.option( 61 | "--quiet", 62 | "-q", 63 | is_flag=True, 64 | default=False, 65 | help="Lower the verbosity of messages to warning", 66 | ) 67 | @click.option( 68 | "--runtime", 69 | "-r", 70 | type=str, 71 | default=None, 72 | help="Set a specific container runtime to use", 73 | ) 74 | @click.option("--arch", type=str, default=None, help="Override the host architecture") 75 | @click.option( 76 | "--pdb", 77 | is_flag=True, 78 | default=False, 79 | help="Enable PDB post-mortem debugging on any exception", 80 | ) 81 | @click.option( 82 | "--scratch", 83 | type=click.Path(file_okay=False), 84 | default=None, 85 | help="Override the scratch folder location", 86 | ) 87 | @click.option( 88 | "--cache/--no-cache", 89 | default=None, 90 | help="Enable or disable caching", 91 | ) 92 | @click.option( 93 | "--cache-config", 94 | "--cc", 95 | type=click.Path(dir_okay=False, exists=True, path_type=Path), 96 | default=None, 97 | help="Path to a cache config file.", 98 | ) 99 | @click.option( 100 | "--cache-targets/--no-cache-targets", 101 | default=None, 102 | help="Force caching even for 'targetted' transforms", 103 | ) 104 | @click.option( 105 | "--cache-trace/--no-cache-trace", 106 | default=None, 107 | help="Turn on cache debug tracing", 108 | ) 109 | @click.option( 110 | "--cache-expect/--no-cache-expect", 111 | default=None, 112 | help="Raise exception if any transform not available in a cache", 113 | ) 114 | def blockwork( 115 | ctx, 116 | cwd: str, 117 | verbose: bool, 118 | verbose_locals: bool, 119 | quiet: bool, 120 | pdb: bool, 121 | runtime: str | None, 122 | arch: str | None, 123 | scratch: str | None, 124 | cache: bool | None, 125 | cache_config: Path, 126 | cache_targets: bool | None, 127 | cache_trace: bool | None, 128 | cache_expect: bool | None, 129 | ) -> None: 130 | # Setup post-mortem debug 131 | DebugScope.current.POSTMORTEM = pdb 132 | # Setup the verbosity 133 | DebugScope.current.VERBOSE = verbose 134 | DebugScope.current.VERBOSE_LOCALS = verbose and verbose_locals 135 | if verbose: 136 | logging.info("Setting logging verbosity to DEBUG") 137 | logging.getLogger().setLevel(logging.DEBUG) 138 | elif quiet: 139 | logging.getLogger().setLevel(logging.WARNING) 140 | # Set a preferred runtime, if provided 141 | if runtime: 142 | Runtime.set_preferred_runtime(runtime) 143 | # Create the context object and attach to click 144 | ctx.obj = Context( 145 | root=Path(cwd).absolute() if cwd else None, 146 | scratch=Path(scratch).absolute() if scratch else None, 147 | cache_enable=cache, 148 | cache_config=cache_config, 149 | cache_targets=cache_targets, 150 | cache_trace=cache_trace, 151 | cache_expect=cache_expect, 152 | ) 153 | # Set the host architecture 154 | if arch: 155 | ctx.obj.host_architecture = HostArchitecture(arch) 156 | # Trigger registration procedures 157 | Tool.setup(ctx.obj.host_root, ctx.obj.config.tooldefs) 158 | Bootstrap.setup(ctx.obj.host_root, ctx.obj.config.bootstrap) 159 | Registry.setup(ctx.obj.host_root, ctx.obj.config.workflows) 160 | Registry.setup(ctx.obj.host_root, ctx.obj.config.config) 161 | 162 | 163 | for activity in activities: 164 | blockwork.add_command(activity) 165 | 166 | 167 | def main(): 168 | with DebugScope(VERBOSE=True, VERBOSE_LOCALS=True, POSTMORTEM=False) as v: 169 | try: 170 | blockwork(auto_envvar_prefix="BW") 171 | sys.exit(0) 172 | except Exception as e: 173 | # Log the exception 174 | if type(e) is not Exception: 175 | logging.error(f"{type(e).__name__}: {e}") 176 | else: 177 | logging.error(str(e)) 178 | if v.VERBOSE: 179 | Console().print_exception(show_locals=v.VERBOSE_LOCALS) 180 | # Enter PDB post-mortem debugging if required 181 | if v.POSTMORTEM: 182 | import pdb 183 | 184 | pdb.post_mortem() 185 | # Exit with failure 186 | sys.exit(1) 187 | 188 | 189 | if __name__ == "__main__": 190 | main() 191 | -------------------------------------------------------------------------------- /blockwork/activities/cache.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import json 17 | from pathlib import Path 18 | from pprint import pprint 19 | 20 | import click 21 | 22 | from ..build.caching import Cache, TraceData, TransformKeyData 23 | from ..context import Context 24 | 25 | 26 | @click.group() 27 | def cache() -> None: 28 | """ 29 | Cache utilities 30 | """ 31 | pass 32 | 33 | 34 | def get_key_data(ctx: Context, key: str, from_cache: str | None = None) -> TransformKeyData | None: 35 | if any(key.startswith(p) for p in ("./", "../", "/")): 36 | print(f"Assuming key '{key}' is a key_file") 37 | 38 | with Path(key).open("r") as f: 39 | return json.load(f) 40 | 41 | print(f"Assuming key '{key}' is a cache key (use ./ prefix if this is a file)") 42 | if not key.startswith(Cache.transform_prefix): 43 | key = Cache.transform_prefix + key 44 | 45 | for cache in ctx.caches: 46 | if from_cache is not None and cache.cfg.name != from_cache: 47 | continue 48 | data = cache.fetch_object(key) 49 | if data is not None: 50 | print(f"Key '{key}' found in cache: '{cache.cfg.name}'") 51 | return data 52 | print(f"Key '{key}' not found") 53 | return None 54 | 55 | 56 | def format_trace(trace: list[TraceData], depth=0, max_depth=0) -> list[str]: 57 | lines = [] 58 | 59 | for typ, ident, own_hash, rolling_hash, sub_trace in trace: 60 | lines.append(f"{depth} {rolling_hash} {own_hash} {depth*' '} {typ}[{ident}]") 61 | if max_depth < 0 or depth < max_depth: 62 | lines.extend(format_trace(sub_trace, depth=depth + 1, max_depth=max_depth)) 63 | return lines 64 | 65 | 66 | @cache.command(name="read-key") 67 | @click.argument("key", type=click.STRING) 68 | @click.option( 69 | "--output", "-o", type=click.Path(writable=True, path_type=Path), required=False, default=None 70 | ) 71 | @click.option("--cache", "-c", type=click.STRING, required=False, default=None) 72 | @click.pass_obj 73 | def read_key(ctx: Context, key: str, output: Path | None, cache: str): 74 | """ 75 | Read transform key data 76 | """ 77 | data = get_key_data(ctx, key, cache) 78 | if data is None: 79 | exit(1) 80 | 81 | if output is None: 82 | pprint(data) 83 | else: 84 | with output.open("w") as f: 85 | json.dump(data, f) 86 | exit(0) 87 | 88 | 89 | @cache.command(name="trace-key") 90 | @click.argument("key", type=click.STRING) 91 | @click.option("--depth", "-d", type=click.INT, required=False, default=-1) 92 | @click.option( 93 | "--output", "-o", type=click.Path(writable=True, path_type=Path), required=False, default=None 94 | ) 95 | @click.option("--cache", "-c", type=click.STRING, required=False, default=None) 96 | @click.pass_obj 97 | def trace_key(ctx: Context, key: str, depth: int, output: Path | None, cache: str | None): 98 | """ 99 | Read transform key data 100 | """ 101 | data = get_key_data(ctx, key, cache) 102 | if data is None: 103 | exit(1) 104 | 105 | if (trace := data.get("trace", None)) is None: 106 | print("No trace data! Did you run with '--cache-trace'?") 107 | exit(1) 108 | 109 | format_trace(trace, max_depth=depth) 110 | 111 | trace_lines = format_trace(trace, max_depth=depth) 112 | if output is None: 113 | for line in trace_lines: 114 | print(line) 115 | else: 116 | with output.open("w") as f: 117 | for line in trace_lines: 118 | f.write(line + "\n") 119 | exit(0) 120 | 121 | 122 | @cache.command(name="fetch-medial") 123 | @click.argument("key", type=click.STRING) 124 | @click.option("--output", "-o", type=click.Path(writable=True, path_type=Path), required=True) 125 | @click.pass_obj 126 | def fetch_medial(ctx: Context, key: str, output: Path): 127 | """ 128 | Fetch a single medial 129 | """ 130 | if not key.startswith(Cache.medial_prefix): 131 | key = Cache.medial_prefix + key 132 | for cache in ctx.caches: 133 | if cache.fetch_item(key, output): 134 | print(f"Item found in cache: '{cache.cfg.name}'") 135 | exit(0) 136 | exit(1) 137 | 138 | 139 | @cache.command(name="drop-key") 140 | @click.argument("key", type=click.STRING) 141 | @click.option("--yes", "-y", default=False, is_flag=True) 142 | @click.pass_obj 143 | def drop_key(ctx: Context, key: str, yes: bool): 144 | """ 145 | Drop transform key data 146 | """ 147 | if not key.startswith(Cache.transform_prefix): 148 | key = Cache.transform_prefix + key 149 | exit_code = 0 150 | for cache in ctx.caches: 151 | if yes or click.confirm(f"Drop key from cache '{cache.cfg.name}'?", default=False): 152 | if cache.drop_item(key): 153 | print(f"Item dropped from cache: '{cache.cfg.name}'") 154 | else: 155 | print(f"Item could not be dropped from cache: '{cache.cfg.name}'") 156 | exit_code = 1 157 | exit(exit_code) 158 | 159 | 160 | @cache.command(name="drop-medial") 161 | @click.argument("key", type=click.STRING) 162 | @click.option("--yes", "-y", default=False, is_flag=True) 163 | @click.pass_obj 164 | def drop_medial(ctx: Context, key: str, yes: bool): 165 | """ 166 | Fetch a single medial 167 | """ 168 | if not key.startswith(Cache.medial_prefix): 169 | key = Cache.medial_prefix + key 170 | exit_code = 0 171 | for cache in ctx.caches: 172 | if yes or click.confirm(f"Drop key from cache '{cache.cfg.name}'?", default=False): 173 | if cache.drop_item(key): 174 | print(f"Item dropped from cache: '{cache.cfg.name}'") 175 | else: 176 | print(f"Item could not be dropped from cache: '{cache.cfg.name}'") 177 | exit_code = 1 178 | exit(exit_code) 179 | -------------------------------------------------------------------------------- /tests/test_state.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | import time 17 | from pathlib import Path 18 | 19 | import pytest 20 | 21 | from blockwork.state import State, StateError 22 | 23 | 24 | class TestState: 25 | def test_state(self, tmp_path: Path) -> None: 26 | """Exercise state preservation and retrieval""" 27 | state_dirx = tmp_path / "state" 28 | # Create state object 29 | state = State(state_dirx) 30 | # Create multiple namespaces 31 | ns_1 = state.ns_1 32 | ns_2 = state.ns_2 33 | ns_3 = state.ns_3 34 | # Check that they're all distinct 35 | assert len({ns_1, ns_2, ns_3}) == 3 36 | # Check no data has been written yet 37 | assert not state_dirx.exists() 38 | # Setup some variables in each namespace 39 | ns_1.a = 123 40 | ns_1.b = "abc" 41 | ns_2.c = 234.567 42 | ns_2.d = False 43 | ns_3.e = 456 44 | ns_3.f = "def" 45 | # Check variables read back correctly 46 | assert ns_1.a == 123 47 | assert ns_1.b == "abc" 48 | assert ns_2.c == 234.567 49 | assert ns_2.d is False 50 | assert ns_3.e == 456 51 | assert ns_3.f == "def" 52 | # Check non-existing values return None 53 | assert ns_1.c is None 54 | assert ns_3.a is None 55 | # Save all state to disk, then check files were written out 56 | state.save_all() 57 | assert state_dirx.exists() 58 | assert (state_dirx / "ns_1.json").is_file() 59 | assert (state_dirx / "ns_2.json").is_file() 60 | assert (state_dirx / "ns_3.json").is_file() 61 | # Check file contents 62 | with (state_dirx / "ns_1.json").open("r", encoding="utf-8") as fh: 63 | assert json.load(fh) == {"a": 123, "b": "abc"} 64 | with (state_dirx / "ns_2.json").open("r", encoding="utf-8") as fh: 65 | assert json.load(fh) == {"c": 234.567, "d": False} 66 | with (state_dirx / "ns_3.json").open("r", encoding="utf-8") as fh: 67 | assert json.load(fh) == {"e": 456, "f": "def"} 68 | # Create a new state object and read back 69 | state_two = State(state_dirx) 70 | assert state_two.ns_1.a == 123 71 | assert state_two.ns_1.b == "abc" 72 | assert state_two.ns_2.c == 234.567 73 | assert state_two.ns_2.d is False 74 | assert state_two.ns_3.e == 456 75 | assert state_two.ns_3.f == "def" 76 | 77 | def test_state_autosave(self, mocker, tmp_path: Path) -> None: 78 | """Check that state registers 'save_all' with atexit""" 79 | # Mock atexit so the save event can be properly triggered 80 | registered = [] 81 | mk_atexit = mocker.patch("blockwork.state.atexit") 82 | 83 | def _register(method): 84 | nonlocal registered 85 | registered.append(method) 86 | 87 | mk_atexit.register.side_effect = _register 88 | # Check nothing is registered 89 | assert len(registered) == 0 90 | # Create a state object 91 | state_dirx = tmp_path / "state" 92 | state = State(state_dirx) 93 | # Check for the registration 94 | assert len(registered) == 1 95 | assert registered[0] == state.save_all 96 | # Write some values 97 | state.test_ns.test_var = 123 98 | # Trigger atexit 99 | for func in registered: 100 | func() 101 | # Check state written out 102 | with (state_dirx / "test_ns.json").open("r", encoding="utf-8") as fh: 103 | assert json.load(fh) == {"test_var": 123} 104 | 105 | def test_state_bad_value(self, tmp_path: Path) -> None: 106 | """Attempt to store an unsupported value""" 107 | state_dirx = tmp_path / "state" 108 | state = State(state_dirx) 109 | with pytest.raises(StateError) as exc: 110 | state.test_ns.test_var = [1, 2, 3] 111 | assert str(exc.value) == "Value of type list is not supported" 112 | 113 | def test_state_alterations(self, tmp_path: Path) -> None: 114 | """Namespaces should only write to disk when alterations have been made""" 115 | # Create a namespace 116 | state_dirx = tmp_path / "state" 117 | state = State(state_dirx) 118 | test_ns = state.test_ns 119 | # Check that the alteration flag starts low 120 | assert not test_ns._StateNamespace__altered 121 | # Make a change and check the flag is now set 122 | test_ns.some_var = 123 123 | assert test_ns._StateNamespace__altered 124 | # File should be written on save 125 | ns_file = state_dirx / "test_ns.json" 126 | assert not ns_file.exists() 127 | state.save_all() 128 | assert ns_file.exists() 129 | mtime = ns_file.stat().st_mtime 130 | # Check alteration flag is now low 131 | assert not test_ns._StateNamespace__altered 132 | # Delay a second to ensure modification time moves forward 133 | time.sleep(1) 134 | # Save again, and check no modification occurred 135 | state.save_all() 136 | assert ns_file.stat().st_mtime == mtime 137 | # Delay a second to ensure modification time moves forward 138 | time.sleep(1) 139 | # Alter a value 140 | test_ns.some_var = 234 141 | assert test_ns._StateNamespace__altered 142 | # Save again, and check file is updated 143 | state.save_all() 144 | new_mtime = ns_file.stat().st_mtime 145 | assert new_mtime > mtime 146 | mtime = new_mtime 147 | # Write the same value a second time, no alteration should be recorded 148 | test_ns.some_var = 234 149 | assert not test_ns._StateNamespace__altered 150 | # Delay a second to ensure modification time moves forward 151 | time.sleep(1) 152 | # Save again, and check no modification occurred 153 | state.save_all() 154 | assert ns_file.stat().st_mtime == mtime 155 | -------------------------------------------------------------------------------- /docs/syntax/bootstrap.md: -------------------------------------------------------------------------------- 1 | Bootstrapping is an extensible mechanism for running custom setup steps to prepare 2 | a checkout for use - some examples: 3 | 4 | * Performing sanity checks on the host system; 5 | * Downloading tools to run within the contained environment; 6 | * Compiling common libraries needed by different stages of a flow. 7 | 8 | In more general terms a good candidate as a bootstrapping step is a relatively 9 | complex action that needs to be performed only once to support many workflows. 10 | The action may change over time, and the bootstrapping mechanism provides means 11 | for testing if an action needs to be re-run (discussed in more detail below). 12 | 13 | ## Declaring a Bootstrap Step 14 | 15 | A bootstrap step is declared simply as a method marked with the `@Bootstrap.register()` 16 | decorator, for example: 17 | 18 | ```python linenums="1" title="infra/bootstrap/tool_a.py" 19 | import tempfile 20 | import zipfile 21 | from datetime import datetime 22 | from pathlib import Path 23 | from urllib import request 24 | 25 | from blockwork.bootstrap import Bootstrap 26 | from blockwork.context import Context 27 | 28 | @Bootstrap.register() 29 | def setup_tool_a(context : Context, last_run : datetime) -> bool: 30 | """ Download a tool from a shared server and extract it locally """ 31 | install_loc = context.host_root / "tools" / "tool_a" 32 | # Check to see if tool is already installed 33 | if install_loc.exists(): 34 | return True 35 | # Fetch the tool 36 | with tempfile.TemporaryDirectory() as tmpdir: 37 | local = Path(tmpdir) / "tool_a.zip" 38 | request.urlretrieve("http://www.example.com/tool_a.zip", local) 39 | with zipfile.ZipFile(local, "r") as zh: 40 | zh.extractall(install_loc) 41 | return False 42 | ``` 43 | 44 | The decorator `@Bootstrap.register()` marks the function that follows it as 45 | defining a bootstrapping step. When the full bootstrapping process is invoked, 46 | all bootstrap steps will be executed in the order they were registered. 47 | 48 | A bootstrap step must have the following attributes: 49 | 50 | * It must accept an argument called `context` which will carry an instance of 51 | the `Context` object - this can be used to locate the root area of the project, 52 | read the configuration, and access state information; 53 | 54 | * It must accept an argument called `last_run` which is a `datetime` instance 55 | carrying the last date the bootstrapping action was run, or set to UNIX time 56 | zero if its never been run before; 57 | 58 | * It must return a boolean value - `True` indicates that the bootstrap step was 59 | already up-to-date (no action was required), while `False` indicates that the 60 | step was not up-to-date (some action was required to setup the work area). 61 | 62 | ## Project Configuration 63 | 64 | Paths to bootstrapping routines must also be added into the 65 | [project configuration](../config/bw_yaml.md) so that Blockwork can discover them. 66 | 67 | For example: 68 | 69 | ```yaml linenums="1" title=".bw.yaml" 70 | !Blockwork 71 | project : my_project 72 | bootstrap: 73 | - infra.bootstrap.tool_a 74 | ``` 75 | 76 | ## Bootstrapping 77 | 78 | All of the registered bootstrapping steps can be executed using the 79 | [bootstrap](../cli/bootstrap.md) command: 80 | 81 | ```bash 82 | $> bw bootstrap 83 | [20:12:06] INFO Importing 1 bootstrapping paths 84 | INFO Invoking 1 bootstrap methods 85 | INFO Ran bootstrap step 'infra.bootstrap.tool_a.setup_tool_a' 86 | INFO Bootstrap complete 87 | ``` 88 | 89 | If any individual bootstrapping step fails, the entire run will be aborted. 90 | 91 | ## Avoiding Redundant Actions 92 | 93 | As a project evolves, it is likely that new bootstrapping methods will be added 94 | and existing ones updated - for example a new tool version may need to be installed. 95 | This means that the project will need to be periodically re-bootstrapped, but if 96 | the project is large this could be a long operation. 97 | 98 | To reduce the amount of unnecessary compute as far as possible, two mechanisms 99 | are available to skip individual steps. 100 | 101 | ### Check Point File 102 | 103 | The simplest mechanism is to define a check point when registering the bootstrap 104 | step - this can be a file or folder path that signals the step is out-of-date 105 | whenever it is newer than the last run of the bootstrapping step. 106 | 107 | In the example below `tools/tool_urls.json` is identified as a file that will 108 | change whenever a new tool is added or a version updated. Blockwork will invoke 109 | the bootstrapping step whenever: 110 | 111 | 1. The check point file does not exist (in case it's an output of the step); 112 | 2. The step has never been run before; 113 | 3. The last recorded run of the step is older than the last modified date of 114 | the checkpoint file. 115 | 116 | 117 | ```python linenums="1" title="infra/bootstrap/tool_a.py" 118 | from datetime import datetime 119 | from pathlib import Path 120 | 121 | from blockwork.bootstrap import Bootstrap 122 | from blockwork.context import Context 123 | 124 | @Bootstrap.register() 125 | @Bootstrap.checkpoint(Path("tools/tool_urls.json")) 126 | def setup_tool_a(context : Context, last_run : datetime) -> bool: 127 | # ...setup the tool... 128 | ``` 129 | 130 | !!!note 131 | 132 | Check point paths are always resolved relative to the root of the project 133 | work area. 134 | 135 | ### Last Run Date 136 | 137 | If check point files are too simple to work for certain bootstrapping steps, 138 | then the alternative mechanism is to test the `last_run` argument within the 139 | method. This check should be made as soon as possible to avoid unnecessary 140 | compute and should return `True` if the step is already up-to-date. 141 | 142 | The example below implements the same check as is performed by check point files, 143 | just to demonstrate how the mechanism may be used: 144 | 145 | ```python linenums="1" title="infra/bootstrap/tool_a.py" 146 | from datetime import datetime 147 | from pathlib import Path 148 | 149 | from blockwork.bootstrap import Bootstrap 150 | from blockwork.context import Context 151 | 152 | @Bootstrap.register() 153 | def setup_tool_a(context : Context, last_run : datetime) -> bool: 154 | urls_file = context.host_root / "tools" / "tool_urls.json" 155 | # If the URL file hasn't changed, return True to signal we're up-to-date 156 | if datetime.fromtimestamp(urls_file.stat().st_mtime) <= last_run: 157 | return True 158 | # ...other stuff to setup the tool... 159 | # Return False to signal that we were NOT up-to-date prior to running 160 | return False 161 | ``` 162 | -------------------------------------------------------------------------------- /blockwork/config/scheduler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, Blockwork, github.com/intuity/blockwork 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from collections import defaultdict 15 | from collections.abc import Hashable, Iterable 16 | from typing import Generic, TypeVar 17 | 18 | from ordered_set import OrderedSet as OSet 19 | 20 | 21 | class SchedulingError(RuntimeError): 22 | "Base class for scheduling errors" 23 | 24 | 25 | class CyclicError(SchedulingError): 26 | "Graph contains a cycle" 27 | 28 | 29 | _Schedulable = TypeVar("_Schedulable", bound=Hashable) 30 | 31 | 32 | class Scheduler(Generic[_Schedulable]): 33 | """ 34 | Generic scheduler for a directed acyclic graphs. 35 | 36 | Usage example:: 37 | 38 | dependant_map = {"x": OSet(("y", )), "y": OSet(("z", ))} 39 | scheduler = Schedular(dependant_map) 40 | while scheduler.incomplete: 41 | for item in scheduler.schedulable: 42 | scheduler.schedule(item) 43 | ...run the step related to the item... 44 | scheduler.finish(item) 45 | """ 46 | 47 | def __init__( 48 | self, 49 | dependency_map: dict[_Schedulable, OSet[_Schedulable]], 50 | targets: Iterable[_Schedulable] | None = None, 51 | reverse: bool = False, 52 | ): 53 | """ 54 | :param dependency_map: A map between items and the items that they depend on. 55 | This must be dense, containing empty values for items 56 | with no dependencies. 57 | :param targets: The target items, only (recursive) dependencies of these items 58 | will be scheduled. If None, all items will be scheduled. 59 | :param reverse: Schedule in reverse order - as if the dependency map is flipped. 60 | """ 61 | if targets is None: 62 | # Assume any item in the dependency map needs to be built 63 | level_targets = OSet(dependency_map.keys()) 64 | else: 65 | level_targets = OSet(targets) 66 | 67 | # Navigate down the tree and find all items that need to 68 | # be scheduled based on the targets 69 | self._all: OSet[_Schedulable] = OSet() 70 | count = 0 71 | while True: 72 | self._all |= level_targets 73 | if count == (count := len(self._all)): 74 | break 75 | next_level_targets = OSet() 76 | for level_target in level_targets: 77 | next_level_targets |= dependency_map.get(level_target, OSet()) 78 | level_targets = next_level_targets 79 | if not level_targets: 80 | break 81 | 82 | # Iterate over the dependency map and prune it to the items that 83 | # we need to schedule for the targets 84 | self._dependent_map: dict[_Schedulable, OSet[_Schedulable]] = defaultdict(OSet) 85 | self._dependency_map: dict[_Schedulable, OSet[_Schedulable]] = defaultdict(OSet) 86 | for dependant, dependencies in dependency_map.items(): 87 | if dependant not in self._all: 88 | continue 89 | if dependencies: 90 | self._dependency_map[dependant] |= dependencies 91 | for dependency in dependencies: 92 | self._dependent_map[dependency].add(dependant) 93 | 94 | if reverse: 95 | self._dependency_map, self._dependent_map = ( 96 | self._dependent_map, 97 | self._dependency_map, 98 | ) 99 | 100 | self._remaining = OSet(self._all) 101 | self._unscheduled = OSet(self._all) 102 | self._scheduled: OSet[_Schedulable] = OSet() 103 | self._complete: OSet[_Schedulable] = OSet() 104 | 105 | @property 106 | def leaves(self) -> OSet[_Schedulable]: 107 | """ 108 | Get leaf items which don't depend on anything else. 109 | Note: dependencies are dropped as items finish so this 110 | will change as the scheduler runs. 111 | """ 112 | dependents = OSet(self._dependency_map.keys()) 113 | return self._remaining - dependents 114 | 115 | @property 116 | def schedulable(self) -> OSet[_Schedulable]: 117 | "Get schedulable items (leaves which haven't been scheduled)" 118 | leaves = self.leaves 119 | if not leaves and not self._scheduled and self.incomplete: 120 | # Detects cycles in the graph 121 | raise CyclicError(f"{self._dependency_map}") 122 | return leaves - self._scheduled 123 | 124 | @property 125 | def blocked(self) -> OSet[_Schedulable]: 126 | "Get non-leaf items which depend on something else" 127 | return self._unscheduled - self.leaves 128 | 129 | @property 130 | def unscheduled(self) -> OSet[_Schedulable]: 131 | "Get any items that haven't been scheduled yet" 132 | return OSet(self._unscheduled) 133 | 134 | @property 135 | def scheduled(self) -> OSet[_Schedulable]: 136 | "Get any items that have been scheduled, but are not complete" 137 | return OSet(self._scheduled) 138 | 139 | @property 140 | def incomplete(self) -> OSet[_Schedulable]: 141 | "Get any items that are not complete" 142 | return self._unscheduled | self._scheduled 143 | 144 | @property 145 | def complete(self) -> OSet[_Schedulable]: 146 | "Get any items that are complete" 147 | return OSet(self._complete) 148 | 149 | def schedule(self, item: _Schedulable): 150 | """ 151 | Schedule an item. This item must come from `schedulable` 152 | or bad things will happen. 153 | """ 154 | self._unscheduled.remove(item) 155 | self._scheduled.add(item) 156 | 157 | def finish(self, item: _Schedulable): 158 | """ 159 | Mark an item as being complete. This will drop it from the graph. 160 | This should only be a previously scheduled item, and must only 161 | be called once per item. 162 | """ 163 | if item in self._dependent_map: 164 | for dependent in self._dependent_map[item]: 165 | self._dependency_map[dependent].remove(item) 166 | if not self._dependency_map[dependent]: 167 | del self._dependency_map[dependent] 168 | 169 | self._remaining.remove(item) 170 | self._scheduled.remove(item) 171 | self._complete.add(item) 172 | --------------------------------------------------------------------------------