├── .eslintignore ├── .eslintrc.json ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── content.md │ ├── infrastructure.md │ └── other.md ├── PULL_REQUEST_TEMPLATE.md ├── actions │ ├── create-conda-envs │ │ └── action.yml │ └── install-dependencies │ │ └── action.yml ├── dependabot.yml └── workflows │ ├── branch_sync.yml │ ├── create-conda-envs.yml │ ├── deploy.yml │ ├── nightly.yml │ ├── shortlog.yml │ ├── test.yml │ ├── undeploy.yml │ └── update_copyright.yml ├── .gitignore ├── .mailmap ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── _config.yml ├── bin ├── autodoc-cli ├── c-locate ├── create-html-redirect ├── link-examples └── version ├── cylc └── doc │ └── __init__.py ├── doc ├── joss-paper │ ├── codemeta.json │ ├── paper.bib │ └── paper.md └── versions.js ├── pyproject.toml ├── setup.cfg ├── src ├── 7-to-8 │ ├── caveats.rst │ ├── cheat-sheet.rst │ ├── index.rst │ ├── major-changes │ │ ├── cli.rst │ │ ├── compatibility-mode.rst │ │ ├── config-changes.rst │ │ ├── continuing-c7-c8.rst │ │ ├── cylc-install.rst │ │ ├── excluding-tasks.rst │ │ ├── index.rst │ │ ├── parameters.rst │ │ ├── platforms.rst │ │ ├── play-pause-stop.rst │ │ ├── python-2-3.rst │ │ ├── remote-owner.rst │ │ ├── scheduling.rst │ │ ├── suicide-triggers.rst │ │ ├── task-job-states.rst │ │ ├── template-vars.rst │ │ └── ui.rst │ └── summary.rst ├── _static │ └── css │ │ └── custom.css ├── _templates │ ├── automodule_job_runner_handlers.rst │ ├── docstring_only.rst │ ├── layout.html │ ├── main_loop_plugin.rst │ └── versions.html ├── conf.py ├── dictionaries │ ├── proper_nouns │ └── words ├── ext │ ├── autodoc_enum_values.py │ ├── autodoc_traits │ │ ├── __init__.py │ │ └── autodoc_traits.py │ └── database_diagram.py ├── glossary.rst ├── hyperlinks.rst.include ├── img │ ├── conditional-triggers.png │ ├── cycling.png │ ├── cylc-favicon.ico │ ├── cylc-logo-colour.svg │ ├── cylc-logo-greyscale.svg │ ├── cylc-logo-white.svg │ ├── cylc-ui-dash.png │ ├── cylc-ui-tree.png │ ├── dep-eg-1.png │ ├── dep-two-cycles.svg │ ├── eg2-dynamic.png │ ├── eg2-static.png │ ├── failure-recovery.png │ ├── fam-to-fam-1.png │ ├── fam-to-fam-2.png │ ├── hub.png │ ├── initial-start-stop-final-cp.svg │ ├── installation │ │ └── conda │ │ │ ├── dashboard.png │ │ │ ├── hub.png │ │ │ ├── treeview.png │ │ │ └── workflows.png │ ├── jinja2-ensemble-graph.png │ ├── jinja2-workflow-graph.png │ ├── n-window.gif │ ├── n-window.png │ ├── new-flow-n.png │ ├── new-flow.png │ ├── niwa-colour-small.png │ ├── niwa-colour.png │ ├── no-flow-n.png │ ├── no-flow.png │ ├── param-1.png │ ├── param-2.png │ ├── params1.png │ ├── pipe-pub.png │ ├── recurrence-sections.svg │ ├── recurrence-sections2.svg │ ├── same-flow-behind.png │ ├── same-flow-n.png │ ├── same-flow-wait-n.png │ ├── same-flow-wait.png │ ├── same-flow.png │ ├── satellite.png │ ├── task-job-icons │ │ ├── README │ │ ├── README.md │ │ ├── job-failed.png │ │ ├── job-running.png │ │ ├── job-submit-failed.png │ │ ├── job-submitted.png │ │ ├── job-succeeded.png │ │ ├── task-expired.png │ │ ├── task-failed.png │ │ ├── task-isHeld.png │ │ ├── task-isQueued.png │ │ ├── task-isRunahead.png │ │ ├── task-preparing.png │ │ ├── task-running-0.png │ │ ├── task-running-100.png │ │ ├── task-running-25.png │ │ ├── task-running-50.png │ │ ├── task-running-75.png │ │ ├── task-running.png │ │ ├── task-submit-failed.png │ │ ├── task-submitted.png │ │ ├── task-succeeded.png │ │ └── task-waiting.png │ ├── task-job.png │ ├── test1.png │ ├── test2.png │ ├── test4.png │ ├── test5.png │ ├── timeline-three-0.svg │ ├── timeline-zero.svg │ ├── tui-1.png │ ├── tui-2.png │ └── tui-3.png ├── index.rst ├── installation.rst ├── lib │ └── cylc_release.py ├── plugins │ ├── cylc-rose.rst │ ├── index.rst │ ├── install │ │ └── index.rst │ ├── job-runners │ │ └── index.rst │ ├── main-loop │ │ └── index.rst │ └── xtriggers │ │ └── index.rst ├── reference │ ├── api │ │ ├── exceptions.rst │ │ ├── index.rst │ │ ├── scan.rst │ │ └── zmq.rst │ ├── architecture │ │ ├── data-flow.rst │ │ ├── img │ │ │ ├── gui-arch-multi-user.svg │ │ │ ├── gui-arch-single-user.svg │ │ │ ├── websocket-communication.png │ │ │ └── websocket-graphql-ws-protocol.png │ │ ├── index.rst │ │ └── ui-server.rst │ ├── changes.rst │ ├── changes │ │ ├── analysis_view.gif │ │ ├── cylc-completion.bash.gif │ │ ├── cylc-graph-group-by-cycle-point.png │ │ ├── cylc-graph.gif │ │ ├── cylc-gui-info-view.gif │ │ ├── cylc-set-outputs.gif │ │ ├── cylc-set.gif │ │ ├── edit-runtime-screenshot.png │ │ ├── gantt_view.png │ │ ├── gui-n-window-selector.gif │ │ ├── log-view-screenshot.png │ │ ├── time_series.png │ │ ├── tui-1.gif │ │ ├── tui-external-editor.gif │ │ ├── ui-view-selector.jpg │ │ ├── ui-workspace-tabs.gif │ │ └── vip-vr.gif │ ├── config │ │ ├── file-format.rst │ │ ├── global.rst │ │ ├── index.rst │ │ ├── sharing-access-to-workflows.rst │ │ ├── shorthand.rst │ │ ├── types.rst │ │ ├── ui-server.rst │ │ ├── workflow.rst │ │ └── writing-platform-configs.rst │ ├── dev-history-major-changes.rst │ ├── environments │ │ ├── conda.rst │ │ └── envs │ │ │ ├── cylc-all.yml │ │ │ ├── cylc-flow-anaconda.yml │ │ │ ├── cylc-flow-base.yml │ │ │ ├── cylc-flow-with-python.yml │ │ │ ├── cylc-flow.yml │ │ │ ├── cylc-rose.yml │ │ │ ├── cylc-uiserver-with-traefik-proxy.yml │ │ │ ├── cylc-uiserver-without-jupyterhub.yml │ │ │ └── cylc-uiserver.yml │ ├── index.rst │ └── job-script-vars │ │ ├── index.rst │ │ └── var-list.txt ├── tutorial │ ├── furthertopics │ │ ├── broadcast.rst │ │ ├── clock-triggered-tasks.rst │ │ ├── family-triggers.rst │ │ ├── index.rst │ │ ├── inheritance.rst │ │ ├── message-triggers.rst │ │ ├── queues.rst │ │ └── retries.rst │ ├── img │ │ ├── cylc-graph-reversible.svg │ │ ├── cylc-graph.png │ │ ├── cylc-gui-scan-view.png │ │ ├── cylc-gui-table-view.png │ │ ├── cylc-gui-tree-view.png │ │ ├── cylc-gui-views-button.png │ │ ├── cylc-tools.png │ │ └── iso8601-dates.svg │ ├── index.rst │ ├── introduction.rst │ ├── runtime │ │ ├── configuration-consolidation │ │ │ ├── families.rst │ │ │ ├── index.rst │ │ │ ├── jinja2.rst │ │ │ └── parameters.rst │ │ ├── index.rst │ │ ├── introduction.rst │ │ ├── runtime-configuration.rst │ │ └── summary.rst │ └── scheduling │ │ ├── datetime-cycling.rst │ │ ├── further-scheduling.rst │ │ ├── graphing.rst │ │ ├── index.rst │ │ └── integer-cycling.rst ├── user-guide │ ├── cheat-sheet.rst │ ├── examples │ │ └── index.rst │ ├── index.rst │ ├── installing-workflows.rst │ ├── interventions │ │ ├── edit-a-tasks-configuration.gui.gif │ │ ├── edit-the-workflow-configuration.tui.gif │ │ ├── index.rst │ │ ├── re-run-a-task.gui.gif │ │ ├── re-run-a-task.tui.gif │ │ ├── re-run-all-failed-tasks.gui.gif │ │ ├── re-run-multiple-tasks.gui.gif │ │ ├── remove.gif │ │ ├── set-a-switch-task.gui.gif │ │ ├── set-task-outputs.gif │ │ ├── set-task-outputs.gui.gif │ │ ├── set-task-outputs.tui.gif │ │ ├── set-task-prerequisites.gui.gif │ │ ├── skip-cycle.gui.gif │ │ └── trigger-while-paused.gif │ ├── introduction.rst │ ├── removing-workflows.rst │ ├── running-workflows │ │ ├── advanced.rst │ │ ├── authentication-files.rst │ │ ├── dynamic-behaviour.rst │ │ ├── index.rst │ │ ├── reflow.rst │ │ ├── retrying-tasks.rst │ │ ├── scheduler-log-files.rst │ │ ├── scheduler-start-up.rst │ │ ├── tasks-jobs-ui.rst │ │ ├── tracking-task-state.rst │ │ ├── workflow-completion.rst │ │ ├── workflow-databases.rst │ │ └── workflow-run-modes.rst │ ├── task-implementation │ │ ├── index.rst │ │ ├── job-scripts.rst │ │ ├── job-submission.rst │ │ ├── skip-mode.rst │ │ └── ssh-job-management.rst │ ├── troubleshooting.rst │ └── writing-workflows │ │ ├── configuration.rst │ │ ├── external-triggers.rst │ │ ├── index.rst │ │ ├── jinja2.rst │ │ ├── parameterized-tasks.rst │ │ ├── runtime.rst │ │ ├── scheduler.rst │ │ ├── scheduling.rst │ │ └── suicide-triggers.rst ├── workflow-design-guide │ ├── efficiency.rst │ ├── general-principles.rst │ ├── index.rst │ ├── portable-workflows.rst │ └── style-guide.rst └── workflows │ ├── integer-pipeline │ └── flow.cylc │ ├── jinja2 │ ├── cities │ │ └── flow.cylc │ └── defaults │ │ └── flow.cylc │ ├── message-triggers │ └── flow.cylc │ ├── queues │ └── flow.cylc │ ├── satellite │ └── ext-trigger │ │ └── flow.cylc │ └── xtrigger │ ├── sequential │ └── flow.cylc │ ├── workflow_state │ ├── downstream │ │ └── flow.cylc │ └── upstream │ │ └── flow.cylc │ └── xrandom │ └── flow.cylc └── tox.ini /.eslintignore: -------------------------------------------------------------------------------- 1 | doc/[0-9]*.* 2 | doc/stable 3 | doc/latest 4 | doc/nightly 5 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "eslint:recommended", 3 | "parserOptions": { 4 | "ecmaVersion": 6 5 | }, 6 | "env": { 7 | "browser": true, 8 | "es6": true, 9 | "jquery": true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Issues with the output, build system or infrastructure 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/content.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Content 3 | about: Request something to be documented 4 | title: '' 5 | labels: content 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/infrastructure.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Infrastructure 3 | about: Changes to the build system, stack, Sphinx extensions, deployment, etc 4 | title: '' 5 | labels: infrastructure 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/other.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Other 3 | about: Anything else 4 | title: '' 5 | labels: help wanted 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **Requirements check-list** 2 | - [ ] I have read `CONTRIBUTING.md` and added my name as a Code Contributor. 3 | -------------------------------------------------------------------------------- /.github/actions/create-conda-envs/action.yml: -------------------------------------------------------------------------------- 1 | name: Build Conda Envs 2 | description: Create Conda environments from documented env files. 3 | 4 | inputs: 5 | python_version: 6 | description: Version to install. 7 | required: true 8 | cylc_flow_version: 9 | description: Version to install. 10 | required: false 11 | working_directory: 12 | description: Path to the cylc-doc repo for this run, if it isn't the default working directory. 13 | required: false 14 | default: '.' 15 | 16 | runs: 17 | using: composite 18 | steps: 19 | - shell: bash 20 | working-directory: ${{ inputs.working_directory }} 21 | run: | 22 | # get cylc version... 23 | if [[ -z '${{ inputs.cylc_flow_version }}' ]]; then 24 | # ... from cylc library 25 | CYLC_RELEASE="$(python ./src/lib/cylc_release.py)" 26 | else 27 | # ... from GitHub action input 28 | CYLC_RELEASE="${{ inputs.cylc_flow_version }}" 29 | fi 30 | echo "CYLC_RELEASE=$CYLC_RELEASE" >> "${GITHUB_ENV}" 31 | 32 | - shell: bash 33 | working-directory: ${{ inputs.working_directory }} 34 | run: | 35 | # use mamba (much faster) 36 | # use this when released: https://github.com/mamba-org/setup-mamba 37 | conda install -n base -c conda-forge -y mamba 38 | 39 | - shell: bash 40 | working-directory: ${{ inputs.working_directory }} 41 | run: | 42 | for env_file in $(echo src/reference/environments/envs/*.yml); do 43 | echo "# creating env from: ${env_file}" 44 | new_env_file="$(sed 's/\(.*\)\.yml/\1-2.yml/' <<< "${env_file}")" 45 | # substitute the python and cylc-flow versions in the env files 46 | # TODO: add explicit python dep 47 | sed \ 48 | -e 's/python =.*/python =${{ inputs.python_version }}/' \ 49 | -e "s/|version|/${CYLC_RELEASE}/" \ 50 | "${env_file}" \ 51 | > "${new_env_file}" 52 | # create the env 53 | conda run -n base mamba env create -f "${new_env_file}" -n cylc-env 54 | # test the env 55 | conda run -n cylc-env cylc version 56 | conda env remove -n cylc-env 57 | done 58 | -------------------------------------------------------------------------------- /.github/actions/install-dependencies/action.yml: -------------------------------------------------------------------------------- 1 | name: Install dependencies 2 | description: Install dependencies and setup + activate conda env 3 | 4 | runs: 5 | using: composite 6 | steps: 7 | - shell: bash 8 | run: | 9 | echo "[command]sudo apt-get update" 10 | sudo apt-get update 11 | 12 | # Explanation of these dependencies: 13 | # - https://github.com/ImageMagick/ImageMagick/issues/374#issuecomment-279252866 14 | # - aspell-en for spellchecking 15 | - shell: bash 16 | run: | 17 | echo "[command]sudo apt-get install -y ..." 18 | sudo apt-get install -y \ 19 | librsvg2-bin \ 20 | ghostscript \ 21 | pkg-config \ 22 | libgraphviz-dev \ 23 | gsfonts \ 24 | libfreetype6-dev \ 25 | libfontconfig1-dev \ 26 | aspell-en 27 | 28 | - uses: conda-incubator/setup-miniconda@3b0f2504dd76ef23b6d31f291f4913fb60ab5ff3 # v2.2.0 29 | with: 30 | channels: conda-forge 31 | python-version: ${{ matrix.python-version || '' }} 32 | miniforge-version: latest 33 | 34 | # use login shell for conda activation 35 | - shell: bash -leo pipefail {0} 36 | run: | 37 | echo "CONDA_PREFIX: ${CONDA_PREFIX}" 38 | conda install -y graphviz 39 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" 11 | labels: 12 | - 'dependencies' 13 | - 'infrastructure' 14 | -------------------------------------------------------------------------------- /.github/workflows/branch_sync.yml: -------------------------------------------------------------------------------- 1 | name: Sync PR 2 | 3 | on: 4 | push: 5 | branches: 6 | - '8.*.x' 7 | schedule: 8 | - cron: '11 11 * * 0' # every Sunday at 11:11 9 | workflow_dispatch: 10 | inputs: 11 | head_branch: 12 | description: Branch to merge into master 13 | required: true 14 | 15 | jobs: 16 | sync: 17 | uses: cylc/release-actions/.github/workflows/branch-sync.yml@v1 18 | with: 19 | head_branch: ${{ inputs.head_branch }} 20 | secrets: inherit 21 | -------------------------------------------------------------------------------- /.github/workflows/create-conda-envs.yml: -------------------------------------------------------------------------------- 1 | name: create conda environments 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '.github/workflows/create-conda-envs.yml' 7 | - '.github/actions/create-conda-envs/*' 8 | - 'src/reference/environments/env/*.ya?ml' 9 | workflow_dispatch: 10 | inputs: 11 | cylc_flow_version: 12 | description: 'Version to install.' 13 | required: true 14 | 15 | concurrency: 16 | group: ${{ github.workflow }}-${{ github.ref }} 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | test_conda_install: 21 | runs-on: ubuntu-latest 22 | timeout-minutes: 30 23 | strategy: 24 | matrix: 25 | python: [3.7, 3.8, 3.9] 26 | steps: 27 | - name: checkout cylc-doc 28 | uses: actions/checkout@v4 29 | 30 | - name: configure python 31 | uses: actions/setup-python@v5 32 | with: 33 | python-version: '3.9' 34 | 35 | - name: install setuptools 36 | run: | 37 | # pick up deps for version processing 38 | pip install 'setuptools>=50' 39 | 40 | - name: checkout cylc-flow init file 41 | run: | 42 | # needed to extract the cylc-flow version 43 | LIB_PATH='src/lib/cylc/flow' 44 | mkdir -p "$LIB_PATH" 45 | curl \ 46 | https://raw.githubusercontent.com/cylc/cylc-flow/master/cylc/flow/__init__.py \ 47 | > "$LIB_PATH/__init__.py" 48 | 49 | - name: create conda envs 50 | uses: ./.github/actions/create-conda-envs 51 | with: 52 | python_version: ${{ matrix.python }} 53 | cylc_flow_version: ${{ github.event.inputs.cylc_flow_version }} 54 | -------------------------------------------------------------------------------- /.github/workflows/shortlog.yml: -------------------------------------------------------------------------------- 1 | name: check contributor list 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | paths: 7 | - 'CONTRIBUTING.md' 8 | - '.mailmap' 9 | - '.github/workflows/shortlog.yml' 10 | 11 | jobs: 12 | shortlog: 13 | runs-on: 'ubuntu-latest' 14 | timeout-minutes: 10 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 # need to fetch all commits to check contributors 20 | 21 | - name: Check CONTRIBUTING.md 22 | uses: cylc/release-actions/check-shortlog@v1 23 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | workflow_dispatch: 8 | inputs: 9 | cylc-flow-tag: 10 | description: 'cylc-flow ref' 11 | required: true 12 | cylc-rose-tag: 13 | description: 'cylc-rose ref' 14 | required: true 15 | cylc-uis-tag: 16 | description: 'cylc-uis ref' 17 | required: true 18 | metomi-rose-tag: 19 | description: 'metomi-rose ref (for cylc-rose install)' 20 | required: true 21 | 22 | concurrency: 23 | group: ${{ github.workflow }}-${{ github.ref }} 24 | cancel-in-progress: true 25 | 26 | jobs: 27 | test: 28 | runs-on: ubuntu-latest 29 | timeout-minutes: 10 30 | defaults: 31 | run: 32 | # use login shell for conda activation 33 | shell: bash -leo pipefail {0} 34 | steps: 35 | - name: configure python 36 | uses: actions/setup-python@v5 37 | with: 38 | python-version: '3.9' 39 | 40 | - name: configure node 41 | uses: actions/setup-node@v4 42 | with: 43 | node-version: '16' 44 | 45 | - name: checkout cylc-doc 46 | uses: actions/checkout@v4 47 | 48 | - name: install dependencies 49 | uses: ./.github/actions/install-dependencies 50 | 51 | - name: install cylc-doc 52 | run: pip install .[all] 53 | 54 | - name: install libs 55 | uses: cylc/release-actions/install-cylc-components@v1 56 | with: 57 | cylc_flow: true 58 | cylc_flow_opts: 'main_loop-log_data_store,main_loop-log_db,main_loop-log_main_loop,main_loop-log_memory' 59 | cylc_uiserver: true 60 | cylc_uiserver_opts: '' 61 | cylc_rose: true 62 | cylc_rose_opts: '' 63 | metomi_rose: true 64 | metomi_rose_opts: '' 65 | 66 | - name: lint 67 | run: | 68 | flake8 69 | npx eslint@v8 . 70 | 71 | - name: (debug only) list language dictionaries 72 | if: runner.debug 73 | shell: python 74 | env: 75 | PYENCHANT_VERBOSE_FIND: true 76 | run: | 77 | import enchant 78 | print(enchant.list_dicts()) 79 | 80 | - name: build & test 81 | run: | 82 | make html spelling linkcheck \ 83 | SPHINXOPTS='-Wn --keep-going' FORCE_COLOR=true 84 | 85 | - name: debug sphinx failure 86 | if: failure() 87 | run: | 88 | cat /tmp/sphinx-err* || true # sphinx traceback 89 | 90 | - name: debug - pip list 91 | if: failure() 92 | run: pip list 93 | -------------------------------------------------------------------------------- /.github/workflows/undeploy.yml: -------------------------------------------------------------------------------- 1 | # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. 2 | # Copyright (C) NIWA & British Crown (Met Office) & Contributors. 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | # Remove a deployed version of the documentation. 18 | 19 | # Note: All changes made to the gh-pages branch are non-destructive 20 | # (i.e. no force pushing) so all changes can be undone. 21 | 22 | name: undeploy 23 | 24 | on: 25 | workflow_dispatch: 26 | inputs: 27 | tag: 28 | description: 'Tag present on both cylc-flow and cylc-doc' 29 | required: true 30 | 31 | concurrency: 32 | group: ${{ github.workflow }}-${{ github.ref }} 33 | 34 | jobs: 35 | undeploy: 36 | runs-on: ubuntu-latest 37 | timeout-minutes: 10 38 | env: 39 | TAG: ${{ inputs.tag }} 40 | steps: 41 | - name: configure python 42 | uses: actions/setup-python@v5 43 | with: 44 | python-version: '3.9' 45 | 46 | - name: configure git 47 | uses: cylc/release-actions/configure-git@v1 48 | 49 | - name: checkout cylc-doc 50 | uses: actions/checkout@v4 51 | with: 52 | path: docs 53 | 54 | - name: checkout gh-pages 55 | uses: actions/checkout@v4 56 | with: 57 | ref: gh-pages 58 | path: gh-pages 59 | 60 | - name: check tag 61 | run: | 62 | for ver in stable latest; do 63 | ver_link="$(readlink gh-pages/${ver})" 64 | if [[ -z "$ver_link" ]] 65 | then 66 | echo "::error::Could not determine the '"$ver"' version" 67 | exit 1 68 | elif [[ "$ver" == "$TAG" ]] 69 | then 70 | echo "::error::Cannot delete the '"$ver"' version" 71 | exit 1 72 | fi 73 | done 74 | 75 | - name: install gh-pages 76 | working-directory: docs 77 | run: | 78 | rm -r doc 79 | ln -s ../gh-pages doc 80 | 81 | - name: remove version 82 | working-directory: gh-pages 83 | run: | 84 | git rm -r "$TAG" 85 | 86 | - name: update version file 87 | working-directory: docs 88 | run: | 89 | bin/version write > doc/versions.json 90 | 91 | - name: push changes 92 | working-directory: gh-pages 93 | run: | 94 | git add -u 95 | git commit -m "remove: ${TAG}" 96 | git push origin HEAD 97 | -------------------------------------------------------------------------------- /.github/workflows/update_copyright.yml: -------------------------------------------------------------------------------- 1 | # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. 2 | # Copyright (C) NIWA & British Crown (Met Office) & Contributors. 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | name: update copyright year 18 | 19 | on: 20 | workflow_dispatch: 21 | schedule: 22 | - cron: '40 4 1 1 *' # Every Jan 1st @ 04:40 UTC 23 | 24 | jobs: 25 | update-copyright: 26 | if: github.repository_owner == 'cylc' 27 | runs-on: ubuntu-latest 28 | timeout-minutes: 10 29 | steps: 30 | 31 | - name: Checkout repo 32 | uses: actions/checkout@v4 33 | 34 | - name: Configure git 35 | uses: cylc/release-actions/configure-git@v1 36 | 37 | - name: Checkout PR branch 38 | uses: cylc/release-actions/checkout-copyright-branch@v1 39 | 40 | - name: Update copyright year 41 | env: 42 | README_FILE: 'README.md' 43 | CONF_FILE: 'src/conf.py' 44 | run: | 45 | pattern="().*(<\/span>)" 46 | sed -i -E "s/${pattern}/\1${YEAR}\2/" "$README_FILE" 47 | 48 | pattern="^(__copyright_year ?= ?['\"]?)[0-9]*(.*)$" 49 | sed -i -E "s/${pattern}/\1${YEAR}\2/" "$CONF_FILE" 50 | 51 | - name: Commit & push 52 | run : | 53 | git commit -a -m "Update copyright year" -m "Workflow: ${{ github.workflow }}, run: ${{ github.run_number }}" 54 | git push origin "$BRANCH_NAME" 55 | 56 | - name: Create pull request 57 | uses: cylc/release-actions/create-pr@v1 58 | env: 59 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 60 | with: 61 | head: ${{ env.BRANCH_NAME }} 62 | title: 'Auto PR: update copyright year' 63 | body: 'Happy new year!' 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # python stuffs 2 | *.py[cod] 3 | /__pycache__/ 4 | 5 | # distutils stuffs 6 | build 7 | dist 8 | MANIFEST 9 | *.egg-info 10 | .eggs 11 | 12 | # build stuffs 13 | doc/* 14 | !doc/joss-paper 15 | 16 | # derived files 17 | src/appendices/command-ref.rst 18 | 19 | # editor stuff 20 | *.swp 21 | *~ 22 | .vscode 23 | 24 | # auto-generated documentation 25 | src/plugins/main-loop/built-in 26 | src/plugins/install/built-in 27 | src/plugins/xtriggers/built-in 28 | src/user-guide/task-implementation/job-runner-handlers 29 | src/dictionaries/sentence_case 30 | src/user-guide/examples/* 31 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | # FORMAT: 2 | # Omit commit-name or commit-email if same as proper 3 | 4 | Bruno Kinoshita Bruno P. Kinoshita 5 | Bruno Kinoshita Bruno P. Kinoshita 6 | David Matthews dpmatthews 7 | Mel Hall Melanie Hall <37735232+datamel@users.noreply.github.com> 8 | Mel Hall <37735232+datamel@users.noreply.github.com> 9 | Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> 10 | Tim Pillinger wxtim <26465611+wxtim@users.noreply.github.com> 11 | Tim Pillinger <26465611+wxtim@users.noreply.github.com> 12 | github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> 13 | Hilary Oliver Hilary James Oliver 14 | Hilary James Oliver 15 | Hilary Oliver 16 | Sadie Bartholomew Sadie L. Bartholomew <30274190+sadielbartholomew@users.noreply.github.com> 17 | Tomek Trzeciak 18 | Tomek Trzeciak TomekTrzeciak 19 | Thomas Coleman <15375218+ColemanTom@users.noreply.github.com> 20 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Cylc Doc: How to Contribute 2 | 3 | ## Report Bugs 4 | 5 | Report bugs by opening an issue [on 6 | Github](https://github.com/cylc/cylc-doc/issues). Give the version 7 | affected by the bug (you should test the latest release if possible) and a 8 | recipe to reproduce the problem. 9 | 10 | ## Request Enhancements 11 | 12 | Request enhancements by opening an issue at [Github 13 | ](https://github.com/cylc/cylc-doc/issues). Describe your use case in 14 | detail. 15 | 16 | ## Contribute Code 17 | 18 | All contributions to project are made via Pull Requests against the *master* 19 | branch of [cylc/cylc-doc](https://github.com/cylc/cylc-doc). 20 | Non-trivial developments must be discussed and agreed in advance in a 21 | [Issue](https://github.com/cylc/cylc-doc/issues) as the team may not 22 | be able to consider large changes that appear out of the blue. New 23 | contributors should add their details to the [Code Contributors 24 | ](#code-contributors) section of this file as part of their first Pull 25 | Request, and reviewers are responsible for checking this before merging the 26 | new branch into *master*. 27 | 28 | ## Code Contributors 29 | 30 | The following people have contributed to this code under the terms of 31 | the Contributor Licence Agreement and Certificate of Origin detailed 32 | below. 33 | 34 | (_except for the parenthesised names, which represent contributions 35 | from outside of NIWA and the Met Office that predate the explicit introduction 36 | of this Agreement in July 2018; they must be un-parenthesised in future pull 37 | requests_). 38 | 39 | 40 | - Hilary Oliver 41 | - Matt Shin 42 | - Oliver Sanders 43 | - Sadie Bartholomew 44 | - Bruno Kinoshita 45 | - Tim Pillinger 46 | - Mel Hall 47 | - Ronnie Dutta 48 | - Damien François 49 | - David Matthews 50 | - (Neil Swart) 51 | - Tomek Trzeciak 52 | - Douglas Lowe 53 | - Alex Pletzer 54 | - Yinjing Lin 55 | - David Sutherland 56 | - Thomas Coleman 57 | - Scott Wales 58 | - Elliot Fontaine 59 | 60 | 61 | (All contributors are identifiable with email addresses in the git version 62 | control logs or otherwise.) 63 | 64 | ## Contributor Licence Agreement and Certificate of Origin 65 | 66 | By making a contribution to this project, I certify that: 67 | 68 | (a) The contribution was created in whole or in part by me and I have 69 | the right to submit it, either on my behalf or on behalf of my 70 | employer, under the terms and conditions as described by this file; 71 | or 72 | 73 | (b) The contribution is based upon previous work that, to the best of 74 | my knowledge, is covered under an appropriate licence and I have 75 | the right or permission from the copyright owner under that licence 76 | to submit that work with modifications, whether created in whole or 77 | in part by me, under the terms and conditions as described by 78 | this file; or 79 | 80 | (c) The contribution was provided directly to me by some other person 81 | who certified (a) or (b) and I have not modified it. 82 | 83 | (d) I understand and agree that this project and the contribution 84 | are public and that a record of the contribution (including my 85 | name and email address) is retained for the full term of 86 | the copyright and may be redistributed consistent with this project 87 | or the licence(s) involved. 88 | 89 | (e) I, or my employer, grant to NIWA and all recipients of 90 | this software a perpetual, worldwide, non-exclusive, no-charge, 91 | royalty-free, irrevocable copyright licence to reproduce, modify, 92 | prepare derivative works of, publicly display, publicly perform, 93 | sub-licence, and distribute this contribution and such modifications 94 | and derivative works consistent with this project or the licence(s) 95 | involved or other appropriate open source licence(s) specified by 96 | the project and approved by the 97 | [Open Source Initiative (OSI)](http://www.opensource.org/). 98 | 99 | (f) If I become aware of anything that would make any of the above 100 | inaccurate, in any way, I will let NIWA know as soon as 101 | I become aware. 102 | 103 | (The Cylc Contributor Licence Agreement and Certificate of Origin is 104 | inspired that of [Rose](https://github.com/metomi/rose), which in turn was 105 | inspired by the Certificate of Origin used by Enyo and the Linux Kernel.) 106 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | 3 | # You can set these variables from the command line: 4 | # e.g. $ make html SPHINXOPTS=-W 5 | SPHINXOPTS = -n 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = src 8 | BUILDDIR = doc/$(shell cylc version) 9 | STABLE = true # Makes this the Stable and default build 10 | LATEST = true # Makes this the Latest build 11 | # The Stable/default build is what you get when navigating to the docs root dir 12 | # For pre-release (except nightly) builds, set STABLE=false LATEST=true 13 | 14 | # Put it first so that "make" without argument is like "make help". 15 | help: 16 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 17 | 18 | clean: 19 | @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | # remove auto-generated content 21 | rm -rf src/plugins/main-loop/built-in 22 | rm -rf src/plugins/install/built-in 23 | rm -rf src/plugins/xtriggers/built-in 24 | rm -rf src/user-guide/task-implementation/job-runner-handlers 25 | 26 | watch: clean 27 | # rebuild incrementally whenever documentation files are changed 28 | sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" \ 29 | --ignore='**plugins/main-loop/built-in/*.rst' \ 30 | --ignore='**plugins/install/built-in/*.rst' \ 31 | --ignore='**/job-runner-handlers/*.rst' \ 32 | --ignore='**/dictionaries/**' \ 33 | --open-browser 34 | 35 | watch-cylc: 36 | # rebuild non-incrementally (SLOW) whenever docs files OR Cylc source 37 | # code files are changed 38 | sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" \ 39 | -a \ 40 | --pre-build='make clean' \ 41 | --ignore='**plugins/main-loop/built-in/*.rst' \ 42 | --ignore='**plugins/install/built-in/*.rst' \ 43 | --ignore='**/job-runner-handlers/*.rst' \ 44 | --ignore='**/dictionaries/**' \ 45 | --watch="$(shell bin/c-locate cylc.flow)" \ 46 | --watch="$(shell bin/c-locate cylc.uiserver)" \ 47 | --watch="$(shell bin/c-locate cylc.rose)" \ 48 | --open-browser 49 | 50 | cleanall: 51 | (cd doc; echo [0-9]*.*) 52 | rm -rI doc/[0-9]*.* 53 | 54 | .PHONY: help clean watch Makefile .EXPORT_ALL_VARIABLES 55 | 56 | # Catch-all target: route all unknown targets to Sphinx using the new 57 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 58 | # NOTE: EXPORT_ALL_VARIABLES exports make vars as env vars 59 | %: Makefile .EXPORT_ALL_VARIABLES 60 | bin/link-examples 61 | # build documentation 62 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 63 | # write out dict of available versions and formats 64 | bin/version write > doc/versions.json 65 | # Redirect doc//index.html -> doc//html/index.html 66 | bin/create-html-redirect "html/index.html" "$(BUILDDIR)/index.html" 67 | ifeq ($(STABLE),true) # makefile conditionals in recipe must be unindented 68 | # Link e.g. doc/stable/ -> doc/7.0/ 69 | rm "$(BUILDDIR)/../stable" || true 70 | ln -sr "$(BUILDDIR)" "$(BUILDDIR)/../stable" 71 | # Redirect doc/index.html -> doc/stable/index.html 72 | bin/create-html-redirect "stable/index.html" "$(BUILDDIR)/../index.html" 73 | endif 74 | ifeq ($(LATEST),true) 75 | # Link e.g. doc/latest/ -> doc/8.0a1/ 76 | rm "$(BUILDDIR)/../latest" || true 77 | ln -sr "$(BUILDDIR)" "$(BUILDDIR)/../latest" 78 | endif 79 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | # This file configures the Jekyll static web builder (https://jekyllrb.com/) as 2 | # used by Github Pages. 3 | 4 | # Jekyll treats directories beginning with underscores as having special meaning 5 | # and omits them from the build. This can be overridden by manually including 6 | # directories using "include". 7 | # https://help.github.com/articles/files-that-start-with-an-underscore-are-missing/ 8 | # NOTE: Only the directory name should be specified not the full path. 9 | include: [ 10 | "_downloads", 11 | "_images", 12 | "_modules", 13 | "_sources", 14 | "_static" 15 | ] 16 | -------------------------------------------------------------------------------- /bin/autodoc-cli: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. 3 | # Copyright (C) NIWA & British Crown (Met Office) & Contributors. 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | # Create appendices/command-ref.rst for inclusion in HTML doc. 19 | 20 | # All paths relative to 'doc/src/custom/' directory: 21 | COMMAND_REF_FILE="$(dirname "$0")/../src/appendices/command-ref.rst" 22 | CYLC="cylc" 23 | 24 | cat > "$COMMAND_REF_FILE" <> "$COMMAND_REF_FILE" <> "$COMMAND_REF_FILE" <> "$COMMAND_REF_FILE" <. 17 | 18 | """Locate Python repository installations.""" 19 | 20 | from pathlib import Path 21 | import sys 22 | 23 | 24 | def main(arg): 25 | if arg == 'cylc.flow': 26 | import cylc.flow 27 | print(Path(cylc.flow.__file__).parent) 28 | elif arg == 'cylc.uiserver': 29 | import cylc.uiserver 30 | print(Path(cylc.uiserver.__file__).parent) 31 | elif arg == 'cylc.rose': 32 | import cylc.rose 33 | print(Path(cylc.rose.__file__).parent) 34 | else: 35 | raise ValueError(f'Unsupported arg: {arg}') 36 | 37 | 38 | if __name__ == '__main__': 39 | main(sys.argv[1]) 40 | -------------------------------------------------------------------------------- /bin/create-html-redirect: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (C) NIWA & British Crown (Met Office) & Contributors. 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | set -eu 18 | 19 | TARGET="$1" # location to redirect to (relative path from $DEST) 20 | DEST="$2" # location to redirect from 21 | # i.e. redirect from $2 to Path($2).parent.joinpath($1) 22 | 23 | mkdir -p "$(dirname "$DEST")" 24 | 25 | cat >"$DEST" << __HTML__ 26 | 27 | 28 | 29 | Cylc Documentation 30 | 31 | 32 | 33 |

If not automatically redirected, please click 34 | Cylc Documentation.

35 | 36 | 37 | __HTML__ 38 | -------------------------------------------------------------------------------- /bin/link-examples: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | # rm cylc-flow-examples || true 6 | examples="$(python -c 'import cylc.flow; from pathlib import Path; print(Path(cylc.flow.__file__).parent)')/etc/examples" 7 | for example in $(echo "${examples}/"*); do 8 | if [[ -d "${example}" ]]; then 9 | target="src/user-guide/examples/$(basename "${example}")" 10 | rm "${target}" || true 11 | ln -s "${example}" "${target}" 12 | fi 13 | done 14 | 15 | -------------------------------------------------------------------------------- /cylc/doc/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.0.1' 2 | -------------------------------------------------------------------------------- /doc/joss-paper/codemeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": "https://raw.githubusercontent.com/codemeta/codemeta/master/codemeta.jsonld", 3 | "@type": "Code", 4 | "author": [ 5 | { 6 | "@id": "https://orcid.org/0000-0002-5715-5279", 7 | "@type": "Person", 8 | "email": "hilary.oliver@niwa.co.nz", 9 | "name": "Hilary J Oliver", 10 | "affiliation": "NIWA" 11 | }, 12 | { 13 | "@id": "https://orcid.org/0000-0002-5904-7511", 14 | "@type": "Person", 15 | "email": "matthew.shin@metoffice.gov.uk", 16 | "name": "Matthew Shin", 17 | "affiliation": "Met Office" 18 | }, 19 | { 20 | "@id": "https://orcid.org/0000-0001-6916-7034", 21 | "@type": "Person", 22 | "email": "oliver.sanders@metoffice.gov.uk", 23 | "name": "Oliver Sanders", 24 | "affiliation": "Met Office" 25 | } 26 | ], 27 | "identifier": "https://doi.org/10.5281/zenodo.594577", 28 | "codeRepository": "https://github.com/cylc/cylc", 29 | "datePublished": "2018-05-22", 30 | "dateModified": "2018-05-22", 31 | "dateCreated": "2018-05-22", 32 | "description": "Cylc is a workflow engine for orchestrating cycling systems", 33 | "keywords": "workflow, metascheduling, scheduling, distributed, automation, cycling, forecasting, climate, weather", 34 | "license": "GPL v3.0", 35 | "title": "Cylc: a workflow engine for cycling systems", 36 | "version": "https://cylc.github.io/cylc/ v7.7.0" 37 | } 38 | -------------------------------------------------------------------------------- /doc/joss-paper/paper.bib: -------------------------------------------------------------------------------- 1 | @misc{zenodo, 2 | publisher = {Zenodo}, 3 | author = {Hilary Oliver and Matt Shin and Oliver Sanders et.al.}, 4 | doi = {10.5281/zenodo.1251084}, 5 | year = {2018}, 6 | month = {May}, 7 | day = {22}, 8 | title = {cylc/cylc: cylc-7.7.0}, 9 | } 10 | -------------------------------------------------------------------------------- /doc/joss-paper/paper.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Cylc: A Workflow Engine for Cycling Systems' 3 | authors: 4 | - affiliation: 1 5 | name: Hilary J Oliver 6 | orcid: 0000-0002-5715-5279 7 | - affiliation: 2 8 | name: Matthew Shin 9 | orcid: 0000-0002-5904-7511 10 | - affiliation: 3 11 | name: Oliver Sanders 12 | orcid: 0000-0001-6916-7034 13 | date: "10 Arpil 2018" 14 | output: 15 | html_document: default 16 | pdf_document: default 17 | bibliography: paper.bib 18 | tags: 19 | - forecasting 20 | - weather 21 | - meteorology 22 | - climate 23 | - hydrology 24 | - ocean 25 | - Python 26 | - workflow 27 | - scheduling 28 | - automation 29 | affiliations: 30 | - index: 1 31 | name: National Institute of Water and Atmospheric Research (NIWA), New Zealand 32 | - index: 2 33 | name: Met Office, UK 34 | - index: 3 35 | name: Met Office, UK 36 | --- 37 | 38 | # Summary 39 | 40 | Cylc is a workflow engine for orchestrating complex distributed *suites* of 41 | inter-dependent cycling (repeating) tasks, as well as ordinary non-cycling 42 | workflows. It has been widely adopted for weather, climate, and related 43 | forecasting applications in research and production HPC environments, and it is 44 | now part of the official software infrastructure for the Unified Model 45 | atmospheric model. Cylc is written in Python and developed primarily by NIWA 46 | (NZ) and Met Office (UK). It has strong support for large production systems, 47 | but ease of use for individuals with smaller workflow automation requirements 48 | remains a key priority. Despite its origins, Cylc is not in any way specialized 49 | to environmental forecasting. 50 | 51 | In cycling workflows tasks repeat on sequences that may represent forecast 52 | cycles, chunks of a simulation too long for a single run, steps in some 53 | multi-program iterative process (e.g. for optimizing model parameters), or 54 | datasets to be processed as they are generated or received, and so forth. 55 | Cycling in Cylc is controlled by ISO 8601 datetime recurrence expressions 56 | (e.g. for environmental forecasting), or integer recurrence expressions (e.g. 57 | for iterative processes). Dependence across cycles creates ongoing, potentially 58 | never-ending, workflows (rather than simply a succession of disconnected single 59 | workflows). Cylc is unique in its ability to manage these workflows without 60 | imposing a global cycle loop, so that one cycle does not have to complete 61 | before the next can start. Instead, Cylc's novel meta-scheduling algorithm runs 62 | cycles concurrently to the full extent allowed by task dependencies. So, for 63 | example, on restarting after extended downtime, a workflow that processes 64 | real-time data can clear its backlog and catch up very quickly by interleaving 65 | cycles. 66 | 67 | As a distributed system, Cylc scales sideways: each workflow is managed by its 68 | own lightweight server program. Existing scripts or programs can be used 69 | without modification in Cylc tasks: they are automatically wrapped in code to 70 | trap errors and report run status via authenticated HTTPS messages or polling. 71 | Cylc workflows are defined with a graph notation that efficiently expresses 72 | dependence between tasks; and task runtime properties (what to run, and where 73 | and how to run the job) are defined in an inheritance hierarchy for efficient 74 | sharing of common settings. Tasks can depend on the wall clock and arbitrary 75 | external events, as well as on the status of other tasks (*submitted*, 76 | *started*, *succeeded*, *failed*, etc.). Dependence between workflows is also 77 | supported: for coupled systems you can choose between a large suite that 78 | controls all tasks, and many smaller suites that depend on each other. 79 | 80 | Other features of Cylc include a comprehensive command line interface and GUI; 81 | restart from workflow state snapshots with automatic determination of the fate 82 | of orphaned jobs; support for common HPC resource managers and batch systems 83 | (PBS, Slurm, etc.) as well as background jobs; edit run (manually modify a job 84 | script at the last minute); configurable automatic retries; flexible event 85 | handling; simulation and dummy modes; support for the Jinja2 template processor 86 | in workflow definitions; internal queues to limit job submission; and conditional 87 | triggering. Major items on the development roadmap include a web GUI and port 88 | to Python 3. 89 | 90 | The Cylc source code is available on GitHub 91 | [http://cylc.github.io/cylc/](http://cylc.github.io/cylc/) and is archived to 92 | Zenodo with the linked DOI: [@zenodo] 93 | 94 | # References 95 | -------------------------------------------------------------------------------- /doc/versions.js: -------------------------------------------------------------------------------- 1 | /* global CUR_VERSION, ROOT_DIR, PAGE_NAME, CUR_FORMAT */ 2 | // these variables come from versions.html in the Sphinx theme 3 | 4 | $('#versions-and-formats') 5 | .append( 6 | $('') 7 | .addClass('rst-current-version') 8 | .attr({'data-toggle': 'rst-current-version'}) 9 | .append( 10 | $('') 11 | .addClass('fa fa-book') 12 | .append('Versions'), 13 | $('').append('v: ' + CUR_VERSION), 14 | $('') 15 | .addClass('fa fa-caret-down') 16 | ), 17 | $('
') 18 | .addClass('rst-other-versions') 19 | .attr({'id': 'versions'}) 20 | .append( 21 | $('

') 22 | .append('Versions') 23 | ), 24 | $('
') 25 | .addClass('rst-other-versions') 26 | .attr({'id': 'formats'}) 27 | .append( 28 | $('

') 29 | .append('Formats') 30 | ) 31 | ) 32 | 33 | // path to JSON file with versions and formats dictionary 34 | const VERSIONS_URL = ROOT_DIR + '/versions.json'; 35 | 36 | function url(version, format) { 37 | // return the URL of the curret page in the documentation 38 | var ret = ROOT_DIR + '/' + version + '/' + format + '/'; 39 | if (format === 'html') { 40 | ret += PAGE_NAME + '.html'; 41 | } else if (format === 'singlehtml') { 42 | ret += 'index.html#document-' + PAGE_NAME; 43 | } else if (format === 'epub') { 44 | ret += 'Cylc.epub'; 45 | } else if (format == 'latex') { 46 | ret += 'cylc.pdf' 47 | } else { 48 | ret += 'index.html'; 49 | } 50 | return ret 51 | } 52 | 53 | $(document).ready(function() { 54 | const version_div = $('#versions'); 55 | const format_div = $('#formats') 56 | 57 | $.ajax({ 58 | 'type': 'GET', 59 | 'url': VERSIONS_URL, 60 | dataType: 'json', 61 | success: function (versions) { 62 | // write list versions (for all formats) 63 | var vn_fmt; 64 | for (let version of Object.keys(versions).sort()) { 65 | if (versions[version].indexOf(CUR_FORMAT) === -1) { 66 | vn_fmt = 'html'; // fallback to html 67 | } else { 68 | vn_fmt = CUR_FORMAT; // link current format 69 | } 70 | version_div.append( 71 | $('') 72 | .attr({'style': 'display: inline-block'}) 73 | .attr({'href': url(version, vn_fmt)}) 74 | .css({'padding-left': '1em'}) 75 | .append(version) 76 | ); 77 | } 78 | 79 | // write formats for current version 80 | for (let format of versions[CUR_VERSION]) { 81 | format_div.append( 82 | $('') 83 | .attr({'href': url(CUR_VERSION, format)}) 84 | .css({'padding-left': '1em'}) 85 | .append(format) 86 | ); 87 | } 88 | } 89 | }); 90 | }); 91 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/pyproject.toml -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. 2 | # Copyright (C) NIWA & British Crown (Met Office) & Contributors. 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | [metadata] 18 | name = cylc-doc 19 | version = attr: cylc.doc.__version__ 20 | url = https://cylc.github.io/cylc-doc/stable/html/index.html 21 | description = Documentation for the Cylc workflow engine 22 | long_description = file: README.md 23 | long_description_content_type = text/markdown 24 | project_urls = 25 | Documentation = https://cylc.github.io/cylc-doc/stable/html/index.html 26 | Source = https://github.com/cylc/cylc-doc 27 | Tracker = https://github.com/cylc/cylc-doc/issues 28 | license = GPL 29 | platforms = any 30 | classifiers = 31 | Environment :: Console 32 | Environment :: Web Environment 33 | Intended Audience :: Developers 34 | Intended Audience :: System Administrators 35 | Intended Audience :: Science/Research 36 | License :: OSI Approved :: GNU General Public License v3 (GPLv3) 37 | Operating System :: POSIX :: Linux 38 | Programming Language :: Python 39 | Programming Language :: Python :: 3.8 40 | Programming Language :: Python :: 3.9 41 | Programming Language :: Python :: 3 :: Only 42 | Programming Language :: Python :: Implementation :: CPython 43 | Topic :: Scientific/Engineering :: Atmospheric Science 44 | 45 | [options] 46 | packages = find_namespace: 47 | include_package_data = True 48 | python_requires = >=3.8 49 | install_requires = 50 | cylc-sphinx-extensions>=1.4.1 51 | eralchemy==1.2.* 52 | hieroglyph>=2.1.0 53 | setuptools>=50 54 | sphinx>=7.1, <7.4 55 | sphinx-design>=0.5.0 56 | sphinx_rtd_theme>=1.0.0 57 | sphinxcontrib-svg2pdfconverter 58 | sphinxcontrib-spelling 59 | 60 | [options.packages.find] 61 | include = cylc* 62 | 63 | [options.extras_require] 64 | test = 65 | flake8>=5.0 66 | flake8-broken-line>=0.5.0 67 | flake8-bugbear>=22.7.0 68 | flake8-builtins>=1.5.0 69 | flake8-comprehensions>=3.10.0 70 | flake8-debugger>=4.1.0 71 | flake8-mutable>=1.2.0 72 | flake8-simplify>=0.19.0 73 | watch = 74 | sphinx-autobuild 75 | all = 76 | %(test)s 77 | %(watch)s 78 | -------------------------------------------------------------------------------- /src/7-to-8/caveats.rst: -------------------------------------------------------------------------------- 1 | Cylc |version| Caveats 2 | ====================== 3 | 4 | There are a few Cylc 7 features which do not yet have direct replacements in 5 | Cylc 8. These features will be re-implemented in future releases. 6 | 7 | 8 | Cylc Flow 9 | --------- 10 | 11 | Multiple Flows 12 | The new :term:`scheduler` can manage multiple flows in the workflow graph. 13 | Commands and options for interacting with flows are still being refined. 14 | 15 | 16 | Browser Based UI 17 | ---------------- 18 | 19 | The old "GUI" has been replaced by the new browser-based "UI". 20 | 21 | Static Graph Visualization 22 | Not yet reimplemented for Cylc 8. As an interim measure the 23 | ``cylc graph`` command can generate a basic PNG image of a workflow 24 | graph if Graphviz is installed in the Cylc environment. 25 | 26 | Multiple Selection 27 | Multiple selection is yet to be implemented, however, it is possible 28 | to issue action for multiple tasks (e.g. ``kill``) without using 29 | multiple selection: 30 | 31 | * From the UI click on a workflow/cycle/task/job. 32 | * Find the action you want to call (e.g. kill). 33 | * Click the pencil symbol next to this action. 34 | * Edit the workflows/cycles/tasks/jobs in the form and press submit. 35 | 36 | * https://github.com/cylc/cylc-ui/issues/434 37 | Installing Workflows 38 | At present there is no way to view or install 39 | :term:`source workflows ` in the UI. 40 | Rose Edit 41 | Rose Edit is awaiting reimplementation. 42 | Xtrigger Visibility 43 | Xtriggers are not yet visible in the UI. 44 | 45 | * https://github.com/cylc/cylc-ui/issues/331 46 | Documentation / Orientation Guide 47 | Some form of documentation will be provided within the UI itself. 48 | 49 | * https://github.com/cylc/cylc-ui/issues/155 50 | 51 | 52 | UI Server 53 | --------- 54 | 55 | CLI via UIS 56 | The ability to route Cylc commands via the UIS is planned for a future release 57 | 58 | * https://github.com/cylc/cylc-flow/issues/3528 59 | -------------------------------------------------------------------------------- /src/7-to-8/index.rst: -------------------------------------------------------------------------------- 1 | Cylc 8 Migration Guide 2 | ====================== 3 | 4 | **A quick guide for Cylc 7 (and Rose 2019) users upgrading to Cylc 8.** 5 | 6 | .. tip:: 7 | 8 | If you need help using or understanding Cylc 8 please post questions to the 9 | `Cylc 8 Migration category `_ 10 | on the `Cylc Forum `_ 11 | 12 | 13 | Cylc 8 differs from Cylc 7 in many ways: architecture, scheduling 14 | algorithm, security, UIs, working practices, and more. 15 | 16 | .. toctree:: 17 | :maxdepth: 1 18 | 19 | summary 20 | cheat-sheet 21 | major-changes/index 22 | caveats 23 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/config-changes.rst: -------------------------------------------------------------------------------- 1 | .. _configuration-changes: 2 | 3 | Configuration Changes at Cylc 8 4 | =============================== 5 | 6 | Some configurations have moved or been renamed at Cylc 8. 7 | 8 | The old configurations are now deprecated, but still supported. 9 | These will be highlighted upon ``cylc validate`` after renaming ``suite.rc`` 10 | to ``flow.cylc``. 11 | 12 | Because some workflows use Jinja2 branches which may not be switched on at 13 | the time of the initial ``cylc validate`` we have also provided 14 | a script, :ref:`cylc lint -r 728 ` to check for Cylc 7 15 | syntax which may be deprecated. 16 | 17 | There are some examples below of how to upgrade: 18 | 19 | 20 | .. _7-to-8.graph_syntax: 21 | 22 | Graph 23 | ----- 24 | 25 | Cylc 7 had unnecessarily deep nesting of graph config sections: 26 | 27 | .. code-block:: cylc 28 | 29 | [scheduling] 30 | initial cycle point = now 31 | [[dependencies]] 32 | [[[R1]]] 33 | graph = "prep => foo" 34 | [[[R/^/P1D]]] 35 | graph = "foo => bar => baz" 36 | 37 | Cylc 8 cleans this up: 38 | 39 | .. code-block:: cylc 40 | 41 | [scheduling] 42 | initial cycle point = now 43 | [[graph]] 44 | R1 = "prep => foo" 45 | R/^/P1D = "foo => bar => baz" 46 | 47 | 48 | Fixing deprecation warnings 49 | --------------------------- 50 | 51 | Take the following example ``flow.cylc`` file: 52 | 53 | .. code-block:: cylc 54 | 55 | [cylc] 56 | UTC mode = True 57 | [scheduling] 58 | initial cycle point = 2000-01-01 59 | [[dependencies]] 60 | [[[R1]]] 61 | graph = foo => bar 62 | [runtime] 63 | [[foo, bar]] 64 | 65 | This workflow will pass validation at Cylc 8, but will give warnings: 66 | 67 | .. code-block:: console 68 | 69 | $ cylc validate . 70 | WARNING - deprecated items were automatically upgraded in "workflow definition" 71 | WARNING - * (8.0.0) [cylc] -> [scheduler] - value unchanged 72 | WARNING - deprecated graph items were automatically upgraded in "workflow definition": 73 | * (8.0.0) [scheduling][dependencies][X]graph -> [scheduling][graph]X - for X in: 74 | R1 75 | Valid for cylc-8.0.0 76 | 77 | The warnings explain what needs to be fixed. After making the following changes, 78 | the workflow will validate without any warnings: 79 | 80 | .. code-block:: diff 81 | 82 | -[cylc] 83 | +[scheduler] 84 | UTC mode = True 85 | [scheduling] 86 | initial cycle point = 2000-01-01 87 | - [[dependencies]] 88 | - [[[R1]]] 89 | - graph = foo => bar 90 | + [[graph]] 91 | + R1 = foo => bar 92 | [runtime] 93 | [[foo, bar]] 94 | 95 | .. tip:: 96 | 97 | Later Cylc releases will not be able to upgrade obsolete Cylc 7 98 | configurations. It's a good idea to address warnings as part of routine 99 | workflow review and maintenance to avoid problems later on. 100 | 101 | 102 | Platforms 103 | --------- 104 | 105 | .. seealso:: 106 | 107 | - :ref:`Platforms at Cylc 8. ` 108 | - :ref:`System admin's guide to writing platforms. ` 109 | 110 | At Cylc 7, job hosts were defined to indicate where a job should run. 111 | At Cylc 8, this has been replaced by Platforms. 112 | 113 | .. code-block:: diff 114 | 115 | [runtime] 116 | [[foo]] 117 | - [[[job]]] 118 | - batch system = slurm 119 | - [[[remote]]] 120 | - host = hpc1.login.1 121 | + platform = hpc1 122 | 123 | For a comprehensive list of valid configuration, see: :ref:`workflow-configuration` 124 | and :ref:`global-configuration`. 125 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/continuing-c7-c8.rst: -------------------------------------------------------------------------------- 1 | .. _compat_continuing_c7_with_c8: 2 | 3 | Continuing a Cylc 7 Workflow with Cylc 8 4 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 5 | 6 | .. admonition:: Does This Change Affect Me? 7 | :class: tip 8 | 9 | Read this if you have a partially complete Cylc 7 workflow that you want to 10 | continue, rather than start from scratch, with Cylc 8. Some cycling 11 | workflows, for example, may need to run expensive "cold start" tasks and 12 | incur a multi-cycle spin-up if started from scratch. 13 | 14 | .. warning:: 15 | 16 | Cylc 8 cannot restart a Cylc 7 workflow in-place, and continuing in a new 17 | run directory involves some careful set up (below). So, **if possible you 18 | should complete the run with Cylc 7**. 19 | 20 | 21 | To continue a Cylc 7 workflow with Cylc 8: 22 | 23 | 1. Stop the Cylc 7 workflow at an convenient place 24 | 25 | - Typically the end of a cycle point, to simplify the continuation 26 | 2. :ref:`Install ` a new instance of the workflow from 27 | source, with Cylc 8 28 | 29 | - Adapt file paths to the new run directory structure, in workflow and task 30 | configurations 31 | - Note Cylc 8 does :ref:`remote file installation <728.remote-install>` 32 | when a job is first submitted to a platform 33 | 3. Copy runtime files needed by upcoming tasks from the old to the new run 34 | directory 35 | 36 | - This could include external files installed by initial tasks at runtime 37 | - Note different files could be present on different job platforms 38 | 4. Start the new Cylc 8 run at the appropriate cycle point or task(s) in the 39 | graph 40 | 41 | - Don't reset the :term:`initial cycle point` (in the ``flow.cylc`` or on 42 | the command line) to the :term:`start point ` of the 43 | Cylc 8 run. That would result in the "cold start" that this continuation 44 | procedure is designed to avoid. Instead use the ``--start-cycle-point`` 45 | option (or ``--start-task``) with ``cylc play``, to start at the right 46 | place within the graph. 47 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/excluding-tasks.rst: -------------------------------------------------------------------------------- 1 | .. _MajorChangesExcludingTasksAtStartup: 2 | 3 | Excluding Tasks at Start-up is Not Supported 4 | ============================================ 5 | 6 | .. admonition:: Does This Change Affect Me? 7 | :class: tip 8 | 9 | This will affect you if your workflows use the following configurations: 10 | 11 | * ``[scheduling][special tasks]include at start-up`` 12 | * ``[scheduling][special tasks]exclude at start-up`` 13 | 14 | 15 | Overview 16 | -------- 17 | 18 | The Cylc 7 scheduler allowed you to exclude tasks from the scheduler at start-up: 19 | 20 | .. code-block:: cylc 21 | 22 | # Cylc 7 only 23 | [scheduling] 24 | [[special tasks]] 25 | include at start-up = foo, bar, baz # Cylc 8 ERROR! 26 | exclude at start-up = bar # Cylc 8 ERROR! 27 | 28 | The first config item above excludes all task names not in the include-list; 29 | the second excludes specific tasks that would otherwise be included. 30 | 31 | The Cylc 7 scheduler started up with an instance of every task in its "task 32 | pool", and the workflow evolved by each task spawning its own next-cycle 33 | instance at the right time. So, if you excluded a task a start-up it would not 34 | run in the workflow at all unless manually inserted later at runtime. 35 | 36 | The Cylc 8 scheduler starts up with only the initial tasks in the graph and the 37 | workflow evolves by spawning new tasks on demand as dictated by the graph. 38 | Consequently excluding a task at start up as described above would have no 39 | effect at all on most tasks. 40 | 41 | This feature also predated the current Cylc dependency graph configuration. To 42 | exclude tasks now without entirely removing them from the workflow definition, 43 | just comment them out of the graph. 44 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/index.rst: -------------------------------------------------------------------------------- 1 | Detailed Description of Major Changes 2 | ===================================== 3 | 4 | This section goes into detail on some of the major changes that you may need to 5 | understand to migrate from Cylc 7 to Cylc 8. 6 | 7 | .. toctree:: 8 | :maxdepth: 1 9 | :glob: 10 | 11 | * 12 | 13 | .. TODO - Split into breaking changes and other Major Changes 14 | 15 | .. TODO - Add a page with a Cylc 7 suite needing upgrading. 16 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/parameters.rst: -------------------------------------------------------------------------------- 1 | Parameters 2 | ========== 3 | 4 | .. admonition:: does this change affect me? 5 | :class: tip 6 | 7 | If you use Cylc parameters with negative offsets (e.g. ``foo``). 8 | 9 | 10 | Overview 11 | -------- 12 | 13 | There has been a subtle change in the way negative offsets are handled in parameters. 14 | 15 | 16 | Example 17 | ------- 18 | 19 | If you have a parameter ``x`` with the values 1, 2 & 3: 20 | 21 | .. code-block:: cylc 22 | 23 | [task parameters] 24 | x = 1..3 25 | 26 | And use it like so: 27 | 28 | .. code-block:: cylc-graph 29 | 30 | a => b => c 31 | 32 | There is some ambiguity about how this should be interpreted when ``x=1`` 33 | because ```` would be ``0`` which is not a valid value for the parameter 34 | ``x``. 35 | 36 | Cylc 7 removed the part of the expression which was out of range resulting in a 37 | partial evaluation of that line: 38 | 39 | .. code-block:: cylc-graph 40 | 41 | b_x1 => c_x1 # x=1 42 | a_x1 => b_x2 => c_x2 # x=2 43 | a_x2 => b_x3 => c_x3 # x=3 44 | 45 | Whereas Cylc 8 will remove everything after the first out-of-range parameter - ```` (so the entire line for this example): 46 | 47 | .. code-block:: cylc-graph 48 | 49 | a_x1 => b_x2 => c_x2 # x=2 50 | a_x2 => b_x2 => c_x2 # x=3 51 | 52 | 53 | Migration 54 | --------- 55 | 56 | If your workflow depends on the Cylc 7 behaviour, then the solution is 57 | to break the expression into two parts which Cylc will then evaluate separately. 58 | 59 | .. code-block:: diff 60 | 61 | - a => b => c 62 | + a => b 63 | + b => c 64 | 65 | Resulting in: 66 | 67 | .. code-block:: cylc-graph 68 | 69 | # a => b 70 | a_x1 => a_x2 # x=2 71 | a_x2 => a_x3 # x=3 72 | 73 | # b => c 74 | b_x1 => c_x1 # x=1 75 | b_x2 => c_x2 # x=2 76 | b_x3 => c_x3 # x=3 77 | 78 | 79 | Line Breaks 80 | ----------- 81 | 82 | Note that these expressions are all equivalent: 83 | 84 | .. list-table:: 85 | :class: grid-table 86 | 87 | * - .. code-block:: 88 | 89 | a => b => c 90 | 91 | - .. code-block:: 92 | 93 | a => 94 | b => 95 | c 96 | 97 | - .. code-block:: 98 | 99 | a => b => \ 100 | c 101 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/play-pause-stop.rst: -------------------------------------------------------------------------------- 1 | .. _728.play_pause_stop: 2 | 3 | Play Pause Stop 4 | =============== 5 | 6 | .. admonition:: Does This Change Affect Me? 7 | :class: tip 8 | 9 | Yes if you run Cylc workflows. 10 | 11 | 12 | Overview 13 | -------- 14 | 15 | Cylc 8 uses a simplified model for controlling workflows based on the controls 16 | of a tape player. 17 | 18 | There are now three controls, play, pause and stop: 19 | 20 | * When a workflow is playing, the :term:`scheduler` is running. 21 | * When a workflow is paused, no new jobs will be submitted. 22 | * When a workflow is stopped the :term:`scheduler` is no longer running. 23 | 24 | These controls are available in the web GUI or on the command line with the 25 | commands: 26 | 27 | * ``cylc play`` 28 | * ``cylc pause`` 29 | * ``cylc stop`` 30 | 31 | A workflow can be safely played, paused and stopped any number of times without 32 | interrupting the workflow. 33 | 34 | 35 | Re-Running Workflows 36 | -------------------- 37 | 38 | The ``cylc play`` command will always pick up where it left off (a 39 | :term:`restart`). 40 | 41 | If you want to re-run the entire workflow again from the start either: 42 | 43 | * :ref:`Install a new run`. 44 | * Or if you want to keep the data from the old run start a new :term:`flow` at 45 | the beginning of the graph, and stop the original flow. 46 | 47 | It is still possible to re-run workflows in-place in the Cylc 7 manner, however, 48 | this is discouraged. 49 | To do this remove the workflow database as well as any other evidence of the 50 | previous run that is no longer desired: 51 | 52 | .. code-block:: bash 53 | 54 | # remove the workflow database, the work, share and log directories 55 | cylc clean --rm .service/db:work:share:log 56 | 57 | # only remove the worflow database 58 | $ cylc clean --rm .service/db 59 | 60 | Then restart with ``cylc play``. 61 | 62 | 63 | Hold & Release 64 | -------------- 65 | 66 | The ``cylc hold`` and ``cylc release`` commands are still present. These 67 | work on individual tasks rather than the workflow as a whole. 68 | 69 | 70 | Mapping To Old Commands 71 | ----------------------- 72 | 73 | .. list-table:: 74 | :class: grid-table 75 | 76 | * - 77 | - **Cylc 7** 78 | - **Rose 2019** 79 | - **Cylc 8** (Rose 2) 80 | 81 | * - Play 82 | - :: 83 | 84 | cylc run 85 | - :: 86 | 87 | rose suite-run 88 | - :: 89 | 90 | cylc play 91 | 92 | * - Pause 93 | - :: 94 | 95 | cylc hold 96 | - :: 97 | 98 | cylc hold 99 | - :: 100 | 101 | cylc pause 102 | 103 | * - Resume 104 | - :: 105 | 106 | cylc release 107 | - :: 108 | 109 | cylc release 110 | - :: 111 | 112 | cylc play 113 | 114 | * - Stop 115 | - :: 116 | 117 | cylc stop 118 | - :: 119 | 120 | rose suite-shutdown 121 | - :: 122 | 123 | cylc stop 124 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/python-2-3.rst: -------------------------------------------------------------------------------- 1 | .. _py23: 2 | 3 | Python 2 => 3 4 | ============= 5 | 6 | .. admonition:: Does This Change Affect Me? 7 | :class: tip 8 | 9 | This change will affect you if your workflows extend Cylc or Jinja2 with 10 | custom Python scripts. 11 | 12 | This does not impact :term:`task` scripts; Cylc can still run Python 2 13 | tasks if desired. 14 | 15 | 16 | Overview 17 | -------- 18 | 19 | .. _six: https://pypi.org/project/six/ 20 | 21 | Cylc 7 ran under Python 2; Cylc 8 runs under Python 3. 22 | 23 | Cylc can be extended with custom Python scripts. These scripts are run under 24 | the same version of Python used by Cylc. 25 | 26 | As a result, if you are moving from Cylc 7 to Cylc 8, you must upgrade any 27 | scripts from Python 2 to Python 3 in the process. 28 | 29 | If you want to support both Cylc 7 and 8, you must support both Python 2 and 3. 30 | There are tools to help you do this, e.g. `six`_. 31 | 32 | 33 | Impacted Scripts 34 | ---------------- 35 | 36 | The following scripts must be upgraded if used: 37 | 38 | :ref:`CustomJinja2Filters` 39 | These allow you to extend Jinja2 with Python code. 40 | 41 | These scripts are located in the following directories within a workflow: 42 | 43 | * ``Jinja2Filters`` 44 | * ``Jinja2Tests`` 45 | * ``Jinja2Globals`` 46 | 47 | :ref:`Modules imported by Jinja2 ` 48 | Python modules can be imported from Jinja2 e.g: 49 | 50 | .. code-block:: jinja 51 | 52 | {% from "os" import path %} 53 | 54 | :ref:`Custom Trigger Functions` 55 | Any custom xtrigger functions. 56 | 57 | 58 | Package Name Changes 59 | -------------------- 60 | 61 | Three Python packages have been renamed between Cylc 7 and Cylc 8: 62 | 63 | * ``cylc`` => ``cylc.flow`` 64 | * ``isodatetime`` => ``metomi.isodatetime`` 65 | * ``rose`` => ``metomi.rose`` 66 | 67 | If you are importing from these packages you will need to update the package names. 68 | 69 | Here are some examples: 70 | 71 | .. rubric:: Convert Python code from Cylc 7 to Cylc 8: 72 | 73 | .. code-block:: diff 74 | 75 | - from cylc import LOG 76 | + from cylc.flow import LOG 77 | - from isodatetime.data import Duration 78 | + from metomi.isodatetime.data import Duration 79 | 80 | .. rubric:: Python code which supports both Cylc 7 & Cylc 8: 81 | 82 | .. code-block:: python 83 | 84 | import sys 85 | if sys.version[0] == '3': 86 | from cylc.flow import LOG 87 | from metomi.isodatetime.data import Duration 88 | else: 89 | from cylc import LOG 90 | from isodatetime.data import Duration 91 | 92 | .. rubric:: Convert Jinja2 code from Cylc 7 to Cylc 8: 93 | 94 | .. code-block:: diff 95 | 96 | #!Jinja2 97 | - {% from "cylc" import LOG %} 98 | + {% from "cylc.flow" import LOG %} 99 | {% do LOG.debug("Hello World!") %} 100 | 101 | .. rubric:: Jinja2 code which supports both Cylc 7 & Cylc 8: 102 | 103 | .. code-block:: jinja 104 | 105 | #!Jinja2 106 | {% from "sys" import version -%} 107 | {% if version[0] == '3' -%} 108 | {% from "cylc.flow" import LOG -%} 109 | {% else -%} 110 | {% from "cylc" import LOG -%} 111 | {% endif -%} 112 | 113 | {% do LOG.debug("Hello World!") %} 114 | 115 | 116 | Jinja2 - integers with leading zeros 117 | ------------------------------------ 118 | 119 | Integers with leading zeros in Jinja2 expressions are now illegal and will 120 | cause an error like ``Jinja2Error: expected token 'x', got 'integer'``. 121 | For example: 122 | 123 | .. code-block:: console 124 | 125 | $ cylc validate my-workflow 126 | Jinja2Error: expected token 'end of statement block', got 'integer' 127 | File ~/cylc-run/my-workflow/flow.cylc 128 | {% if START_HOUR == 06 or START_HOUR == 12 %} <-- TemplateSyntaxError 129 | 130 | The solution in this case is: 131 | 132 | .. code-block:: diff 133 | 134 | -{% if START_HOUR == 06 or START_HOUR == 12 %} 135 | +{% if START_HOUR == 6 or START_HOUR == 12 %} 136 | 137 | 138 | Rose 139 | ---- 140 | 141 | The same changes also impact Rose extensions: 142 | 143 | * :ref:`Rose Macros ` 144 | * :ref:`Rose Ana Tasks ` 145 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/remote-owner.rst: -------------------------------------------------------------------------------- 1 | .. _728.remote_owner: 2 | 3 | Remote Usernames 4 | ================ 5 | 6 | .. admonition:: does this change affect me? 7 | :class: tip 8 | 9 | * If you set :cylc:conf:`flow.cylc[runtime][][remote]owner` 10 | * If you use ``--owner`` on the command line 11 | 12 | 13 | Overview 14 | -------- 15 | 16 | If your username differs between the :term:`scheduler` host and job hosts, then 17 | you may have configured Cylc to run jobs under the correct account using 18 | :cylc:conf:`flow.cylc[runtime][][remote]owner` 19 | or used the ``--owner`` Cylc command line option 20 | with commands which access remote hosts. 21 | 22 | .. _SSH configuration file: https://man.openbsd.org/ssh_config 23 | 24 | Cylc no longer supports specifying the username in this way, we suggest 25 | configuring your remote username using the `SSH configuration file`_ e.g: 26 | 27 | .. code-block:: none 28 | 29 | Host MyHost 30 | User root 31 | 32 | SSH will then automatically use the configured username when connecting to the 33 | remote machine. 34 | 35 | Since Cylc uses SSH and Rsync to manage job hosts, the SSH config also configures 36 | Cylc. 37 | 38 | .. note:: 39 | 40 | This approach using the SSH configuration file also works with Cylc 7. 41 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/scheduling.rst: -------------------------------------------------------------------------------- 1 | .. _728.scheduling_algorithm: 2 | 3 | Scheduling Algorithm 4 | ==================== 5 | 6 | .. seealso:: 7 | 8 | Migration Guide: 9 | 10 | * :ref:`728.optional_outputs` 11 | 12 | User Guide: 13 | 14 | * :ref:`User Guide Required Outputs` 15 | * :ref:`User Guide Optional Outputs` 16 | * :ref:`user-guide-reflow` 17 | * :ref:`n-window` 18 | 19 | Cylc can manage infinite workflows of repeating tasks: 20 | 21 | .. image:: ../../img/cycling.png 22 | :align: center 23 | 24 | Cylc 8 has a new scheduling algorithm that: 25 | 26 | - Is much more efficient because it only has to manage 27 | :term:`active tasks `. 28 | 29 | - Tasks are not loaded into memory before they are needed. 30 | - Tasks are not retained in memory once they complete. 31 | - There is no costly indiscriminate dependency matching. 32 | - Distinguishes between :term:`optional ` and 33 | :term:`required ` task outputs, to support: 34 | 35 | - :term:`graph branching` without :term:`suicide triggers ` 36 | - correct diagnosis of :ref:`workflow completion` 37 | - Causes no implicit dependence on previous-instance job submit 38 | 39 | - instances of same task can run out of cycle point order 40 | - the workflow will not unnecessarily stall downstream of failed tasks 41 | - Provides a sensible activity-based window on the evolving workflow 42 | 43 | - (to fully understand which tasks appeared in the Cylc 7 GUI you had to 44 | understand the scheduling algorithm) 45 | - Supports multiple concurrent :term:`flows` within the same workflow. 46 | - Can start a workflow from any task or tasks in the graph (no need for 47 | checkpoint restart) 48 | - Can limit activity within as well as across cycles, without risking a stall 49 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/suicide-triggers.rst: -------------------------------------------------------------------------------- 1 | .. _728.optional_outputs: 2 | 3 | Graph branching, optional outputs and suicide triggers 4 | ====================================================== 5 | 6 | Cylc 8 has a :ref:`new scheduling algorithm <728.scheduling_algorithm>` and 7 | a new syntax for dealing with tasks that may not necessarily complete. 8 | It handles :term:`graphs ` in an event-driven manner which means 9 | that a workflow can follow different paths in different eventualities (without 10 | the need for suicide triggers). This is called :term:`graph branching`. 11 | 12 | .. admonition:: Does This Change Affect Me? 13 | :class: tip 14 | 15 | This change affects you if you are upgrading a Cylc 7 workflow that 16 | contains graph branches that are not necessarily expected to complete 17 | at runtime. You might get a ``GraphParseError`` during validation with 18 | Cylc 8. 19 | 20 | Typically this will be the case if you are using 21 | :term:`suicide triggers ` (marked by ``!`` before the 22 | task name in the graph, e.g. ``foo:fail => !foo``). 23 | 24 | You should *not* perform this upgrade if still in :ref:`cylc_7_compat_mode` 25 | (``suite.rc`` filename). 26 | 27 | 28 | Required Changes 29 | ^^^^^^^^^^^^^^^^ 30 | 31 | Any :term:`task outputs ` that are not necessarily expected to 32 | complete must be marked as :term:`optional ` using ``?``. 33 | Suicide triggers can then be removed. 34 | 35 | Example 36 | ^^^^^^^ 37 | 38 | Here is an example Cylc 7 :term:`graph`: 39 | 40 | .. code-block:: cylc-graph 41 | 42 | foo:fail => recover 43 | 44 | foo | recover => bar 45 | 46 | # Remove the "recover" task in the success case. 47 | foo => ! recover 48 | # Remove the "foo" task in the fail case. 49 | recover => ! foo 50 | 51 | .. digraph:: Example 52 | :align: center 53 | 54 | subgraph cluster_1 { 55 | label = ":fail" 56 | color = "red" 57 | fontcolor = "red" 58 | style = "dashed" 59 | recover 60 | } 61 | 62 | foo -> recover 63 | recover -> bar [arrowhead="onormal"] 64 | foo -> bar [arrowhead="onormal" weight=2] 65 | 66 | Validating this with Cylc 8 will give an error: 67 | 68 | .. code-block:: console 69 | 70 | $ cylc validate . 71 | GraphParseError: Opposite outputs foo:succeeded and foo:failed must both be optional if both are used 72 | 73 | In Cylc 8, all task outputs are :term:`required ` to complete 74 | unless otherwise indicated. However, it is impossible for both ``:succeed`` 75 | and ``:fail`` to occur when a task runs. 76 | 77 | The solution is to mark the outputs which are :term:`optional ` 78 | (in this case ``foo:succeed`` and ``foo:fail``) with a ``?`` in the graph. 79 | Also, the suicide triggers can be removed. 80 | 81 | .. code-block:: diff 82 | 83 | - foo:fail => recover 84 | + foo:fail? => recover 85 | 86 | - foo | recover => bar 87 | + foo? | recover => bar 88 | 89 | - # Remove the "recover" task in the success case. 90 | - foo => ! recover 91 | - # Remove the "foo" task in the fail case. 92 | - recover => ! foo 93 | 94 | In Cylc 7, suicide triggers were used to remove tasks that did not complete 95 | during runtime. Cylc 8's event-driven graph handling allows such graph 96 | branching using optional output syntax, without the need for suicide triggers. 97 | (Suicide triggers are still supported in Cylc 8; however, they are most 98 | likely unnecessary.) 99 | 100 | .. tip:: 101 | 102 | Remember: ``foo?`` is short for ``foo:succeed?``. It is the *output* 103 | that is optional, not the task itself. 104 | 105 | .. seealso:: 106 | 107 | - :ref:`Required ` and 108 | :ref:`optional ` outputs in the User Guide. 109 | 110 | - :ref:`Graph Branching` in the user guide. 111 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/task-job-states.rst: -------------------------------------------------------------------------------- 1 | .. _728.task_job_states: 2 | 3 | Task/Job States 4 | =============== 5 | 6 | .. seealso:: 7 | 8 | - User Guide :ref:`task-job-states` 9 | 10 | :term:`Tasks ` are nodes in the abstract workflow graph representing 11 | processes that should run once their prerequisites are satisfied. :term:`Jobs 12 | ` are the real processes submitted to execute these tasks (or at least, at 13 | the submission stage, real job scripts). A task can have multiple jobs, by 14 | automatic retries and manual re-triggering. 15 | 16 | Cylc 7 had 13 task/job states. The GUI only showed tasks, with job data 17 | from the latest job. 18 | 19 | Cylc 8 has only 8 task/job states. The Cylc 8 UI shows both task and jobs. 20 | Task icons are monochrome circles; job icons are coloured squares. The running 21 | task icon incorporates a radial progress indicator. 22 | 23 | .. image:: ../../img/task-job.png 24 | :align: center 25 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/template-vars.rst: -------------------------------------------------------------------------------- 1 | Template Variables 2 | ================== 3 | 4 | .. admonition:: Does This Change Affect Me? 5 | :class: tip 6 | 7 | Read this section if you set Cylc template variables on the command line 8 | using the ``-s``, ``--set`` or ``-set-file`` options. 9 | 10 | This does *not* affect the Rose ``jinja2:suite.rc`` 11 | variables set using the ``-S`` option to the 12 | ``rose suite-run`` command. 13 | 14 | 15 | Overview 16 | -------- 17 | 18 | Template variables are passed to :ref:`Jinja2 ` 19 | for parsing the workflow definition in the 20 | :cylc:conf:`flow.cylc` file. 21 | 22 | In Cylc 7 template variables could only be strings, in Cylc 8 they can be any 23 | valid Python literal including numbers, booleans, and lists. 24 | 25 | 26 | Changes 27 | ------- 28 | 29 | Strings must be explicitly quoted i.e. ``key="value"`` rather than ``key=value``. 30 | 31 | 32 | Example 33 | ------- 34 | 35 | .. rubric:: Setting template variables on the command line: 36 | 37 | .. code-block:: bash 38 | 39 | # Cylc 7 40 | cylc run -s 'FOO=abc' 41 | # Cylc 8 42 | cylc play -s 'FOO="abc"' 43 | 44 | 45 | .. rubric:: Setting template variables in a "set file" (using ``--set-file``): 46 | 47 | .. code-block:: python 48 | 49 | # Cylc 7 50 | FOO = abc 51 | BAR = bcd 52 | 53 | # Cylc 8 54 | FOO = "abc" 55 | BAR = "bcd" 56 | 57 | 58 | New Features 59 | ------------ 60 | 61 | Any valid Python literals 62 | ^^^^^^^^^^^^^^^^^^^^^^^^^ 63 | 64 | Template variables can now be any valid Python literals e.g: 65 | 66 | .. code-block:: python 67 | 68 | "string" # string 69 | 123 # integer 70 | 12.34 # float 71 | True # boolean 72 | None # None type 73 | [1, 2, 3] # list 74 | (1, 2, 3) # tuple 75 | {1, 2, 3} # set 76 | {"a": 1, "b": 2, "c": 3} # dictionary 77 | 78 | See :ref:`jinja2-template-variables` for more information. 79 | 80 | Shorthand for list of strings 81 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 82 | 83 | .. versionadded:: 8.2 84 | 85 | A new shorthand argument (``-z``/``--set-list``/``--set-template``) 86 | has been introduced allowing easier definition of template 87 | variables containing lists of strings on the command line: 88 | 89 | .. code-block:: shell 90 | 91 | # Before (still works) 92 | cylc --set "X=['a', 'b', 'c']" 93 | 94 | # After 95 | cylc --set-list X=a,b,c 96 | -------------------------------------------------------------------------------- /src/7-to-8/major-changes/ui.rst: -------------------------------------------------------------------------------- 1 | .. _728.ui: 2 | 3 | Cylc 8 UIs 4 | ========== 5 | 6 | There are two UI options available to monitor and control workflows at Cylc 8. 7 | 8 | 9 | Cylc Tui 10 | -------- 11 | 12 | Cylc Tui (TUI = Terminal User Interface) is a command line version of the GUI 13 | which comes packaged with the Cylc scheduler. 14 | 15 | It can be used to monitor and control any workflows running under your user 16 | account, trigger tasks, access log files and perform other common activities. 17 | 18 | Start Tui by running the ``cylc tui`` command. 19 | 20 | .. figure:: ../../img/tui-1.png 21 | :figwidth: 80% 22 | :align: center 23 | 24 | Tui showing the details of a failed job. 25 | 26 | .. figure:: ../../img/tui-2.png 27 | :figwidth: 80% 28 | :align: center 29 | 30 | A list of actions that can be performed on the selected task. 31 | 32 | .. figure:: ../../img/tui-3.png 33 | :figwidth: 80% 34 | :align: center 35 | 36 | Tui displaying the workflow configuration. It can also show scheduler and 37 | job log files. 38 | 39 | Tui replaces the Cylc 7 ``cylc monitor`` command. 40 | 41 | 42 | Cylc Web GUI 43 | ------------ 44 | 45 | The Cylc GUI application is a monitoring and control application which runs in 46 | a web browser, it is distributed in the `Cylc UI Server`_ package which comes 47 | separately from the core scheduler. 48 | 49 | Start the GUI server and open the web app in your browser by running 50 | ``cylc gui``. 51 | 52 | .. figure:: ../../img/cylc-ui-dash.png 53 | :figwidth: 80% 54 | :align: center 55 | 56 | The GUI homepage. 57 | 58 | .. figure:: ../../img/cylc-ui-tree.png 59 | :figwidth: 80% 60 | :align: center 61 | 62 | The GUI displaying a workflow using the "tree" view. 63 | 64 | See :ref:`UI_Server_config` for how to configure the GUI. 65 | 66 | As some workflows can be very large, or even infinite, the GUI uses a "window" 67 | system to determine what to display. For more information, see :ref:`n-window`. 68 | 69 | 70 | .. _Cylc Hub: 71 | 72 | Cylc Hub 73 | -------- 74 | 75 | The Cylc 8 GUI can be deployed with Jupyter Hub to support multi-user access 76 | where it is possible to grant users the permission to view and interact with 77 | workflows running under other user accounts. In these deployments, users 78 | authenticate when they open the GUI in the browser. 79 | 80 | The central server is started by the ``cylc hub`` command. 81 | 82 | .. figure:: ../../img/hub.png 83 | :figwidth: 80% 84 | :align: center 85 | 86 | The Jupyter Hub authentication page in a multi-user setup. 87 | 88 | Multi-user setups need to be configured by site administrators, for more 89 | information see :ref:`cylc.uiserver.multi-user`. 90 | -------------------------------------------------------------------------------- /src/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | * Copyright (C) NIWA & British Crown (Met Office) & Contributors. 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | * ------------------------------------------------------------------------- */ 17 | 18 | /* Fix zero height auto size of svg */ 19 | .logo { 20 | height: 7em !important; 21 | } 22 | -------------------------------------------------------------------------------- /src/_templates/automodule_job_runner_handlers.rst: -------------------------------------------------------------------------------- 1 | {{ name | escape | underline }} 2 | 3 | .. automodule:: {{ fullname }} 4 | -------------------------------------------------------------------------------- /src/_templates/docstring_only.rst: -------------------------------------------------------------------------------- 1 | {{ fullname | escape | underline }} 2 | 3 | .. automodule:: {{ fullname }} 4 | -------------------------------------------------------------------------------- /src/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | -------------------------------------------------------------------------------- /src/_templates/main_loop_plugin.rst: -------------------------------------------------------------------------------- 1 | {{ fullname | escape | underline }} 2 | 3 | .. automodule:: {{ fullname }} 4 | :members: {{ functions | join(', ') }} 5 | 6 | .. set members so that we only document what is actually defined 7 | otherwise you can end up with imported modules and mess like that 8 | 9 | {% block functions %} 10 | 11 | Python API 12 | ---------- 13 | 14 | {% if functions %} 15 | .. rubric:: Coroutines 16 | 17 | .. write out a table of functions (coroutines in this context) 18 | for quick reference 19 | 20 | .. autosummary:: 21 | :nosignatures: 22 | 23 | {% for item in functions %} 24 | {{ item }} 25 | {%- endfor %} 26 | 27 | {% endif %} 28 | {% endblock %} 29 | -------------------------------------------------------------------------------- /src/_templates/versions.html: -------------------------------------------------------------------------------- 1 | 5 |
10 | 11 | {% set ROOT_DIR = pathto('').rsplit('.html', 1)[0] + '../..' %} 12 | {% set CUR_VERSION = sidebar_version_name if sidebar_version_name else release %} 13 | 14 | 22 | 23 | 27 | -------------------------------------------------------------------------------- /src/dictionaries/proper_nouns: -------------------------------------------------------------------------------- 1 | Aldergrove 2 | ana 3 | Boolean 4 | Booleans 5 | Camborne 6 | conda 7 | Conda 8 | Cylc 9 | cylc 10 | Datapoint 11 | Empy 12 | Exeter 13 | Gantt 14 | Gedit 15 | Graphene 16 | Graphviz 17 | GScan 18 | Heathrow 19 | Hoverwork 20 | IDL 21 | Jinja 22 | jinja 23 | Jupyter 24 | libmamba 25 | loadleveler 26 | lsf 27 | Micromamba 28 | moab 29 | Moab 30 | Nano 31 | npm 32 | pbs 33 | Posix 34 | Protobuf 35 | Pympler 36 | Pyro 37 | pyproject 38 | rsync 39 | sge 40 | slurm 41 | Slurm 42 | sqlite 43 | svn 44 | toml 45 | Trac 46 | Tui 47 | Vuex 48 | xtrace 49 | -------------------------------------------------------------------------------- /src/dictionaries/words: -------------------------------------------------------------------------------- 1 | GUI 2 | Popen 3 | app 4 | apps 5 | arg 6 | args 7 | auth 8 | authenticator 9 | authoriser 10 | backticks 11 | baz 12 | boolean 13 | booleans 14 | cfg 15 | changelog 16 | changelogs 17 | conf 18 | config 19 | configargs 20 | configs 21 | coroutine 22 | coroutines 23 | cprofile 24 | ctx 25 | cyclepoint 26 | dataset 27 | datasets 28 | datetime 29 | datetimes 30 | deprecations 31 | dev 32 | dict 33 | diff 34 | diffs 35 | dir 36 | dirs 37 | durations 38 | env 39 | etc 40 | executables 41 | facade 42 | fallback 43 | filename 44 | filenames 45 | filesystem 46 | filesystems 47 | fillchar 48 | foo 49 | fpath 50 | freeform 51 | fullmatch 52 | globals 53 | graining 54 | gscan 55 | hardcodes 56 | hardwired 57 | hardwiring 58 | heathrow 59 | heredocs 60 | hetjob 61 | hostname 62 | hostnames 63 | inbox 64 | inline 65 | inlined 66 | inlines 67 | inlining 68 | int 69 | integrations 70 | intercycle 71 | interoperability 72 | interoperable 73 | intra 74 | intranet 75 | iso 76 | iterable 77 | jitter 78 | jobscript 79 | json 80 | kwarg 81 | kwargs 82 | latin 83 | lifecycle 84 | linearization 85 | linter 86 | linting 87 | localhost 88 | lowercase 89 | markup 90 | max 91 | metadata 92 | mineshafts 93 | misconfiguration 94 | multiline 95 | namelist 96 | namelists 97 | namespace 98 | namespaces 99 | obs 100 | offline 101 | onwards 102 | packjob 103 | parameterization 104 | parameterize 105 | parameterized 106 | parentless 107 | passX 108 | passphrase 109 | passwordless 110 | performant 111 | placeholder 112 | placeholders 113 | plugin 114 | plugins 115 | postprocessing 116 | pre 117 | predated 118 | preempt 119 | preempted 120 | preemption 121 | preloaded 122 | prempted 123 | premption 124 | prepackaged 125 | prepend 126 | prepended 127 | preprocessing 128 | preset 129 | proc 130 | programmatically 131 | proleptic 132 | py 133 | pythonically 134 | queueing 135 | qux 136 | randint 137 | rc 138 | refactor 139 | regex 140 | reimplement 141 | reimplementation 142 | reimplemented 143 | reinstallation 144 | reinstalls 145 | replier 146 | repo 147 | requeue 148 | requeued 149 | resends 150 | resolvers 151 | resubmission 152 | ret 153 | retrigger 154 | retriggered 155 | retriggering 156 | rmtree 157 | roadmap 158 | ruleset 159 | rulesets 160 | runahead 161 | runnable 162 | runtime 163 | schd 164 | screenshot 165 | secs 166 | setup 167 | setups 168 | shebang 169 | shutdown 170 | shutil 171 | situ 172 | speedup 173 | ssh 174 | st 175 | startup 176 | stderr 177 | stdin 178 | stdout 179 | str 180 | strftime 181 | strptime 182 | subcommands 183 | subdir 184 | subdirectories 185 | subdirectory 186 | subprocess 187 | subprocesses 188 | subshell 189 | subshelled 190 | subshells 191 | suicided 192 | swapper 193 | symlink 194 | symlinked 195 | symlinking 196 | symlinks 197 | sys 198 | taskdef 199 | templated 200 | templating 201 | th 202 | timeout 203 | timeouts 204 | timestamp 205 | timestep 206 | timezone 207 | timezones 208 | toolbar 209 | tuple 210 | uiserver 211 | unparsed 212 | unpause 213 | unpaused 214 | unpausing 215 | untracked 216 | untrappable 217 | url 218 | username 219 | usernames 220 | uuid 221 | vip 222 | vis 223 | vs 224 | wallclock 225 | websocket 226 | websockets 227 | whitelist 228 | whitelisted 229 | whitelisting 230 | whitespace 231 | workflow 232 | workflows 233 | workspaces 234 | xrandom 235 | xtrigger 236 | xtriggers 237 | -------------------------------------------------------------------------------- /src/ext/autodoc_traits/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) NIWA & British Crown (Met Office) & Contributors. 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | """Autodoc extension for configurable traits. 16 | This code auto documents traits from Cylc UI Server: 17 | """ 18 | 19 | __version__ = "0.1.0" 20 | 21 | 22 | def setup(app, *args, **kwargs): 23 | from .autodoc_traits import setup 24 | 25 | return setup(app, *args, **kwargs) 26 | -------------------------------------------------------------------------------- /src/img/conditional-triggers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/conditional-triggers.png -------------------------------------------------------------------------------- /src/img/cycling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/cycling.png -------------------------------------------------------------------------------- /src/img/cylc-favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/cylc-favicon.ico -------------------------------------------------------------------------------- /src/img/cylc-logo-colour.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/img/cylc-logo-greyscale.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/img/cylc-logo-white.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/img/cylc-ui-dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/cylc-ui-dash.png -------------------------------------------------------------------------------- /src/img/cylc-ui-tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/cylc-ui-tree.png -------------------------------------------------------------------------------- /src/img/dep-eg-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/dep-eg-1.png -------------------------------------------------------------------------------- /src/img/eg2-dynamic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/eg2-dynamic.png -------------------------------------------------------------------------------- /src/img/eg2-static.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/eg2-static.png -------------------------------------------------------------------------------- /src/img/failure-recovery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/failure-recovery.png -------------------------------------------------------------------------------- /src/img/fam-to-fam-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/fam-to-fam-1.png -------------------------------------------------------------------------------- /src/img/fam-to-fam-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/fam-to-fam-2.png -------------------------------------------------------------------------------- /src/img/hub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/hub.png -------------------------------------------------------------------------------- /src/img/installation/conda/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/installation/conda/dashboard.png -------------------------------------------------------------------------------- /src/img/installation/conda/hub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/installation/conda/hub.png -------------------------------------------------------------------------------- /src/img/installation/conda/treeview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/installation/conda/treeview.png -------------------------------------------------------------------------------- /src/img/installation/conda/workflows.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/installation/conda/workflows.png -------------------------------------------------------------------------------- /src/img/jinja2-ensemble-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/jinja2-ensemble-graph.png -------------------------------------------------------------------------------- /src/img/jinja2-workflow-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/jinja2-workflow-graph.png -------------------------------------------------------------------------------- /src/img/n-window.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/n-window.gif -------------------------------------------------------------------------------- /src/img/n-window.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/n-window.png -------------------------------------------------------------------------------- /src/img/new-flow-n.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/new-flow-n.png -------------------------------------------------------------------------------- /src/img/new-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/new-flow.png -------------------------------------------------------------------------------- /src/img/niwa-colour-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/niwa-colour-small.png -------------------------------------------------------------------------------- /src/img/niwa-colour.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/niwa-colour.png -------------------------------------------------------------------------------- /src/img/no-flow-n.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/no-flow-n.png -------------------------------------------------------------------------------- /src/img/no-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/no-flow.png -------------------------------------------------------------------------------- /src/img/param-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/param-1.png -------------------------------------------------------------------------------- /src/img/param-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/param-2.png -------------------------------------------------------------------------------- /src/img/params1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/params1.png -------------------------------------------------------------------------------- /src/img/pipe-pub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/pipe-pub.png -------------------------------------------------------------------------------- /src/img/same-flow-behind.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/same-flow-behind.png -------------------------------------------------------------------------------- /src/img/same-flow-n.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/same-flow-n.png -------------------------------------------------------------------------------- /src/img/same-flow-wait-n.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/same-flow-wait-n.png -------------------------------------------------------------------------------- /src/img/same-flow-wait.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/same-flow-wait.png -------------------------------------------------------------------------------- /src/img/same-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/same-flow.png -------------------------------------------------------------------------------- /src/img/satellite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/satellite.png -------------------------------------------------------------------------------- /src/img/task-job-icons/README: -------------------------------------------------------------------------------- 1 | Note - these icons are auto-generated by the cylc-ui "component" tests. 2 | -------------------------------------------------------------------------------- /src/img/task-job-icons/README.md: -------------------------------------------------------------------------------- 1 | Note - these icons are auto-generated by the cylc-ui "component" tests. 2 | -------------------------------------------------------------------------------- /src/img/task-job-icons/job-failed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/job-failed.png -------------------------------------------------------------------------------- /src/img/task-job-icons/job-running.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/job-running.png -------------------------------------------------------------------------------- /src/img/task-job-icons/job-submit-failed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/job-submit-failed.png -------------------------------------------------------------------------------- /src/img/task-job-icons/job-submitted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/job-submitted.png -------------------------------------------------------------------------------- /src/img/task-job-icons/job-succeeded.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/job-succeeded.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-expired.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-expired.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-failed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-failed.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-isHeld.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-isHeld.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-isQueued.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-isQueued.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-isRunahead.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-isRunahead.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-preparing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-preparing.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-running-0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-running-0.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-running-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-running-100.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-running-25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-running-25.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-running-50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-running-50.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-running-75.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-running-75.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-running.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-running.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-submit-failed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-submit-failed.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-submitted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-submitted.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-succeeded.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-succeeded.png -------------------------------------------------------------------------------- /src/img/task-job-icons/task-waiting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job-icons/task-waiting.png -------------------------------------------------------------------------------- /src/img/task-job.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/task-job.png -------------------------------------------------------------------------------- /src/img/test1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/test1.png -------------------------------------------------------------------------------- /src/img/test2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/test2.png -------------------------------------------------------------------------------- /src/img/test4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/test4.png -------------------------------------------------------------------------------- /src/img/test5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/test5.png -------------------------------------------------------------------------------- /src/img/tui-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/tui-1.png -------------------------------------------------------------------------------- /src/img/tui-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/tui-2.png -------------------------------------------------------------------------------- /src/img/tui-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/img/tui-3.png -------------------------------------------------------------------------------- /src/index.rst: -------------------------------------------------------------------------------- 1 | Cylc Documentation 2 | ================== 3 | 4 | Cylc ("*silk*") is a workflow engine for cycling systems - it orchestrates 5 | distributed workflows of interdependent cycling tasks that may continue to run 6 | indefinitely. 7 | 8 | .. image:: https://img.shields.io/discourse/users?label=Join%20the%20community%20on%20Discourse&logo=discourse&server=https%3A%2F%2Fcylc.discourse.group&style=socia0l 9 | :align: center 10 | :width: 50% 11 | :target: https://cylc.discourse.group 12 | 13 | | 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | 18 | 7-to-8/index 19 | installation 20 | tutorial/index 21 | user-guide/index 22 | plugins/index 23 | workflow-design-guide/index 24 | reference/index 25 | glossary 26 | -------------------------------------------------------------------------------- /src/lib/cylc_release.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from cylc.flow import __version__ 4 | 5 | from pkg_resources import parse_version 6 | 7 | 8 | def pinned_version(version): 9 | """Return a short version suitible for pinning Cylc Flow in recipes. 10 | 11 | For most uses pinning to the minor version is likely the safest option:: 12 | 13 | conda install cylc-flow=8.1 14 | 15 | For major pre-releases (e.g. 8.0b1) we keep the pre release tag. 16 | 17 | See https://github.com/cylc/cylc-admin/issues/130 18 | 19 | Examples: 20 | >>> pinned_version('8.0.1') 21 | '8.0' 22 | >>> pinned_version('8.0.1b1') 23 | '8.0' 24 | >>> pinned_version('8.0.1.dev') 25 | '8.0' 26 | >>> pinned_version('8.0b1') 27 | '8.0b1' 28 | >>> pinned_version('8.0b1.dev') 29 | '8.0b1' 30 | >>> pinned_version('8.0rc1') 31 | '8.0rc1' 32 | 33 | """ 34 | ver = parse_version(version) 35 | if ver.pre and ver.minor == 0 and ver.micro == 0: 36 | # special handling of major version pre-releases 37 | ret = (ver.major, str(ver.minor) + ''.join(map(str, ver.pre))) 38 | else: 39 | ret = (ver.major, ver.minor) 40 | return '.'.join(map(str, ret)) 41 | 42 | 43 | CYLC_RELEASE = pinned_version(__version__) 44 | 45 | 46 | if __name__ == '__main__': 47 | print(CYLC_RELEASE) 48 | -------------------------------------------------------------------------------- /src/plugins/cylc-rose.rst: -------------------------------------------------------------------------------- 1 | .. _Cylc Rose: 2 | 3 | Cylc Rose 4 | ========= 5 | 6 | .. automodule:: cylc.rose 7 | -------------------------------------------------------------------------------- /src/plugins/index.rst: -------------------------------------------------------------------------------- 1 | Plugins 2 | ======= 3 | 4 | Cylc supports plugins for providing additional functionality. 5 | 6 | The following are "core" plugins maintained by the Cylc team: 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | 11 | cylc-rose 12 | main-loop/index 13 | install/index 14 | job-runners/index 15 | xtriggers/index 16 | 17 | .. warning:: 18 | 19 | Plugins are a new feature of Cylc 8 and the interface is likely to change in 20 | future releases. 21 | -------------------------------------------------------------------------------- /src/plugins/job-runners/index.rst: -------------------------------------------------------------------------------- 1 | .. _CustomJobSubmissionMethods: 2 | 3 | Custom Job Submission Methods 4 | ----------------------------- 5 | 6 | .. autoclass:: cylc.flow.job_runner_handlers.documentation.ExampleHandler 7 | :members: 8 | -------------------------------------------------------------------------------- /src/plugins/main-loop/index.rst: -------------------------------------------------------------------------------- 1 | Main Loop Plugins 2 | ================= 3 | 4 | .. automodule:: cylc.flow.main_loop 5 | -------------------------------------------------------------------------------- /src/plugins/xtriggers/index.rst: -------------------------------------------------------------------------------- 1 | Xtrigger Plugins 2 | ====================================== 3 | 4 | .. versionadded:: 8.3 5 | 6 | Xtrigger plugins allow you to install and use 7 | :ref:`xtriggers
` without them being 8 | in ``/lib/python/`` or ``$CYLC_PYTHONPATH``. 9 | 10 | .. seealso:: 11 | 12 | * :ref:`Built-in Clock Triggers` 13 | * :ref:`Built-in Workflow State Triggers` 14 | * :ref:`Built-in Toy Xtriggers` 15 | 16 | 17 | .. _developing.xtrigger.plugins: 18 | 19 | Developing ``xtrigger`` plugins 20 | ------------------------------- 21 | 22 | Cylc uses the ``cylc.xtriggers`` entry point registered by setuptools to search 23 | for xtrigger plugins. Each xtrigger is registered individually. 24 | 25 | Example 26 | ^^^^^^^ 27 | 28 | Consider a package called ``my_package`` with the following structure: 29 | 30 | .. code-block:: python 31 | :caption: ``my_package/foo.py`` 32 | 33 | def foo(): 34 | ... 35 | 36 | def bar(): 37 | ... 38 | 39 | .. code-block:: python 40 | :caption: ``my_package/baz.py`` 41 | 42 | def baz(): 43 | ... 44 | 45 | These xtriggers can be registered in the package's ``setup.cfg`` or 46 | ``pyproject.toml`` file. 47 | 48 | .. code-block:: ini 49 | :caption: ``setup.cfg`` 50 | 51 | [options.entry_points] 52 | cylc.xtriggers = 53 | foo = my_package.foo 54 | bar = my_package.foo 55 | baz = my_package.baz 56 | 57 | .. code-block:: toml 58 | :caption: ``pyproject.toml`` 59 | 60 | [project.entry-points."cylc.xtriggers"] 61 | foo = "my_package.foo" 62 | bar = "my_package.foo" 63 | baz = "my_package.baz" 64 | 65 | .. tip:: 66 | 67 | It is recommended to implement only one xtrigger per module. This allows 68 | you to write a ``validate`` function for each xtrigger - see 69 | :ref:`user-guide.xtrigger-validation-functions`. 70 | 71 | -------------------------------------------------------------------------------- /src/reference/api/exceptions.rst: -------------------------------------------------------------------------------- 1 | Exceptions 2 | ========== 3 | 4 | .. automodule:: cylc.flow.exceptions 5 | :members: 6 | 7 | .. automodule:: cylc.flow.parsec.exceptions 8 | :members: 9 | -------------------------------------------------------------------------------- /src/reference/api/index.rst: -------------------------------------------------------------------------------- 1 | Cylc API 2 | ======== 3 | 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | scan 9 | zmq 10 | exceptions 11 | -------------------------------------------------------------------------------- /src/reference/api/scan.rst: -------------------------------------------------------------------------------- 1 | Scan API 2 | ======== 3 | 4 | .. automodule:: cylc.flow.network.scan 5 | -------------------------------------------------------------------------------- /src/reference/api/zmq.rst: -------------------------------------------------------------------------------- 1 | Workflow Runtime Interface 2 | ========================== 3 | 4 | Cylc workflows are TCP servers which use the ZeroMQ protocol to communicate with 5 | clients and jobs. 6 | 7 | Cylc provides a Python client to communicate with this server 8 | :py:class:`cylc.flow.network.client.WorkflowRuntimeClient` 9 | 10 | .. code-block:: python 11 | 12 | >>> from cylc.flow.network.client import WorkflowRuntimeClient 13 | >>> client = WorkflowRuntimeClient('my-workflow') 14 | >>> client('ping_workflow') 15 | True 16 | 17 | Cylc also provides sub-command called ``cylc client`` which is a simple 18 | wrapper of the Python client. 19 | 20 | .. code-block:: console 21 | 22 | $ cylc client generic ping_workflow -n 23 | true 24 | 25 | The available "commands" or ("endpoints") are contained in 26 | :py:class:`cylc.flow.network.server.WorkflowRuntimeServer` class. 27 | 28 | 29 | Client 30 | ------ 31 | 32 | .. autoclass:: cylc.flow.network.client.WorkflowRuntimeClient 33 | 34 | 35 | Server 36 | ------ 37 | 38 | .. autoclass:: cylc.flow.network.server.WorkflowRuntimeServer 39 | :members: 40 | -------------------------------------------------------------------------------- /src/reference/architecture/img/websocket-communication.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/architecture/img/websocket-communication.png -------------------------------------------------------------------------------- /src/reference/architecture/img/websocket-graphql-ws-protocol.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/architecture/img/websocket-graphql-ws-protocol.png -------------------------------------------------------------------------------- /src/reference/architecture/index.rst: -------------------------------------------------------------------------------- 1 | .. _architecture-reference: 2 | 3 | Architecture 4 | ============ 5 | 6 | This section provides an in-depth explanation of the architecture of the 7 | Cylc components intended for developers and system administrators. 8 | 9 | The main Cylc 8 system components are: 10 | 11 | Cylc Scheduler 12 | -------------- 13 | 14 | - The workflow engine core, Python 3 based 15 | - Includes the **CLI** (Command Line Interface) 16 | - And **TUI**, a new Terminal UI application 17 | 18 | Cylc UI 19 | ------- 20 | 21 | - In-browser web UI, includes: 22 | - A dashboard with summary information and documentation links 23 | - Integrated gscan (multi-workflow) side-panel 24 | - Responsive web design (from desktop to table to mobile) 25 | - Tabbed interface to display multiple workflow views 26 | - Command integration for interacting with task, jobs, and schedulers 27 | 28 | Cylc UI Server 29 | -------------- 30 | 31 | - Interacts with Schedulers and the filesystem 32 | - Serves the UI to users 33 | - Can be launched by the privileged Hub, for multi-user installations 34 | - Or run standalone for use by a single user 35 | - (The UI Server is a `Jupyter Server 36 | `_ extension) 37 | 38 | Cylc Hub 39 | -------- 40 | 41 | - For proving the UI in multi-user setups. 42 | - Authenticates users, spawns and proxies Cylc UI Servers 43 | - Can run as a regular or privileged user 44 | - (The Hub is a `Jupyterhub `_ instance) 45 | 46 | Network layers 47 | -------------- 48 | 49 | - Incremental push updates (c.f. polled full-state updates in Cylc 7) 50 | 51 | Further Reading 52 | --------------- 53 | 54 | .. toctree:: 55 | :maxdepth: 2 56 | 57 | ui-server 58 | data-flow 59 | -------------------------------------------------------------------------------- /src/reference/architecture/ui-server.rst: -------------------------------------------------------------------------------- 1 | .. _CylcUIServer.architecture: 2 | 3 | Cylc UI Server 4 | ============== 5 | 6 | `Cylc Flow`_ 7 | Provides a command line utility for monitoring and controlling 8 | Cylc workflows called ``cylc tui``. 9 | `Cylc UI Server`_ 10 | Provides a graphical utility for use in a web browser. 11 | 12 | The `Cylc UI Server`_ connects to running workflows to provide "live" data 13 | and accesses workflow databases and the filesystem to provide "offline" data. 14 | 15 | 16 | Jupyter Server 17 | -------------- 18 | 19 | The `Cylc UI Server`_ is a `Jupyter Server`_ extension like `Jupyter Lab`_. 20 | 21 | `Jupyter Server`_ provides the web server infrastructure which is shared by 22 | its extensions which can used to run multiple extensions simultaneously. 23 | 24 | If desired other extensions (e.g. `Jupyter Lab`_) can be installed and 25 | configured to run in the same server as the `Cylc UI Server`_. 26 | 27 | See :ref:`managing-multiple-extensions` for details on managing which 28 | extensions are run by `Jupyter Server`_. 29 | 30 | `Jupyter Server`_ can be run in two ways, single-user (token authenticated) 31 | and multi-user (hub authenticated). 32 | 33 | 34 | .. _single-user mode: 35 | 36 | Single-User Mode 37 | ---------------- 38 | 39 | In single-user mode users must start their own UI Servers from the command line. 40 | 41 | `Jupyter Server`_ will provide them with a URL to access their server including 42 | a secure token which provides authentication. 43 | 44 | .. admonition:: Authentication Overview 45 | :class: hint 46 | 47 | See :ref:`server_security`: 48 | 49 | Users can only monitor and control their own workflows. 50 | 51 | .. image:: img/gui-arch-single-user.svg 52 | :width: 70% 53 | :align: center 54 | 55 | 56 | Jupyter Hub 57 | ----------- 58 | 59 | .. _Cylc Hub configuration file: https://github.com/cylc/cylc-uiserver/blob/master/cylc/uiserver/jupyter_config.py 60 | 61 | `Jupyter Hub`_ is a multi-user server which spawns and manages a configured 62 | service for authenticated users. 63 | 64 | The "Cylc Hub" is a Jupyter Hub instance which is pre-configured to spawn 65 | Cylc UI Servers, launched by the ``cylc hub`` command. 66 | It is also possible to configure Jupyter Hub yourself, see the Cylc Hub 67 | configuration file for more information. 68 | 69 | Jupyter Hub supports a variety of different implementations and plugin interfaces 70 | for: 71 | 72 | * `Authenticating users `_ 73 | * `Spawning user's servers `_ 74 | * `Proxying user's servers `_ 75 | 76 | 77 | .. _multi-user mode: 78 | 79 | Multi-User Mode 80 | ---------------- 81 | 82 | Multi-user mode requires `Jupyter Hub`_ to be installed. 83 | 84 | An administrator must start `Jupyter Hub`_ under a user account with 85 | the required privileges to spawn UI Servers on behalf of the user. 86 | 87 | Users then visit `Jupyter Hub`_ where they authenticate. `Jupyter Hub`_ 88 | spawns UI Servers on behalf of users and provides each with a fixed URL 89 | (derived from the user name) using the configured proxy 90 | (usually `Configurable HTTP Proxy`_). 91 | 92 | Users can access each other's UI Servers providing they have been granted 93 | permission. 94 | 95 | *Authentication* is provided by either `Jupyter Server`_ or `Jupyter Hub`_. 96 | 97 | *Authorization* in the `Cylc UI Server`_ is provided by Cylc. In 98 | multi-user mode this allows users to connect to each other's UI Servers for 99 | monitoring or control purposes. 100 | 101 | For more information on security and configuration see 102 | :ref:`cylc.uiserver.multi-user`. 103 | 104 | .. _Jupyter Hub technical overview: https://jupyterhub.readthedocs.io/en/stable/reference/technical-overview.html 105 | 106 | For information on the architecture of `Jupyter Hub`_ and the 107 | `Configurable HTTP Proxy`_ see the `Jupyter Hub technical overview`_. 108 | 109 | .. admonition:: Authentication Overview 110 | :class: hint 111 | 112 | .. _Security In Jupyter Hub: https://jupyterhub.readthedocs.io/en/stable/reference/websecurity.html 113 | 114 | See `Security In Jupyter Hub`_. 115 | 116 | .. image:: img/gui-arch-multi-user.svg 117 | :width: 100% 118 | -------------------------------------------------------------------------------- /src/reference/changes/analysis_view.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/analysis_view.gif -------------------------------------------------------------------------------- /src/reference/changes/cylc-completion.bash.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/cylc-completion.bash.gif -------------------------------------------------------------------------------- /src/reference/changes/cylc-graph-group-by-cycle-point.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/cylc-graph-group-by-cycle-point.png -------------------------------------------------------------------------------- /src/reference/changes/cylc-graph.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/cylc-graph.gif -------------------------------------------------------------------------------- /src/reference/changes/cylc-gui-info-view.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/cylc-gui-info-view.gif -------------------------------------------------------------------------------- /src/reference/changes/cylc-set-outputs.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/cylc-set-outputs.gif -------------------------------------------------------------------------------- /src/reference/changes/cylc-set.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/cylc-set.gif -------------------------------------------------------------------------------- /src/reference/changes/edit-runtime-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/edit-runtime-screenshot.png -------------------------------------------------------------------------------- /src/reference/changes/gantt_view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/gantt_view.png -------------------------------------------------------------------------------- /src/reference/changes/gui-n-window-selector.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/gui-n-window-selector.gif -------------------------------------------------------------------------------- /src/reference/changes/log-view-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/log-view-screenshot.png -------------------------------------------------------------------------------- /src/reference/changes/time_series.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/time_series.png -------------------------------------------------------------------------------- /src/reference/changes/tui-1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/tui-1.gif -------------------------------------------------------------------------------- /src/reference/changes/tui-external-editor.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/tui-external-editor.gif -------------------------------------------------------------------------------- /src/reference/changes/ui-view-selector.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/ui-view-selector.jpg -------------------------------------------------------------------------------- /src/reference/changes/ui-workspace-tabs.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/ui-workspace-tabs.gif -------------------------------------------------------------------------------- /src/reference/changes/vip-vr.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/reference/changes/vip-vr.gif -------------------------------------------------------------------------------- /src/reference/config/file-format.rst: -------------------------------------------------------------------------------- 1 | .. _file-format: 2 | 3 | The .cylc File Format 4 | ===================== 5 | 6 | Cylc global and workflow configuration files are written in a nested 7 | `INI`_-based format. 8 | 9 | .. _syntax: 10 | 11 | Syntax 12 | ------ 13 | 14 | Comments 15 | Comments follow a ``#`` character. 16 | 17 | Settings 18 | Configuration items (settings) are written as ``key = value`` pairs, and can 19 | be contained within sections. Setting names (the keys) may contain spaces. 20 | 21 | String Values 22 | Quoting single-line string values is optional: 23 | 24 | .. code-block:: cylc 25 | 26 | [animals] 27 | cat = dusty 28 | dog = "fido" # or single quotes: 'fido' 29 | 30 | Multiline string values must be triple-quoted: 31 | 32 | .. code-block:: cylc 33 | 34 | [song] 35 | lyrics = """ # (or triple single-quotes: '''value''') 36 | No stop signs 37 | Speed limit 38 | Nobody's gonna slow me down 39 | """ 40 | 41 | List Values 42 | List values are comma-separated: 43 | 44 | .. code-block:: cylc 45 | 46 | animals = dusty, fido, cujo 47 | 48 | 49 | Boolean Values 50 | Booleans are capitalized: 51 | 52 | .. code-block:: cylc 53 | 54 | ice cream is good = True # or False 55 | 56 | 57 | Sections and Sub-sections 58 | Settings have a level-dependent number of square brackets: 59 | 60 | .. code-block:: cylc 61 | 62 | [section] 63 | [[sub-section]] 64 | [[[sub-sub-section]]] 65 | 66 | 67 | Indentation 68 | It is advisable to indent sections and subsections, for clarity. However, 69 | Cylc ignores indentation, so this: 70 | 71 | .. code-block:: cylc 72 | 73 | [section] 74 | a = A 75 | [[sub-section]] 76 | b = B 77 | b = C # WARNING: this is still in sub-section! 78 | 79 | is equivalent to this: 80 | 81 | .. code-block:: cylc 82 | 83 | [section] 84 | a = A 85 | [[sub-section]] 86 | b = C 87 | 88 | 89 | Duplicate Sections and Items 90 | Duplicate sections get merged into one. Duplicate settings overwrite 91 | previously defined values. So this: 92 | 93 | .. code-block:: cylc 94 | 95 | [animals] 96 | cat = fluffy 97 | [animals] 98 | dog = fido 99 | cat = dusty 100 | 101 | is equivalent to this: 102 | 103 | .. code-block:: cylc 104 | 105 | [animals] 106 | cat = dusty 107 | dog = fido 108 | 109 | 110 | The only exception to this rule is :term:`graph strings `, 111 | which get merged. So these graph strings: 112 | 113 | .. code-block:: cylc-graph 114 | 115 | R1 = "foo => bar" 116 | R1 = "foo => baz" 117 | 118 | merge to this: 119 | 120 | .. code-block:: cylc-graph 121 | 122 | R1 = "foo => bar & baz" 123 | 124 | 125 | Continuation lines 126 | If necessary, you can continue on the next line after a backslash character: 127 | 128 | .. code-block:: cylc 129 | 130 | verse = "the quick \ 131 | brown fox" 132 | 133 | However, backslash line continuation is fragile (trailing invisible 134 | whitespace breaks it). Long :term:`graph strings ` strings 135 | should be split on graph symbols instead: 136 | 137 | .. code-block:: cylc-graph 138 | 139 | R1 = """ 140 | (foo & bar ) | 141 | baz => 142 | qux 143 | """ 144 | # Equivalent to: 145 | R1 = """ 146 | (foo & bar ) | baz => qux 147 | """ 148 | 149 | Include-files 150 | ``flow.cylc`` fragments can be included verbatim with the ``%include`` 151 | directive. Include-files can be included multiple times, and even nested. 152 | Include-file paths should relative to the ``flow.cylc`` location: 153 | 154 | .. code-block:: 155 | 156 | %include "inc/site-a.cylc" 157 | 158 | :ref:`Jinja2's ` template inclusion mechanism can be used with Cylc 159 | too. 160 | 161 | .. _config_item_shorthand: 162 | 163 | Shorthand 164 | --------- 165 | 166 | .. include:: shorthand.rst 167 | 168 | .. code-block:: cylc 169 | 170 | # This: 171 | # [runtime][task-a][environment]FOO = foo 172 | # Means: 173 | [runtime] 174 | [[task-a]] 175 | [[[environment]]] 176 | FOO = foo 177 | -------------------------------------------------------------------------------- /src/reference/config/global.rst: -------------------------------------------------------------------------------- 1 | .. _global-configuration: 2 | 3 | Global Configuration 4 | ==================== 5 | 6 | Cylc global configurations use the :ref:`.cylc file format `. 7 | 8 | .. auto-cylc-conf:: cylc.flow.cfgspec.globalcfg.SPEC 9 | -------------------------------------------------------------------------------- /src/reference/config/index.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | file-format 8 | workflow 9 | global 10 | ui-server 11 | sharing-access-to-workflows 12 | types 13 | writing-platform-configs 14 | -------------------------------------------------------------------------------- /src/reference/config/shorthand.rst: -------------------------------------------------------------------------------- 1 | We often use a compact single-line notation to refer to nested config items: 2 | 3 | ``[section]`` 4 | An entire section. 5 | ``[section]setting`` 6 | A setting within a section. 7 | ``[section]setting=value`` 8 | The value of a setting within a section. 9 | ``[section][sub-section]another-setting`` 10 | A setting within a sub-section. 11 | 12 | In the file, however, section headings need additional brackets at each level. 13 | -------------------------------------------------------------------------------- /src/reference/config/types.rst: -------------------------------------------------------------------------------- 1 | Cylc Configuration Data Types 2 | ============================= 3 | 4 | The Cylc configuration file format supports the following data types. 5 | 6 | .. auto-cylc-type:: 7 | cylc.flow.parsec.validate.ParsecValidator.V_TYPE_HELP 8 | cylc.flow.parsec.validate.CylcConfigValidator.V_TYPE_HELP 9 | -------------------------------------------------------------------------------- /src/reference/config/ui-server.rst: -------------------------------------------------------------------------------- 1 | .. _UI_Server_config: 2 | 3 | UI Server Configuration 4 | ======================= 5 | 6 | Cylc UI Server can be configured using a ``jupyter_config.py``. 7 | 8 | Site level configuration, such as ``c.CylcUIServer.site_authorization`` should 9 | be defined in ``/etc/cylc/uiserver/jupyter_config.py``, or, alternatively, the 10 | environment variable ``CYLC_SITE_CONF_PATH``. 11 | User level configuration should be located in ``~/.cylc/uiserver/jupyter_config.py``. 12 | 13 | .. automodule:: cylc.uiserver.app 14 | 15 | .. autoconfigurable:: cylc.uiserver.app.CylcUIServer 16 | :inherited-members: False 17 | -------------------------------------------------------------------------------- /src/reference/config/workflow.rst: -------------------------------------------------------------------------------- 1 | .. _workflow-configuration: 2 | 3 | Workflow Configuration 4 | ====================== 5 | 6 | Cylc workflow configurations use the :ref:`.cylc file format `. 7 | 8 | .. auto-cylc-conf:: cylc.flow.cfgspec.workflow.SPEC 9 | -------------------------------------------------------------------------------- /src/reference/dev-history-major-changes.rst: -------------------------------------------------------------------------------- 1 | Cylc Development History - Major Changes 2 | ======================================== 3 | 4 | - **pre-cylc-3** 5 | 6 | - early versions focused on the new 7 | scheduling algorithm. A suite was a collection of "task definition files" 8 | that encoded the prerequisites and outputs of each task, 9 | exposing Cylc's self-organising nature. Tasks could be transferred 10 | from one suite to another by simply copying their taskdef files over 11 | and checking prerequisite and output consistency. Global suite 12 | structure was not easy to discern until run time (although cylc-2 13 | could generate resolved run time dependency graphs). 14 | 15 | - **cylc-3** 16 | 17 | - a new suite design interface: dependency graph and task runtime properties 18 | defined in a single structured, validated, configuration file 19 | - graphical user interface 20 | - suite graphing 21 | 22 | - **cylc-4** 23 | 24 | - refined and organized the suite config file structure 25 | - task runtime properties defined by an efficient inheritance hierarchy 26 | - support for the Jinja2 template processor in suite configurations 27 | 28 | - **cylc-5** 29 | 30 | - multi-threading for continuous network request handling and job submission 31 | - more task states to distinguish job submission from execution 32 | - dependence between suites via new run databases 33 | - polling and killing of real jobs 34 | - polling as task communications option 35 | 36 | - **cylc-6** 37 | 38 | - specification of all datetimes and cycling suites via ISO8601 39 | datetimes, durations, and recurrence expressions 40 | - integer cycling 41 | - a multi-process pool to execute job submissions, event handlers, and poll 42 | and kill commands 43 | 44 | - **cylc-7** 45 | 46 | - Replaced the Pyro communications layer with RESTful HTTPS 47 | - Removed deprecated pre cylc-6 syntax and features 48 | 49 | - **cylc-8** 50 | 51 | - Migrated to Python 3 52 | - New architecture, network layers, security 53 | - New web UI 54 | - New efficient scheduling algorithm 55 | - Removed deprecated pre cylc-6 syntax and features 56 | -------------------------------------------------------------------------------- /src/reference/environments/conda.rst: -------------------------------------------------------------------------------- 1 | .. _conda environments: 2 | 3 | Conda Environments 4 | ================== 5 | 6 | Cylc consists of multiple components. 7 | 8 | This Conda environment will provide you with the "recommended" installation of 9 | all components: 10 | 11 | .. literalsubinclude:: envs/cylc-all.yml 12 | :language: YAML 13 | :substitutions: 14 | 15 | The following sections outline alternatives for finer control over installation. 16 | 17 | 18 | Cylc Flow 19 | --------- 20 | 21 | The simplest Cylc installation consists only of `Cylc Flow`_: 22 | 23 | .. literalsubinclude:: envs/cylc-flow.yml 24 | :language: YAML 25 | :substitutions: 26 | 27 | .. tip:: 28 | 29 | We suggest installing `Cylc Flow`_ at a "minor" version (e.g. ``8.1`` rather 30 | than ``8.1.2``) to pick up the latest "maintenance" release. 31 | 32 | If you do not specify your Python version you will be provided with the most 33 | recent compatible one. 34 | 35 | If you are installing Cylc on multiple machines across a network it is 36 | advisable to keep the Python versions consistent. 37 | 38 | You can do this by adding Python into the environment at a particular version: 39 | 40 | .. literalsubinclude:: envs/cylc-flow-with-python.yml 41 | :language: YAML 42 | :substitutions: 43 | 44 | `Cylc Flow`_ provides a cut-down package containing only the bare essentials 45 | called ``cylc-flow-base``. 46 | 47 | This may be useful for installing onto job hosts where client-facing extra 48 | features are not of interest: 49 | 50 | .. literalsubinclude:: envs/cylc-flow-base.yml 51 | :language: YAML 52 | :substitutions: 53 | 54 | 55 | Cylc UIServer 56 | ------------- 57 | 58 | The default `Cylc UIServer`_ package comes with `Jupyter Hub`_ and the 59 | `Configurable HTTP Proxy`_ (required by `Jupyter Hub`_) bundled: 60 | 61 | .. literalsubinclude:: envs/cylc-uiserver.yml 62 | :language: YAML 63 | :substitutions: 64 | 65 | .. tip:: 66 | 67 | We suggest *not* specifying the version of other Cylc components to 68 | (i.e. `Cylc UIServer`_ and `Cylc Rose`_). This will pick up the most recent 69 | version compatible with the specified `Cylc Flow`_ version. 70 | 71 | `Jupyter Hub`_ is only required for multi-user setups, the `Cylc UIServer`_ 72 | can be run as a standalone application. 73 | 74 | To exclude `Jupyter Hub`_ from the installation use ``cylc-uiserver-base``: 75 | 76 | .. literalsubinclude:: envs/cylc-uiserver-without-jupyterhub.yml 77 | :language: YAML 78 | :substitutions: 79 | 80 | If you want to use `Jupyter Hub`_ with an alternative proxy use 81 | ``cylc-uiserver-hub-base`` (this depends on ``jupyterhub-base``, see the 82 | `Jupyter Hub`_ documentation for details): 83 | 84 | .. literalsubinclude:: envs/cylc-uiserver-with-traefik-proxy.yml 85 | :language: YAML 86 | :substitutions: 87 | 88 | 89 | Cylc Rose 90 | --------- 91 | 92 | For working with Rose add ``metomi-rose`` and ``cylc-rose`` (the bridge 93 | between Cylc & Rose): 94 | 95 | .. literalsubinclude:: envs/cylc-rose.yml 96 | :language: YAML 97 | :substitutions: 98 | 99 | We will look at providing more installation options for Rose in the near 100 | future. 101 | 102 | 103 | Adding Cylc To Your Conda Package 104 | --------------------------------- 105 | 106 | If you want to publish a package (e.g. to Conda Forge) that depends on Cylc, 107 | consider using the minimal package (e.g. ``cylc-flow-base``) as a dependency 108 | to allow the installer to maintain flexibility over the installation. 109 | 110 | 111 | Working With Other Conda Channels 112 | --------------------------------- 113 | 114 | Cylc projects are published to Conda Forge and the above environments install 115 | all dependencies from Conda Forge. 116 | 117 | If you want to install other dependencies (e.g. Python), from other channels 118 | (e.g. Anaconda), list the dependency explicitly and place the channel *above* 119 | the Conda Forge channel. 120 | 121 | .. literalsubinclude:: envs/cylc-flow-anaconda.yml 122 | :language: YAML 123 | :substitutions: 124 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-all.yml: -------------------------------------------------------------------------------- 1 | name: cylc-|version| 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - cylc-flow =|version| 6 | - cylc-uiserver 7 | - cylc-rose 8 | - metomi-rose 9 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-flow-anaconda.yml: -------------------------------------------------------------------------------- 1 | name: cylc-flow 2 | channels: 3 | - defaults 4 | - conda-forge 5 | dependencies: 6 | - python 7 | - cylc-flow =|version| 8 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-flow-base.yml: -------------------------------------------------------------------------------- 1 | name: cylc-flow-base 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - cylc-flow-base =|version| 6 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-flow-with-python.yml: -------------------------------------------------------------------------------- 1 | name: cylc-flow-with-python-3.7 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python =3.7 6 | - cylc-flow =|version| 7 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-flow.yml: -------------------------------------------------------------------------------- 1 | name: cylc-flow 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - cylc-flow =|version| 6 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-rose.yml: -------------------------------------------------------------------------------- 1 | name: cylc-rose 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - cylc-flow =|version| 6 | - cylc-rose 7 | - metomi-rose 8 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-uiserver-with-traefik-proxy.yml: -------------------------------------------------------------------------------- 1 | name: cylc-uiserver-with-traefik-proxy 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - cylc-flow =|version| 6 | - cylc-uiserver-hub-base 7 | - pip 8 | - pip: 9 | - jupyterhub-traefik-proxy 10 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-uiserver-without-jupyterhub.yml: -------------------------------------------------------------------------------- 1 | name: cylc-uiserver-without-jupyterhub 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - cylc-flow =|version| 6 | - cylc-uiserver-base 7 | -------------------------------------------------------------------------------- /src/reference/environments/envs/cylc-uiserver.yml: -------------------------------------------------------------------------------- 1 | name: cylc-uiserver 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - cylc-flow =|version| 6 | - cylc-uiserver 7 | -------------------------------------------------------------------------------- /src/reference/index.rst: -------------------------------------------------------------------------------- 1 | .. _reference: 2 | 3 | Reference 4 | ========= 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | changes 11 | config/index 12 | job-script-vars/index 13 | architecture/index 14 | api/index 15 | environments/conda.rst 16 | dev-history-major-changes 17 | -------------------------------------------------------------------------------- /src/reference/job-script-vars/index.rst: -------------------------------------------------------------------------------- 1 | Job Script Environment Variables 2 | ================================ 3 | 4 | These environment variables provided by the :term:`scheduler` are available 5 | to Cylc job scripts at run time: 6 | 7 | .. literalinclude:: var-list.txt 8 | :language: sub 9 | 10 | 11 | .. seealso:: 12 | 13 | :ref:`Task Job Script Variables` 14 | -------------------------------------------------------------------------------- /src/reference/job-script-vars/var-list.txt: -------------------------------------------------------------------------------- 1 | CYLC_VERSION # Version of cylc installation used 2 | CYLC_VERBOSE # Verbose mode, true or false 3 | CYLC_DEBUG # Debug mode (even more verbose), true or false 4 | 5 | CYLC_CYCLING_MODE # Cycling mode, e.g. gregorian 6 | ISODATETIMECALENDAR # Calendar mode for the `isodatetime` command, 7 | # defined with the value of CYLC_CYCLING_MODE 8 | # when in any datetime cycling mode 9 | 10 | CYLC_WORKFLOW_FINAL_CYCLE_POINT # Final cycle point 11 | CYLC_WORKFLOW_INITIAL_CYCLE_POINT # Initial cycle point 12 | CYLC_WORKFLOW_ID # Workflow ID 13 | # e.g. "a/b/c/run1" 14 | CYLC_WORKFLOW_NAME # Workflow ID with the run name removed 15 | # (use CYLC_WORKFLOW_ID for most purposes) 16 | # e.g. "a/b/c" 17 | CYLC_WORKFLOW_NAME_BASE # The basename of the workflow name 18 | # (use CYLC_WORKFLOW_ID for most purposes) 19 | # e.g. "c" 20 | CYLC_UTC # UTC mode, True or False 21 | TZ # Set to "UTC" in UTC mode or not defined 22 | 23 | CYLC_WORKFLOW_RUN_DIR # Location of the run directory in 24 | # job host, e.g. ~/cylc-run/foo 25 | CYLC_WORKFLOW_HOST # Host running the workflow process 26 | CYLC_WORKFLOW_OWNER # User ID running the workflow process 27 | 28 | CYLC_WORKFLOW_SHARE_DIR # Workflow (or task!) shared directory 29 | CYLC_WORKFLOW_UUID # Workflow UUID string 30 | CYLC_WORKFLOW_WORK_DIR # Workflow work directory 31 | 32 | CYLC_TASK_JOB # Job identifier expressed as 33 | # CYCLE-POINT/TASK-NAME/SUBMIT-NUMBER 34 | # e.g. 20110511T1800Z/t1/01 35 | 36 | CYLC_TASK_CYCLE_POINT # Cycle point, e.g. 20110511T1800Z 37 | ISODATETIMEREF # Reference time for the `isodatetime` command, 38 | # defined with the value of CYLC_TASK_CYCLE_POINT 39 | # when in any datetime cycling mode 40 | 41 | CYLC_TASK_NAME # Job's task name, e.g. t1 42 | CYLC_TASK_ID # Task instance identifier CYCLE-POINT/TASK-NAME 43 | # e.g. 20110511T1800Z/t1 44 | 45 | CYLC_TASK_SUBMIT_NUMBER # Job's submit number, e.g. 1, 46 | # increments with every submit 47 | CYLC_TASK_TRY_NUMBER # Number of execution tries, e.g. 1 48 | # increments with automatic execution retry delays. 49 | CYLC_TASK_FLOW_NUMBERS # Flows this task belongs to, e.g. 1,2 50 | 51 | CYLC_TASK_LOG_DIR # Location of the job log directory 52 | # e.g. ~/cylc-run/foo/log/job/20110511T1800Z/t1/01/ 53 | CYLC_TASK_LOG_ROOT # The job script path 54 | # e.g. ~/cylc-run/foo/log/job/20110511T1800Z/t1/01/job 55 | CYLC_TASK_WORK_DIR # Location of task work directory (see below) 56 | # e.g. ~/cylc-run/foo/work/20110511T1800Z/t1 57 | CYLC_TASK_SHARE_CYCLE_DIR # The share/cycle/ directory 58 | # for this task. 59 | 60 | CYLC_TASK_NAMESPACE_HIERARCHY # Linearised family namespace of the task, 61 | # e.g. root postproc t1 62 | 63 | CYLC_TASK_COMMS_METHOD # Set to "ssh" if communication method is "ssh" 64 | CYLC_TASK_SSH_LOGIN_SHELL # With "ssh" communication, if set to "True", 65 | # use login shell on workflow host 66 | -------------------------------------------------------------------------------- /src/tutorial/furthertopics/index.rst: -------------------------------------------------------------------------------- 1 | .. _tutorials.furthertopics: 2 | 3 | Further Topics 4 | ============== 5 | 6 | This section looks at further topics in cylc. 7 | 8 | .. toctree:: 9 | :name: cylc-further-topics 10 | :maxdepth: 1 11 | 12 | clock-triggered-tasks 13 | broadcast 14 | family-triggers 15 | inheritance 16 | queues 17 | retries 18 | message-triggers 19 | -------------------------------------------------------------------------------- /src/tutorial/furthertopics/queues.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial.furthertopics.queues: 2 | 3 | Queues 4 | ====== 5 | 6 | Queues are used to put a limit on the number of tasks that will be active at 7 | any one time, even if their dependencies are satisfied. This avoids swamping 8 | systems with too many tasks at once. 9 | 10 | 11 | Example 12 | ------- 13 | 14 | In this example, our workflow manages a particularly understaffed restaurant. 15 | 16 | Create a new workflow called ``queues-tutorial``:: 17 | 18 | mkdir -p ~/cylc-src/queues-tutorial 19 | cd ~/cylc-src/queues-tutorial 20 | 21 | And paste the following into :cylc:conf:`flow.cylc`: 22 | 23 | .. code-block:: cylc 24 | 25 | [scheduling] 26 | [[graph]] 27 | R1 = """ 28 | open_restaurant => steak1 & steak2 & pasta1 & pasta2 & pasta3 & \ 29 | pizza1 & pizza2 & pizza3 & pizza4 30 | steak1 => ice_cream1 31 | steak2 => cheesecake1 32 | pasta1 => ice_cream2 33 | pasta2 => sticky_toffee1 34 | pasta3 => cheesecake2 35 | pizza1 => ice_cream3 36 | pizza2 => ice_cream4 37 | pizza3 => sticky_toffee2 38 | pizza4 => ice_cream5 39 | """ 40 | 41 | [runtime] 42 | [[open_restaurant]] 43 | [[MAINS]] 44 | [[DESSERT]] 45 | [[steak1,steak2,pasta1,pasta2,pasta3,pizza1,pizza2,pizza3,pizza4]] 46 | inherit = MAINS 47 | [[ice_cream1,ice_cream2,ice_cream3,ice_cream4,ice_cream5]] 48 | inherit = DESSERT 49 | [[cheesecake1,cheesecake2,sticky_toffee1,sticky_toffee2]] 50 | inherit = DESSERT 51 | 52 | .. note:: 53 | 54 | In graph sections, lines can be split on ``&``, i.e. the 55 | following two examples are equivalent: 56 | 57 | .. code-block:: cylc-graph 58 | 59 | foo => bar & 60 | baz 61 | 62 | .. code-block:: cylc-graph 63 | 64 | foo => bar & baz 65 | 66 | ``|`` (or), and ``=>`` act in the same way. 67 | 68 | Validate, install and play the workflow:: 69 | 70 | cylc validate . 71 | cylc install --run-name without-queues 72 | 73 | Look at the workflow with :ref:`tutorial.gui` or :ref:`tutorial.tui` 74 | 75 | Play the workflow, either from the GUI or the command line:: 76 | 77 | cylc play queues-tutorial/without-queues 78 | 79 | You will see that all the ``steak``, ``pasta``, and ``pizza`` tasks are run 80 | at once, swiftly followed by all the ``ice_cream``, ``cheesecake``, 81 | ``sticky_toffee`` tasks as the customers order from the dessert menu. 82 | 83 | 84 | This will overwhelm our restaurant staff! The chef responsible for ``MAINS`` 85 | can only handle 3 tasks at any given time, and the ``DESSERT`` chef can only 86 | handle 2. 87 | 88 | We need to add some queues. Add a ``[queues]`` section to the ``[scheduling]`` 89 | section like so: 90 | 91 | .. code-block:: cylc 92 | 93 | [scheduling] 94 | [[queues]] 95 | [[[mains_chef_queue]]] 96 | limit = 3 # Only 3 mains dishes at one time. 97 | members = MAINS 98 | [[[dessert_chef_queue]]] 99 | limit = 2 # Only 2 dessert dishes at one time. 100 | members = DESSERT 101 | 102 | Install and play the workflow:: 103 | 104 | cylc validate . 105 | cylc install --run-name tutorial-with-queues 106 | 107 | Play the workflow using the :ref:`GUI ` 108 | or :ref:`TUI `. 109 | 110 | You should see that there are now never more than 3 active ``MAINS`` tasks 111 | running and never more than 2 active ``DESSERT`` tasks running. 112 | 113 | The customers will obviously have to wait! 114 | 115 | 116 | Further Reading 117 | --------------- 118 | 119 | For more information, see the `Cylc User Guide`_. 120 | -------------------------------------------------------------------------------- /src/tutorial/img/cylc-graph-reversible.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 15 | 16 | 18 | 19 | _anonymous_0 20 | cluster_1 21 | 22 | 23 | cluster_2 24 | 25 | 26 | 27 | 1/foo 28 | 29 | foo 30 | 31 | 32 | 1/baz 33 | 34 | baz 35 | 36 | 37 | 1/foo->1/baz 38 | 39 | 40 | 41 | 42 | 1/bar 43 | 44 | bar 45 | 46 | 47 | 1/bar->1/baz 48 | 49 | 50 | 51 | 52 | 2/bar 53 | 54 | bar 55 | 56 | 57 | 2/baz 58 | 59 | baz 60 | 61 | 62 | 2/foo->2/baz 63 | 64 | 65 | 66 | 67 | 2/foo 68 | 69 | foo 70 | 71 | 72 | 2/bar->2/baz 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /src/tutorial/img/cylc-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/tutorial/img/cylc-graph.png -------------------------------------------------------------------------------- /src/tutorial/img/cylc-gui-scan-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/tutorial/img/cylc-gui-scan-view.png -------------------------------------------------------------------------------- /src/tutorial/img/cylc-gui-table-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/tutorial/img/cylc-gui-table-view.png -------------------------------------------------------------------------------- /src/tutorial/img/cylc-gui-tree-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/tutorial/img/cylc-gui-tree-view.png -------------------------------------------------------------------------------- /src/tutorial/img/cylc-gui-views-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/tutorial/img/cylc-gui-views-button.png -------------------------------------------------------------------------------- /src/tutorial/img/cylc-tools.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/tutorial/img/cylc-tools.png -------------------------------------------------------------------------------- /src/tutorial/index.rst: -------------------------------------------------------------------------------- 1 | .. _Cylc Tutorial: 2 | 3 | Tutorial 4 | ======== 5 | 6 | This tutorial shows how to write and run basic Cylc workflows. 7 | 8 | .. toctree:: 9 | :name: cylc-tutorial 10 | :maxdepth: 2 11 | 12 | introduction 13 | scheduling/index 14 | runtime/index 15 | furthertopics/index 16 | -------------------------------------------------------------------------------- /src/tutorial/introduction.rst: -------------------------------------------------------------------------------- 1 | .. _cylc-introduction: 2 | 3 | Introduction 4 | ============ 5 | 6 | .. _cylc-what-is-a-workflow: 7 | 8 | What Is A Workflow? 9 | ------------------- 10 | 11 | .. epigraph:: 12 | 13 | A workflow consists of an orchestrated and repeatable pattern of business 14 | activity enabled by the systematic organization of resources into processes 15 | that transform materials, provide services, or process information. 16 | 17 | -- Wikipedia 18 | 19 | .. ifnotslides:: 20 | 21 | In research, business and other fields we may need to repeat processes in 22 | the course of our work. At its simplest a workflow is a set of steps that 23 | must be followed in a particular order to achieve some end goal. 24 | 25 | We can represent each "step" in a workflow as a node in a graph, and the 26 | order with arrows between them. 27 | 28 | .. nextslide:: 29 | 30 | .. digraph:: bakery 31 | :align: center 32 | 33 | "purchase ingredients" -> "make dough" -> "bake bread" -> "sell bread" 34 | "bake bread" -> "clean oven" 35 | "pre-heat oven" -> "bake bread" 36 | 37 | 38 | .. _cylc-what-is-cylc: 39 | 40 | What Is Cylc? 41 | ------------- 42 | 43 | .. ifnotslides:: 44 | 45 | Cylc (pronounced silk) is a workflow engine, a system that automatically 46 | executes tasks according to schedules and dependencies. 47 | 48 | In a Cylc workflow each step is a computational task that runs a script or 49 | application of some kind. Cylc runs each task as soon as it is appropriate 50 | to do so. 51 | 52 | .. minicylc:: 53 | :align: center 54 | :theme: demo 55 | 56 | a => b => c 57 | b => d => f 58 | e => f 59 | 60 | .. nextslide:: 61 | 62 | Cylc can automatically: 63 | 64 | - Submit tasks across computer systems and resource managers. 65 | - Recover from failures. 66 | - Repeat workflows. 67 | 68 | .. ifnotslides:: 69 | 70 | Cylc was originally developed at NIWA (The National Institute of Water and 71 | Atmospheric Research, New Zealand) for running their weather forecasting 72 | workflows. It is now developed by an international partnership including 73 | NIWA, the Met Office (UK), and members of the Unified Model Consortium. 74 | Though initially developed for meteorological purposes Cylc is a general 75 | purpose tool as applicable in business as it is in scientific research. 76 | 77 | .. nextslide:: 78 | 79 | .. ifslides:: 80 | 81 | * Originally developed at NIWA (New Zealand) 82 | * Now developed by an international partnership including the 83 | Met Office (UK). 84 | * General purpose tool as applicable in business as in 85 | scientific research. 86 | 87 | .. nextslide:: 88 | 89 | Cylc provides a variety of command line and GUI tools for visualising, 90 | monitoring, and controlling workflows. The Cylc TUI (Terminal 91 | User Interface), web GUI, and ``cylc scan`` (bottom left) are shown below. 92 | 93 | .. image:: /tutorial/img/cylc-tools.png 94 | :alt: A screenshot of several Cylc tools. 95 | 96 | .. nextslide:: 97 | 98 | .. ifslides:: 99 | 100 | :ref:`tutorial-cylc-graphing` 101 | -------------------------------------------------------------------------------- /src/tutorial/runtime/index.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial-runtime: 2 | 3 | Runtime 4 | ======= 5 | 6 | This section covers: 7 | 8 | * Associating workflow tasks with executable applications (scripts and 9 | programs). 10 | * Providing executables with runtime configurations, within the workflow. 11 | * Running Cylc workflows. 12 | 13 | .. toctree:: 14 | :name: rug-runtime-toc 15 | :maxdepth: 2 16 | 17 | introduction 18 | runtime-configuration 19 | configuration-consolidation/index 20 | summary 21 | -------------------------------------------------------------------------------- /src/tutorial/runtime/summary.rst: -------------------------------------------------------------------------------- 1 | Tutorial Summary 2 | ================ 3 | 4 | You've made it to the end of the Cylc tutorial! 5 | 6 | The Cylc tutorial aims to introduce you to the main concepts in Cylc to 7 | prepare you for working with and writing Cylc workflows. 8 | But there is so much more we didn't have time for. 9 | 10 | :ref:`Rose Tutorial ` 11 | `Rose`_ is a tool for creating configurable applications. It's often used 12 | with Cylc to configure more complex tasks, or even the workflow itself. 13 | 14 | Rose configurations can have metadata, support validation and can be edited 15 | using a GUI. 16 | 17 | You might want to try the :ref:`Rose Tutorial` if these things are of interest. 18 | :ref:`Further Topics ` 19 | There some extra tutorials which cover some of the things the main tutorial 20 | doesn't in the :ref:`further topics section `. 21 | `Discourse`_ 22 | If you get stuck, encounter an issue, have a question, or just fancy a chat 23 | about Cylc, feel free to reach out to the Cylc community on our `Discourse`_ 24 | forum. 25 | 26 | We also make announcements (e.g. new Cylc releases) on the forum, so it's a 27 | good to keep an eye on it. 28 | :ref:`User Guide ` 29 | There is a comprehensive :ref:`user guide` which goes into Cylc's 30 | capabilities in detail. 31 | :ref:`Workflow Design Guide ` 32 | This covers recommended code style and best practice. 33 | :ref:`Reference ` 34 | The reference section contains all sorts of technical details. 35 | 36 | It also lists all of the configurations Cylc supports for workflow 37 | definition :cylc:conf:`flow.cylc`, and for site/user setup 38 | :cylc:conf:`global.cylc`. 39 | -------------------------------------------------------------------------------- /src/tutorial/scheduling/further-scheduling.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial-cylc-further-scheduling: 2 | 3 | Further Scheduling 4 | ================== 5 | 6 | .. admonition:: Aims 7 | :class: aims 8 | 9 | | You should be aware of some more advanced scheduling features: 10 | | ✅ Task state qualifiers. 11 | | ✅ Clock triggers. 12 | | ✅ Alternative calendars. 13 | 14 | 15 | .. _tutorial-qualifiers: 16 | 17 | Qualifiers 18 | ---------- 19 | 20 | .. ifnotslides:: 21 | 22 | So far we have written dependencies like ``foo => bar``. This is, in fact, 23 | shorthand for ``foo:succeed => bar``. It means that the task ``bar`` will run 24 | once ``foo`` has finished successfully. If ``foo`` were to fail then ``bar`` 25 | would not run. We will talk more about these :term:`task states ` 26 | in the :ref:`Runtime Section `. 27 | 28 | We refer to the ``:succeed`` descriptor as a :term:`qualifier`. 29 | There are qualifiers for different :term:`task states ` e.g: 30 | 31 | .. ifslides:: 32 | 33 | .. code-block:: cylc-graph 34 | 35 | foo => bar 36 | foo:succeed => bar 37 | foo:fail => bar 38 | 39 | ``:start`` 40 | When the task has started running. 41 | ``:fail`` 42 | When the task finishes if it fails (produces non-zero return code). 43 | ``:finish`` 44 | When the task has completed (either succeeded or failed). 45 | 46 | .. nextslide:: 47 | 48 | It is also possible to create your own :term:`qualifiers ` 49 | to handle events within your code (custom outputs). 50 | 51 | .. ifnotslides:: 52 | 53 | *For more information see the* `Cylc User Guide`_. 54 | 55 | 56 | .. _tutorial-clock-triggers: 57 | 58 | Clock Triggers 59 | -------------- 60 | 61 | .. ifnotslides:: 62 | 63 | In Cylc, :term:`cycle points ` are just task labels. Tasks are 64 | triggered when their dependencies are met, regardless of cycle point. 65 | But *clock triggers* can be used to force tasks to wait for a particular 66 | real time, relative to their cycle point, before running. 67 | This is necessary for certain operational and monitoring systems, e.g. for 68 | tasks that process real-time data. 69 | 70 | For example in the following workflow the cycle ``2000-01-01T12Z`` will wait 71 | until 11:00 on the 1st of January 2000 before running: 72 | 73 | .. code-block:: cylc 74 | 75 | [scheduling] 76 | initial cycle point = 2000-01-01T00Z 77 | [[xtriggers]] 78 | PT1H_trigger = wall_clock(offset=-PT1H) 79 | [[graph]] 80 | # "daily" will run, at the earliest, one hour before midday. 81 | T12 = @PT1H_trigger => daily 82 | 83 | .. tip:: 84 | 85 | See the :ref:`tutorial-cylc-clock-trigger` tutorial for more information. 86 | 87 | 88 | Alternative Calendars 89 | --------------------- 90 | 91 | .. ifnotslides:: 92 | 93 | By default Cylc uses the Gregorian calendar for :term:`datetime cycling`, 94 | but it also supports: 95 | 96 | - Integer cycling. 97 | - 360-day calendar (12 months of 30 days each in a year). 98 | - 365-day calendar (never a leap year). 99 | - 366-day calendar (always a leap year). 100 | 101 | .. code-block:: cylc 102 | 103 | [scheduling] 104 | cycling mode = 360day 105 | 106 | .. ifnotslides:: 107 | 108 | .. seealso:: :cylc:conf:`[scheduling]cycling mode` 109 | 110 | .. nextslide:: 111 | 112 | .. ifslides:: 113 | 114 | Next section: :ref:`Runtime Introduction 115 | ` 116 | -------------------------------------------------------------------------------- /src/tutorial/scheduling/index.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial-scheduling: 2 | 3 | Scheduling 4 | ========== 5 | 6 | This section shows how to create basic Cylc workflows. 7 | 8 | .. toctree:: 9 | :name: rug-scheduling-toc 10 | :maxdepth: 2 11 | 12 | graphing 13 | integer-cycling 14 | datetime-cycling 15 | further-scheduling 16 | -------------------------------------------------------------------------------- /src/user-guide/examples/index.rst: -------------------------------------------------------------------------------- 1 | .. _user-guide.examples: 2 | 3 | Examples 4 | ======== 5 | 6 | These examples cover some of the main patterns for implementing Cylc workflows. 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | :glob: 11 | 12 | */index 13 | -------------------------------------------------------------------------------- /src/user-guide/index.rst: -------------------------------------------------------------------------------- 1 | .. _user guide: 2 | 3 | User Guide 4 | ========== 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | introduction 11 | writing-workflows/index 12 | installing-workflows 13 | running-workflows/index 14 | task-implementation/index 15 | interventions/index 16 | removing-workflows 17 | cheat-sheet 18 | examples/index 19 | troubleshooting 20 | -------------------------------------------------------------------------------- /src/user-guide/interventions/edit-a-tasks-configuration.gui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/edit-a-tasks-configuration.gui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/edit-the-workflow-configuration.tui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/edit-the-workflow-configuration.tui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/re-run-a-task.gui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/re-run-a-task.gui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/re-run-a-task.tui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/re-run-a-task.tui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/re-run-all-failed-tasks.gui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/re-run-all-failed-tasks.gui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/re-run-multiple-tasks.gui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/re-run-multiple-tasks.gui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/remove.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/remove.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/set-a-switch-task.gui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/set-a-switch-task.gui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/set-task-outputs.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/set-task-outputs.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/set-task-outputs.gui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/set-task-outputs.gui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/set-task-outputs.tui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/set-task-outputs.tui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/set-task-prerequisites.gui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/set-task-prerequisites.gui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/skip-cycle.gui.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/skip-cycle.gui.gif -------------------------------------------------------------------------------- /src/user-guide/interventions/trigger-while-paused.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cylc/cylc-doc/4c88e90189e856f6c9f590a6a4971a7d7a2b42ec/src/user-guide/interventions/trigger-while-paused.gif -------------------------------------------------------------------------------- /src/user-guide/introduction.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | 5 | What is a Workflow? 6 | ------------------- 7 | 8 | .. epigraph:: 9 | 10 | A workflow consists of an orchestrated and repeatable pattern of business 11 | activity enabled by the systematic organization of resources into processes 12 | that transform materials, provide services, or process information. 13 | 14 | -- Wikipedia 15 | 16 | 17 | .. tutorial:: What is a Workflow? 18 | 19 | 20 | What is Cylc? 21 | ------------- 22 | 23 | Cylc (pronounced silk) is a workflow engine, a system that automatically 24 | executes tasks according to schedules and dependencies. 25 | 26 | In a Cylc workflow each step is an application: an executable command, script, 27 | or program. Cylc runs each task as soon as it is appropriate to do so. 28 | 29 | 30 | .. tutorial:: What is Cylc? 31 | 32 | 33 | Cylc and Cycling Workflows 34 | -------------------------- 35 | 36 | A *cycling workflow* is a repetitive process involving many interdependent 37 | tasks. Cylc tasks wrap arbitrary applications: executable commands, 38 | scripts, or programs. Example use cases include: 39 | 40 | - Processing many similar datasets, through a pipeline or graph of tasks 41 | - Forecasting systems that generate new forecasts at regular intervals 42 | - Splitting a long model run and associated processing tasks into many smaller runs 43 | - Iterative tuning of model parameters by model, processing, and validation tasks 44 | 45 | Cycling systems were traditionally managed by repeat-running the single-cycle 46 | workflow, finishing each new cycle before starting the next. Sometimes, however, 47 | it can be much more efficient to run multiple cycles at once. Even in real time 48 | forecasting systems that normally have a gap between cycles waiting on new 49 | data, this can greatly speed catch up from delays or downtime. But it can't 50 | be done if the workflow manager has a global loop that handles only one cycle 51 | at a time and does not understand any intercycle dependence that may be present. 52 | 53 | .. important:: 54 | 55 | Cylc handles inter- and intra-cycle dependence equally, and it unrolls the 56 | cycle loop to create a single non-cycling workflow of repeating tasks, each 57 | with its own individual *cycle point*. 58 | 59 | .. image:: ../img/cycling.png 60 | :align: center 61 | 62 | This removes the artificial barrier between cycles. Cylc tasks can advance 63 | constrained only by their individual dependencies, for maximum concurrency 64 | across as well as within cycles. This allows fast catch-up from delays in 65 | real time systems, and sustained high throughput off the clock. 66 | 67 | 68 | How to Run User Guide Examples 69 | ------------------------------ 70 | 71 | Many Cylc concepts and features in this document are illustrated with minimal 72 | snippets of workflow configuration. Most of these can be turned into a complete 73 | workflow that you can actually run, with a few easy steps: 74 | 75 | - Add scheduling section headings, if missing, above the graph 76 | - Use ``[scheduler]allow implicit tasks = True`` to automatically create dummy definitions 77 | for each task 78 | - Configure the ``root`` task family to make the dummy jobs take a little time 79 | to run, so the workflow doesn't evolve too quickly 80 | - For cycling graphs, configure an initial cycle point to start at 81 | 82 | For example, here is a small cycling workflow graph: 83 | 84 | .. code-block:: cylc-graph 85 | 86 | # Avoid caffeine withdrawal 87 | PT6H = "grind_beans => make_coffee => drink_coffee" 88 | 89 | And here it is as a complete runnable workflow: 90 | 91 | .. code-block:: cylc 92 | 93 | [scheduler] 94 | allow implicit tasks = True 95 | [scheduling] 96 | initial cycle point = now 97 | [[graph]] 98 | # Avoid caffeine withdrawal 99 | PT6H = "grind_beans => make_coffee => drink_coffee" 100 | [runtime] 101 | [[root]] 102 | script = "sleep 10" 103 | -------------------------------------------------------------------------------- /src/user-guide/removing-workflows.rst: -------------------------------------------------------------------------------- 1 | .. _Removing-workflows: 2 | 3 | Removing Workflows 4 | ================== 5 | 6 | To delete an installed workflow :term:`run directory`, we recommend using 7 | the ``cylc clean`` command. ``cylc clean`` takes care of deleting workflows 8 | on the local filesystem and any remote install targets. 9 | It follows any symlink directories specified in 10 | :cylc:conf:`global.cylc[install][symlink dirs]` 11 | (see :ref:`CleanSymlinkDirsNote` below). You can also use ``cylc clean`` to 12 | just delete certain files or subdirectories (see :ref:`TargetedClean` below). 13 | 14 | If you've used ``rose suite-clean`` before, the functionality is similar, but 15 | not identical. 16 | 17 | .. note:: 18 | 19 | ``cylc clean`` only affects workflow :term:`run directories ` 20 | (located in the :term:`cylc-run directory`). It will not affect 21 | workflow :term:`source directories `. 22 | 23 | .. warning:: 24 | 25 | ``cylc clean`` is intended for use on workflows installed with 26 | ``cylc install``. If you clean a workflow that was instead written 27 | directly in the cylc-run directory and not backed up elsewhere, 28 | it will be lost. 29 | 30 | Simple example of using ``cylc clean``: 31 | 32 | .. code-block:: console 33 | 34 | $ cylc clean myflow/run1 35 | INFO - Removing directory: ~/cylc-run/myflow/run1 36 | 37 | 38 | Cleaning on remote install targets 39 | ---------------------------------- 40 | 41 | If any jobs in your workflow ran on a remote :term:`platform`, Cylc will 42 | automatically remove the workflow files on there in addition to the local 43 | filesystem. 44 | 45 | .. code-block:: console 46 | 47 | $ cylc clean remote-example 48 | INFO - Cleaning on install target: enterprise1701 49 | INFO - [enterprise1701] 50 | INFO - Removing directory: /home/user/cylc-run/remote-example/run1 51 | INFO - Removing directory: ~/cylc-run/remote-example/run1 52 | 53 | .. note:: 54 | 55 | This relies on determining which platforms were used from the workflow 56 | database. If the platforms recorded in the database are out of date with 57 | the global configuration, or if the database is missing altogether, you will 58 | have to manually remove the files on remote install targets. 59 | 60 | You can also clean on just the local filesystem using the ``--local`` option, 61 | or just the remote install target using the ``--remote`` option. 62 | 63 | 64 | .. _TargetedClean: 65 | 66 | Cleaning specific subdirectories or files 67 | ----------------------------------------- 68 | 69 | You can clean specific subdirectories or files inside a run directory using 70 | the ``--rm`` option (we refer to this as a targeted clean). 71 | For example, to remove the ``log`` and ``work`` directories: 72 | 73 | .. code-block:: console 74 | 75 | $ cylc clean myflow --rm log --rm work 76 | 77 | Colons can be used to delimit the items to clean, so the following is 78 | equivalent: 79 | 80 | .. code-block:: console 81 | 82 | $ cylc clean myflow --rm log:work 83 | 84 | You can also use globs. E.g., to remove all job logs for cycle points 85 | beginning with ``2020``: 86 | 87 | .. code-block:: console 88 | 89 | $ cylc clean myflow --rm 'log/job/2020*' 90 | 91 | .. note:: 92 | 93 | Make sure to place glob patterns in quotes. 94 | 95 | .. _CleanSymlinkDirsNote: 96 | 97 | A note on symlink directories 98 | ----------------------------- 99 | 100 | .. admonition:: Does this affect me? 101 | :class: tip 102 | 103 | If you use symlink directories specified in 104 | :cylc:conf:`global.cylc[install][symlink dirs]`, you might want to read 105 | this explanation of how Cylc handles them during cleaning. 106 | 107 | If you manually delete a run directory (e.g., using the ``rm`` command or the 108 | file manager), only the symlinks themselves will be deleted, not the actual 109 | targets. In contrast, ``cylc clean`` follows the symlinks and deletes the 110 | targets. 111 | 112 | - It only does this for the symlinks that can be set in 113 | :cylc:conf:`global.cylc[install][symlink dirs]`, not any custom 114 | user-created symlinks. 115 | - It does not actually look up the global configuration at time of cleaning; 116 | it simply detects what symlinks are present out of the possible ones. 117 | - It also does this for targeted clean (using the ``--rm`` option). 118 | -------------------------------------------------------------------------------- /src/user-guide/running-workflows/advanced.rst: -------------------------------------------------------------------------------- 1 | Advanced 2 | ======== 3 | 4 | 5 | Scheduler Signals 6 | ----------------- 7 | 8 | The Cylc scheduler will shutdown gracefully on receipt of any of the following 9 | signals: 10 | 11 | * ``SIGINT`` 12 | * ``SIGTERM`` 13 | * ``SIGHUP`` 14 | 15 | The signal will cause the scheduler to shutdown in ``--now`` mode. 16 | 17 | If the scheduler is already shutting down in ``--now`` mode, the signal will 18 | escalate shutdown to ``--now --now`` mode. 19 | 20 | See ``cylc stop --help`` for details on stop modes. 21 | 22 | 23 | .. _PreemptionHPC: 24 | 25 | Handling Job Preemption 26 | ----------------------- 27 | 28 | Some HPC facilities allow job preemption: the resource manager can kill 29 | or suspend running low priority jobs in order to make way for high 30 | priority jobs. The preempted jobs may then be automatically restarted 31 | by the resource manager, from the same point (if suspended) or requeued 32 | to run again from the start (if killed). 33 | 34 | Suspended jobs will poll as still running (their job status file says they 35 | started running, and they still appear in the resource manager queue). 36 | Loadleveler jobs that are preempted by kill-and-requeue ("job vacation") are 37 | automatically returned to the submitted state by Cylc. This is possible 38 | because Loadleveler sends the SIGUSR1 signal before SIGKILL for preemption. 39 | Other :term:`job runners ` just send SIGTERM before SIGKILL as normal, so Cylc 40 | cannot distinguish a preemption job kill from a normal job kill. After this the 41 | job will poll as failed (correctly, because it was killed, and the job status 42 | file records that). 43 | -------------------------------------------------------------------------------- /src/user-guide/running-workflows/authentication-files.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _ConnectionAuthentication: 3 | 4 | Client-Server Interaction 5 | ------------------------- 6 | 7 | :term:`Schedulers ` listen on dedicated network ports for 8 | TCP communications from Cylc clients (jobs and user-invoked commands). 9 | 10 | Use ``cylc scan`` to see which workflows are listening on which ports on 11 | scanned hosts. 12 | 13 | Cylc generates public-private key pairs on the workflow server and job hosts 14 | which are used for authentication. 15 | 16 | 17 | .. _Authentication Files: 18 | 19 | Authentication Files 20 | -------------------- 21 | 22 | Cylc uses `CurveZMQ`_ to ensure that 23 | any data, sent between the :term:`scheduler ` and the client, 24 | remains protected during transmission. Public keys are used to encrypt the 25 | data, private keys for decryption. 26 | 27 | Authentication files will be created in your 28 | ``$HOME/cylc-run//.service/`` directory at start-up. You can 29 | expect to find one client public key per file system for remote jobs. 30 | 31 | On the workflow host, the directory structure should contain: 32 | 33 | .. code-block:: none 34 | 35 | ~/cylc-run/workflow_x 36 | `-- .service 37 | |-- client_public_keys 38 | | |-- client_localhost.key 39 | | `-- 40 | |-- client.key_secret 41 | |-- server.key 42 | `-- server.key_secret 43 | 44 | On the remote job host, the directory structure should contain: 45 | 46 | .. code-block:: none 47 | 48 | ~/cylc-run/workflow_x 49 | `-- .service 50 | |-- client.key 51 | |-- client.key_secret 52 | `-- server.key 53 | 54 | Keys are removed as soon as they are no longer required. 55 | -------------------------------------------------------------------------------- /src/user-guide/running-workflows/dynamic-behaviour.rst: -------------------------------------------------------------------------------- 1 | .. _cylc-broadcast: 2 | 3 | Cylc Broadcast 4 | -------------- 5 | 6 | The ``cylc broadcast`` command overrides task :cylc:conf:`[runtime]` 7 | settings in a running scheduler. You can think of it as communicating 8 | new configuration settings (including information via environment variables) to 9 | selected upcoming tasks via the scheduler. 10 | 11 | See ``cylc broadcast --help`` for detailed information. 12 | 13 | Broadcast settings targeting a specific cycle point expire as the workflow 14 | moves on. Otherwise they persist for lifetime of the run, and across restarts, 15 | unless cleared with another invocation of the command. 16 | 17 | .. seealso:: 18 | 19 | :ref:`broadcast-tutorial` 20 | 21 | .. TODO: document sub-workflows 22 | -------------------------------------------------------------------------------- /src/user-guide/running-workflows/index.rst: -------------------------------------------------------------------------------- 1 | .. _RunningWorkflows: 2 | 3 | Running Workflows 4 | ================= 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | scheduler-start-up 10 | tasks-jobs-ui 11 | tracking-task-state 12 | workflow-completion 13 | reflow 14 | workflow-run-modes 15 | scheduler-log-files 16 | dynamic-behaviour 17 | authentication-files 18 | workflow-databases 19 | advanced 20 | -------------------------------------------------------------------------------- /src/user-guide/running-workflows/retrying-tasks.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | Retrying Tasks 4 | ============== 5 | 6 | This section has moved to :ref:`TaskRetries`. 7 | -------------------------------------------------------------------------------- /src/user-guide/running-workflows/workflow-completion.rst: -------------------------------------------------------------------------------- 1 | .. _workflow completion: 2 | 3 | Workflow Completion 4 | =================== 5 | 6 | A workflow can :term:`shut down ` once all 7 | :term:`active tasks ` complete without spawning further 8 | downstream activity - i.e., when :term:`n=0 window ` empties out. 9 | 10 | .. _scheduler stall: 11 | 12 | Scheduler Stall 13 | =============== 14 | 15 | A workflow has stalled if: 16 | 17 | * No tasks are waiting on unstatisfied external events, like clock triggers and xtriggers. 18 | * AND All activity has ceased. 19 | * AND The workflow has not run to completion. 20 | 21 | A workflow which has stalled requires manual intervention to continue. 22 | 23 | Stalls are caused by :term:`final status incomplete tasks ` 24 | and :term:`partially satisfied tasks `. 25 | 26 | These most often result from task failures that the workflow does not 27 | handle automatically by retries or optional branching. 28 | 29 | A stalled scheduler stays alive for a configurable timeout period 30 | to allow you to intervene, e.g. by manually triggering an incomplete 31 | task after fixing the bug that caused it to fail. 32 | 33 | If a stalled workflow does eventually shut down, on the stall timeout 34 | or by stop command, it will immediately stall again on restart to await 35 | manual intervention. 36 | 37 | .. warning:: 38 | 39 | Look in the :term:`scheduler log` to see which tasks caused a stall. 40 | 41 | .. seealso:: 42 | 43 | * :cylc:conf:`[scheduler][events]stall timeout` 44 | * :cylc:conf:`[scheduler][events]abort on stall timeout` 45 | * :cylc:conf:`[scheduler][events]stall handlers` 46 | -------------------------------------------------------------------------------- /src/user-guide/running-workflows/workflow-databases.rst: -------------------------------------------------------------------------------- 1 | .. _Workflow Run Databases: 2 | 3 | Workflow Run Databases 4 | ---------------------- 5 | 6 | Schedulers maintain two ``sqlite`` databases to record 7 | information on run history: 8 | 9 | .. code-block:: console 10 | 11 | $HOME/cylc-run//log/db # public workflow DB 12 | $HOME/cylc-run//.service/db # private workflow DB 13 | 14 | The private DB is for use only by the :term:`scheduler`. The identical 15 | public DB is provided for use by external commands such as 16 | ``cylc workflow-state``, and ``cylc report-timings``. 17 | If the public DB gets locked for too long by 18 | an external reader, the :term:`scheduler` will eventually delete it and 19 | replace it with a new copy of the private DB, to ensure that both correctly 20 | reflect the workflow state. 21 | 22 | You can interrogate the public DB with the ``sqlite3`` command line tool, 23 | the ``sqlite3`` module in the Python standard library, or any other 24 | sqlite interface. 25 | 26 | .. code-block:: console 27 | 28 | $ sqlite3 ~/cylc-run/foo/log/db << _END_ 29 | > .headers on 30 | > select * from task_events where name is "foo"; 31 | > _END_ 32 | name|cycle|time|submit_num|event|message 33 | foo|1|2017-03-12T11:06:09Z|1|submitted| 34 | foo|1|2017-03-12T11:06:09Z|1|output completed|started 35 | foo|1|2017-03-12T11:06:09Z|1|started| 36 | foo|1|2017-03-12T11:06:19Z|1|output completed|succeeded 37 | foo|1|2017-03-12T11:06:19Z|1|succeeded| 38 | 39 | The diagram shown below contains the database tables, their columns, 40 | and how the tables are related to each other. For more details on how 41 | to interpret the diagram, refer to the 42 | `Entity–relationship model Wikipedia article `_. 43 | 44 | .. cylc-db-graph:: 45 | :align: center 46 | -------------------------------------------------------------------------------- /src/user-guide/running-workflows/workflow-run-modes.rst: -------------------------------------------------------------------------------- 1 | .. _workflow-run-modes: 2 | 3 | Workflow Run Modes 4 | ================== 5 | 6 | Cylc can run a workflow without running the real jobs, which may be 7 | useful when developing workflows. Tasks 8 | in :ref:`workflow-run-modes.dummy` and 9 | :ref:`workflow-run-modes.simulation` can help you to understand how different outcomes 10 | and run lengths will affect the workflow. 11 | 12 | To start a workflow in one of these modes: 13 | 14 | .. code-block:: console 15 | 16 | $ cylc play --mode=dummy 17 | $ cylc play --mode=simulation 18 | 19 | .. admonition:: Limitations 20 | 21 | * A workflow cannot be :term:`restarted ` in a different run mode. Instead, 22 | install a new instance of the workflow and run it from scratch in the new mode. 23 | * Workflows run in Dummy and Simulation mode do not allow the run mode of 24 | individual tasks to be overridden by :ref:`task-run-modes.skip` 25 | 26 | 27 | .. _workflow-run-modes.dummy: 28 | 29 | Dummy Mode 30 | ---------- 31 | 32 | **Dummy mode** replaces real jobs with background jobs on the 33 | scheduler host which use ``sleep`` to simulate the run length according to 34 | the settings described for simulation mode. 35 | This avoids :term:`job runner` directives that request compute 36 | resources for real workflow tasks, and it allows any workflow configuration to 37 | be run locally in dummy mode. 38 | 39 | .. admonition:: Limitations 40 | 41 | * Dummy mode can only be applied to a whole workflow. 42 | * Dummy tasks run locally, so dummy mode does not test communication with remote 43 | job platforms. However, it is easy to write a live-mode test workflow with 44 | simple ``sleep 10`` tasks that submit to a remote platform. 45 | 46 | 47 | .. _workflow-run-modes.simulation: 48 | 49 | Simulation Mode 50 | --------------- 51 | 52 | **Simulation mode** does not run real jobs, and does not generate job 53 | log files. Instead, the scheduler internally simulates task completion to evolve 54 | the workflow. 55 | 56 | 57 | Simulated Run Length 58 | ^^^^^^^^^^^^^^^^^^^^ 59 | 60 | The default dummy or simulated job run length is 10 seconds. It can be 61 | changed with :cylc:conf:`[runtime][][simulation]default run length`. 62 | 63 | If :cylc:conf:`[runtime][]execution time limit` and 64 | :cylc:conf:`[runtime][][simulation]speedup factor` are both set, 65 | run length is computed by dividing the 66 | execution time limit by the speedup factor. 67 | 68 | Simulated Failure 69 | ^^^^^^^^^^^^^^^^^ 70 | 71 | Tasks always complete all custom outputs, and by default they will succeed. 72 | 73 | If you want to test emitting only some custom outputs 74 | you can run the workflow in live mode with task run modes 75 | set to :ref:`skip mode `. 76 | 77 | You can set some or all instances of a task to fail using 78 | :cylc:conf:`[runtime][][simulation]fail cycle points`, 79 | which takes either a list of cycle point strings or "all". 80 | 81 | Tasks set to fail will succeed on their second or following simulated 82 | submission. If you want all submissions to fail, set 83 | :cylc:conf:`[runtime][][simulation]fail try 1 only = False`. 84 | 85 | For example, to simulate a task you know to be flaky on the half 86 | hour but not on the hour: 87 | 88 | .. code-block:: cylc 89 | 90 | [[get_observations]] 91 | execution retry delays = PT30S 92 | [[[simulation]]] 93 | fail cycle points = 2022-01-01T00:30Z, 2022-01-01T01:30Z 94 | 95 | In another case you might not expect the retry to work, and want to test 96 | whether your failure handling works correctly: 97 | 98 | .. code-block:: cylc 99 | 100 | [[get_data]] 101 | execution retry delays = PT30S 102 | [[[simulation]]] 103 | fail try 1 only = false 104 | fail cycle points = 2022-01-01T03:00Z 105 | 106 | .. admonition:: Limitations 107 | 108 | * Simulation mode can only be applied to a whole workflow. 109 | * Alternate path branching is difficult to simulate effectively. You can 110 | configure certain tasks to fail via 111 | :cylc:conf:`[runtime][][simulation]`, but all branches based 112 | on mutually exclusive custom outputs will run because all custom outputs get 113 | artificially completed in dummy mode and in simulation mode. 114 | -------------------------------------------------------------------------------- /src/user-guide/task-implementation/index.rst: -------------------------------------------------------------------------------- 1 | Task Implementation 2 | ------------------- 3 | 4 | This section covers the technical detail of how Cylc runs :term:`tasks `. 5 | 6 | In Cylc :term:`tasks ` represent activities within the workflow. 7 | :term:`Tasks ` submit :term:`jobs ` when they are run. 8 | 9 | Tasks are the "abstract" workflow component. Jobs are the concrete 10 | representation of a task. One task could submit many jobs (for example if the 11 | job fails and the task is re-run). 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | 16 | job-scripts 17 | job-submission 18 | ssh-job-management 19 | skip-mode 20 | -------------------------------------------------------------------------------- /src/user-guide/task-implementation/skip-mode.rst: -------------------------------------------------------------------------------- 1 | .. _task-run-modes.skip: 2 | 3 | .. cylc-scope:: flow.cylc[runtime][] 4 | 5 | Skip Mode 6 | ========= 7 | 8 | .. versionadded:: 8.4.0 9 | 10 | Skip mode is designed as an aid to workflow control: 11 | 12 | * It allows creation of :term:`dummy tasks` as part of workflow 13 | design. 14 | * It allows skipping of tasks in a running workflow using either: 15 | * ``cylc broadcast -s 'run mode = skip'`` (for when it is ready to run). 16 | This will work with any future task or family. 17 | * ``cylc set --out skip`` (to immediately skip). Note that globs only match 18 | :term:`active tasks `. Otherwise task names must be explicit. 19 | 20 | .. note:: 21 | 22 | Setting :cylc:conf:`run mode=skip` in your ``flow.cylc`` 23 | will lead to ``cylc validate`` returning a warning:: 24 | 25 | WARNING - The following tasks are set to run in skip mode: 26 | * example_task 27 | 28 | This is designed to prevent users running a task in skip mode by mistake. 29 | If you are using skip mode deliberately then this can be 30 | ignored. 31 | 32 | .. _skip_mode.task_outputs: 33 | 34 | Task Outputs 35 | ------------ 36 | 37 | Skip mode allows the user to specify which task outputs 38 | will be emitted using :cylc:conf:`[skip]outputs`. 39 | 40 | By default: 41 | 42 | * All required outputs will be generated. 43 | * ``succeeded`` will be generated even if success is :term:`optional `. 44 | * If :cylc:conf:`[skip]outputs` is specified and does not include either 45 | ``succeeded`` or ``failed``, then ``succeeded`` will still be generated. 46 | 47 | The outputs submitted and started are always generated and do not 48 | need to be defined in outputs. 49 | 50 | Task Event Handlers 51 | ------------------- 52 | 53 | By default task event handlers are disabled by skip mode, but they 54 | can be enabled using 55 | :cylc:conf:`[skip]disable task event handlers`. 56 | 57 | Skip Mode Examples 58 | ------------------ 59 | 60 | Set task to succeeded 61 | ^^^^^^^^^^^^^^^^^^^^^ 62 | 63 | .. admonition:: Scenario 64 | 65 | We want to turn off a future task or cycle of a workflow. 66 | We don't want to set the outputs right now. 67 | But when it would have run we want to set it to succeed. 68 | 69 | Broadcast :cylc:conf:`run mode` setting the 70 | value to ``skip``. 71 | 72 | .. code-block:: console 73 | 74 | # Set a single future task to run in skip mode: 75 | cylc broadcast myworkflow// -p 4 -n mytask -s 'run mode = skip' 76 | 77 | # Skip Cycle 4: 78 | cylc broadcast myworkflow// -p 4 -n '*' -s 'run mode = skip' 79 | 80 | # Skip mytask for all cycles: 81 | cylc broadcast myworkflow// -n mytask -s 'run mode = skip' 82 | 83 | 84 | Create a Graph Control Task 85 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ 86 | 87 | .. admonition:: Scenario 88 | 89 | We have a large family to large family trigger. 90 | 91 | If we increase N, the number of dependencies being tracked 92 | by the scheduler is N^2. 93 | 94 | In this scenario the addition of a skip-mode task between two 95 | families improves the efficiency of the Cylc scheduler. 96 | 97 | .. seealso:: 98 | 99 | This scenario is explained in detail in 100 | :ref:`EfficientInterFamilyTriggering` 101 | 102 | .. _skip_mode.parameter_exclusion: 103 | 104 | Parameter Exclusion 105 | ^^^^^^^^^^^^^^^^^^^ 106 | 107 | .. admonition:: Scenario 108 | 109 | We want to skip a small number of tasks from a parameterized 110 | group of tasks: 111 | 112 | .. code-block:: cylc 113 | 114 | [task parameters] 115 | # House number 13 doesn't actually exist on this street... 116 | house_number = 1..20 117 | 118 | We can use skip mode to make sure that a parameter task always 119 | succeeds without running anything: 120 | 121 | .. code-block:: cylc 122 | 123 | [runtime] 124 | [[post parcel]] 125 | script = send letter 126 | [[post parcel]] 127 | run mode = skip 128 | 129 | .. cylc-scope:: 130 | -------------------------------------------------------------------------------- /src/user-guide/task-implementation/ssh-job-management.rst: -------------------------------------------------------------------------------- 1 | Remote Job Management 2 | ===================== 3 | 4 | Managing tasks in a workflow requires more than just job execution: Cylc 5 | performs additional actions with ``rsync`` for file transfer, and 6 | direct execution of ``cylc`` sub-commands over non-interactive SSH. 7 | 8 | 9 | SSH-free Job Management? 10 | ------------------------ 11 | 12 | Some sites may want to restrict access to job hosts by whitelisting SSH 13 | connections to allow only ``rsync`` for file transfer, and allowing job 14 | execution only via a local :term:`job runner` that sees the job hosts [1]_ . 15 | We are investigating the feasibility of SSH-free job management when a local 16 | job runner is available, but this is not yet possible unless your workflow 17 | and job hosts also share a filesystem, which allows Cylc to treat jobs as 18 | entirely local [2]_ . 19 | 20 | 21 | SSH-based Job Management 22 | ------------------------ 23 | 24 | Cylc does not have persistent agent processes running on job hosts to act on 25 | instructions received over the network [3]_ so instead we execute job 26 | management commands directly on job hosts over SSH. Reasons for this include: 27 | 28 | - It works equally for :term:`job runner` and background jobs. 29 | - SSH is *required* for background jobs, and for jobs in other job runners if the 30 | job runner is not available on the workflow host. 31 | - Querying the job runner alone is not sufficient for full job 32 | polling functionality. 33 | 34 | - This is because jobs can complete (and then be forgotten by 35 | the job runner) while the network, workflow host, or :term:`scheduler` is 36 | down (e.g. between workflow shutdown and restart). 37 | - To handle this we get the automatic job wrapper code to write 38 | job messages and exit status to *job status files* that are 39 | interrogated by :term:`schedulers ` during job polling 40 | operations. 41 | - Job status files reside on the job host, so the interrogation 42 | is done over SSH. 43 | 44 | - Job status files also hold job runner name and job ID; this is 45 | written by the job submit command, and read by job poll and kill commands 46 | 47 | 48 | Other Cases Where Cylc Uses SSH Directly 49 | ---------------------------------------- 50 | 51 | .. TODO - do a scan through the codebase to assert that this is still the only 52 | uses of SSH in Cylc Flow. 53 | 54 | - To see if a workflow is running on another host with a shared 55 | filesystem - see ``cylc/flow/workflow_files:detect_old_contact_file``. 56 | 57 | 58 | .. [1] A malicious script could be ``rsync``'d and run from a batch 59 | job, but jobs in job runners are considered easier to audit. 60 | .. [2] The job ID must also be valid to query and kill the job via the local 61 | :term:`job runner`. This is not the case for Slurm, unless the 62 | ``--cluster`` option is explicitly used in job query and kill commands, 63 | otherwise the job ID is not recognized by the local Slurm instance. 64 | .. [3] This would be a more complex solution, in terms of implementation, 65 | administration, and security. 66 | -------------------------------------------------------------------------------- /src/user-guide/writing-workflows/index.rst: -------------------------------------------------------------------------------- 1 | Writing Workflows 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | configuration 8 | scheduling 9 | runtime 10 | parameterized-tasks 11 | jinja2 12 | external-triggers 13 | suicide-triggers 14 | scheduler 15 | -------------------------------------------------------------------------------- /src/user-guide/writing-workflows/suicide-triggers.rst: -------------------------------------------------------------------------------- 1 | .. _SuicideTriggers: 2 | 3 | Suicide Triggers 4 | ================ 5 | 6 | .. warning:: 7 | 8 | Suicided triggers were needed in Cylc 7 and earlier to remove pre-spawned 9 | waiting tasks from graph branches not taken at runtime, which would stall 10 | the workflow. 11 | 12 | However, **Cylc 8 does not need suicide triggers for** :term:`graph 13 | branching `. 14 | 15 | They remain supported, and documented, for backward compatibility and for 16 | a rare :ref:`edge case `. 17 | 18 | 19 | Suicide triggers remove waiting tasks from the :term:`n=0 window `. 20 | 21 | They are activated just like normal :term:`task triggers ` but 22 | they remove the downstream task (prefixed with ``!``) instead of triggering it 23 | to run. 24 | 25 | Here, the task ``bar`` will be removed if ``foo`` succeeds: 26 | 27 | .. code-block:: cylc-graph 28 | 29 | foo => !bar 30 | 31 | Suicide triggers combine in the same way as normal triggers, so this graph: 32 | 33 | .. code-block:: cylc-graph 34 | 35 | foo => !baz 36 | bar => !baz 37 | 38 | is equivalent to this: 39 | 40 | .. code-block:: cylc-graph 41 | 42 | foo & bar => !baz 43 | 44 | i.e. both ``foo`` and ``bar`` must succeed for ``baz`` to be removed. 45 | 46 | To remove a task after any one of several events, use an OR operator: 47 | 48 | .. code-block:: cylc-graph 49 | 50 | foo | bar => !baz 51 | 52 | .. note:: 53 | 54 | At present, only :term:`active tasks ` that are waiting 55 | (e.g. queued tasks) can be removed by ``cylc remove``. 56 | 57 | 58 | .. _remaining-use-cases: 59 | 60 | Remaining Use Case 61 | ------------------ 62 | 63 | Suicide triggers may be needed to remove a waiting :term:`active task` when it 64 | can be inferred from the status of another task that the external trigger will 65 | never be satisfied. 66 | 67 | In the following example imagine that the two xtriggers watch two locations for 68 | the same file to appear. The file will be delivered to one location or the 69 | other but not to both, so if one xtrigger is satisfied the other will never be. 70 | The stuck waiting task can be removed with a suicide trigger, so that it 71 | doesn't stall the workflow: 72 | 73 | .. code-block:: cylc-graph 74 | 75 | @xtrigger1 => A 76 | @xtrigger2 => B 77 | 78 | A => !B # If A succeeds, remove B 79 | B => !A # If B succeeds, remove A 80 | -------------------------------------------------------------------------------- /src/workflow-design-guide/index.rst: -------------------------------------------------------------------------------- 1 | .. _workflow design guide: 2 | 3 | Workflow Design Guide 4 | ===================== 5 | 6 | .. rubric:: Cylc/Rose Workflow Design Best Practice Guide. 7 | 8 | This document provides guidance on making complex Cylc + Rose workflows that 9 | are clear, maintainable, and portable. Note that best practice advice may 10 | evolve over time with the capabilities of Rose and Cylc. 11 | 12 | Content is drawn from the Rose and Cylc user guides, earlier Met Office workflow 13 | design and operational workflow review documents, experience with real workflows 14 | across the Unified Model Consortium, and discussion among members of the UM 15 | TISD (Technical Infrastructure Workflow Design) working group. 16 | 17 | We start with the most general topics (coding style, general principles), 18 | move on to more advanced topics (efficiency and maintainability, portable 19 | workflows), and end with some pointers to future developments. 20 | 21 | .. note:: 22 | 23 | A good working knowledge of Cylc and Rose is assumed. For further details, 24 | please consult the: 25 | 26 | - `Cylc User Guide`_ 27 | - `Rose Documentation`_ 28 | 29 | .. note:: 30 | 31 | For non-Rose users: this document comes out of the Unified Model 32 | Consortium wherein Cylc is used within the Rose *workflow management 33 | framework*. However, the bulk of the information in this guide is about 34 | Cylc workflow design; which parts are Rose-specific should be clear from 35 | context. 36 | 37 | .. toctree:: 38 | :maxdepth: 2 39 | 40 | style-guide 41 | general-principles 42 | efficiency 43 | portable-workflows 44 | -------------------------------------------------------------------------------- /src/workflows/integer-pipeline/flow.cylc: -------------------------------------------------------------------------------- 1 | [scheduling] 2 | cycling mode = integer 3 | initial cycle point = 1 4 | [[graph]] 5 | R/^/P1 = """ 6 | A => B => C 7 | A[-P1] => A 8 | B[-P1] => B 9 | C[-P1] => C 10 | """ 11 | -------------------------------------------------------------------------------- /src/workflows/jinja2/cities/flow.cylc: -------------------------------------------------------------------------------- 1 | #!Jinja2 2 | [meta] 3 | title = "Jinja2 city workflow example." 4 | description = """ 5 | Illustrates use of variables and math expressions, and programmatic 6 | generation of groups of related dependencies and runtime properties. 7 | """ 8 | 9 | [scheduler] 10 | allow implicit tasks = True 11 | 12 | {% set HOST = "SuperComputer" %} 13 | {% set CITIES = 'NewYork', 'Philadelphia', 'Newark', 'Houston', 'SantaFe', 'Chicago' %} 14 | {% set CITYJOBS = 'one', 'two', 'three', 'four' %} 15 | {% set LIMIT_MINS = 20 %} 16 | 17 | {% set CLEANUP = True %} 18 | 19 | [scheduling] 20 | initial cycle point = 2011-08-08T12 21 | [[graph]] 22 | {% if CLEANUP %} 23 | T23 = "clean" 24 | {% endif %} 25 | T00,T12 = """ 26 | setup => get_lbc & get_ic # foo 27 | {% for CITY in CITIES %} {# comment #} 28 | get_lbc => {{ CITY }}_one 29 | get_ic => {{ CITY }}_two 30 | {{ CITY }}_one & {{ CITY }}_two => {{ CITY }}_three & {{ CITY }}_four 31 | {% if CLEANUP %} 32 | {{ CITY }}_three & {{ CITY }}_four => cleanup 33 | {% endif %} 34 | {% endfor %} 35 | """ 36 | 37 | [runtime] 38 | [[on_{{ HOST }} ]] 39 | [[[remote]]] 40 | host = {{ HOST }} 41 | # (remote cylc directory is set in site/user config for this host) 42 | [[[directives]]] 43 | wall_clock_limit = "00:{{ LIMIT_MINS|int() + 2 }}:00,00:{{ LIMIT_MINS }}:00" 44 | 45 | {% for CITY in CITIES %} 46 | [[ {{ CITY }} ]] 47 | inherit = on_{{ HOST }} 48 | {% for JOB in CITYJOBS %} 49 | [[ {{ CITY }}_{{ JOB }} ]] 50 | inherit = {{ CITY }} 51 | {% endfor %} 52 | {% endfor %} 53 | -------------------------------------------------------------------------------- /src/workflows/jinja2/defaults/flow.cylc: -------------------------------------------------------------------------------- 1 | #!Jinja2 2 | [meta] 3 | title = "Jinja2 example: use of defaults and external input" 4 | 5 | description = """ 6 | The template variable N_MEMBERS can be set on the command line with 7 | --set or --set-file=FILE; but if not a default values is supplied. 8 | """ 9 | 10 | [scheduling] 11 | initial cycle point = 20100808T00 12 | final cycle point = 20100816T00 13 | [[graph]] 14 | T00 = """ 15 | foo => ENS 16 | ENS:succeed-all => bar 17 | """ 18 | 19 | [runtime] 20 | [[foo, bar]] 21 | [[ENS]] 22 | {% for I in range( 0, N_MEMBERS | default( 3 )) %} 23 | [[ mem_{{ I }} ]] 24 | inherit = ENS 25 | {% endfor %} 26 | -------------------------------------------------------------------------------- /src/workflows/message-triggers/flow.cylc: -------------------------------------------------------------------------------- 1 | [scheduling] 2 | initial cycle point = 20140801T00 3 | final cycle point = 20141201T00 4 | [[graph]] 5 | P2M = """ 6 | foo:out1 => bar 7 | foo[-P2M]:out2 => baz 8 | """ 9 | [runtime] 10 | [[foo]] 11 | script = """ 12 | sleep 5 13 | cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" "file 1 done" 14 | sleep 10 15 | cylc message -- "${CYLC_WORKFLOW_ID}" "${CYLC_TASK_JOB}" "file 2 done" 16 | sleep 10 17 | """ 18 | [[[outputs]]] 19 | out1 = "file 1 done" 20 | out2 = "file 2 done" 21 | [[bar, baz]] 22 | script = sleep 10 23 | -------------------------------------------------------------------------------- /src/workflows/queues/flow.cylc: -------------------------------------------------------------------------------- 1 | [meta] 2 | title = demonstrates internal queueing 3 | description = """ 4 | Two trees of tasks: the first uses the default queue set to a limit of 5 | two active tasks at once; the second uses another queue limited to three 6 | active tasks at once. 7 | """ 8 | 9 | [scheduler] 10 | allow implicit tasks = True 11 | 12 | [scheduling] 13 | [[queues]] 14 | [[[default]]] 15 | limit = 2 16 | [[[foo]]] 17 | limit = 3 18 | members = n, o, p, FAM2, u, v, w, x, y, z 19 | [[graph]] 20 | R1 = """ 21 | a => b & c => FAM1 22 | n => o & p => FAM2 23 | FAM1:succeed-all => h & i & j & k & l & m 24 | FAM2:succeed-all => u & v & w & x & y & z 25 | """ 26 | 27 | [runtime] 28 | [[FAM1, FAM2]] 29 | [[d,e,f,g]] 30 | inherit = FAM1 31 | [[q,r,s,t]] 32 | inherit = FAM2 33 | -------------------------------------------------------------------------------- /src/workflows/xtrigger/sequential/flow.cylc: -------------------------------------------------------------------------------- 1 | [scheduler] 2 | cycle point format = %Y 3 | [scheduling] 4 | initial cycle point = 2010 5 | sequential xtriggers = True 6 | [[xtriggers]] 7 | upstream = workflow_state("up//%(point)s/foo:x"):PT10S 8 | clock_0 = wall_clock(offset=PT0H, sequential=False) 9 | [[graph]] 10 | P1Y = """ 11 | @clock_0 => foo 12 | @clock_0 & @upstream => FAM:succeed-all => blam 13 | """ 14 | [runtime] 15 | [[root]] 16 | script = sleep 5 17 | [[foo, blam]] 18 | [[FAM]] 19 | [[f1,f2,f3]] 20 | inherit = FAM 21 | -------------------------------------------------------------------------------- /src/workflows/xtrigger/workflow_state/downstream/flow.cylc: -------------------------------------------------------------------------------- 1 | [scheduler] 2 | cycle point format = %Y 3 | [scheduling] 4 | initial cycle point = 2010 5 | [[xtriggers]] 6 | upstream = workflow_state("up//%(point)s/foo:x"):PT10S 7 | clock_0 = wall_clock(offset=PT0H) 8 | [[graph]] 9 | P1Y = """ 10 | foo 11 | @clock_0 & @upstream => FAM:succeed-all => blam 12 | """ 13 | [runtime] 14 | [[root]] 15 | script = sleep 5 16 | [[foo, blam]] 17 | [[FAM]] 18 | [[f1,f2,f3]] 19 | inherit = FAM 20 | -------------------------------------------------------------------------------- /src/workflows/xtrigger/workflow_state/upstream/flow.cylc: -------------------------------------------------------------------------------- 1 | [scheduler] 2 | cycle point format = %Y 3 | [scheduling] 4 | initial cycle point = 2005 5 | final cycle point = 2015 6 | [[graph]] 7 | P1Y = "foo => bar" 8 | [runtime] 9 | [[bar]] 10 | script = sleep 10 11 | [[foo]] 12 | script = sleep 5; cylc message "data ready" 13 | [[[outputs]]] 14 | x = "data ready" 15 | -------------------------------------------------------------------------------- /src/workflows/xtrigger/xrandom/flow.cylc: -------------------------------------------------------------------------------- 1 | [scheduling] 2 | cycling mode = integer 3 | initial cycle point = 1 4 | final cycle point = 5 5 | [[xtriggers]] 6 | # Called once for all dependent tasks (all cycles). 7 | x1 = xrandom(percent=25, secs=2):PT5S 8 | # Called once per dependent task name (all cycles). 9 | x2 = xrandom(percent=25, secs=2, _=%(name)s):PT5S 10 | # Called once per cycle for all dependent tasks. 11 | x3 = xrandom(percent=25, secs=2, _=%(point)s):PT5S 12 | [[graph]] 13 | P1 = """ 14 | # all instances of foo and bar should trigger at once, together 15 | @x1 => foo & bar 16 | 17 | # all instances of cat should trigger at once, and separately, all 18 | # instances of baz should trigger at once. 19 | @x2 => cat & dog 20 | 21 | # each instance of qux should trigger separately 22 | @x3 => qux 23 | 24 | # Result: 25 | # - x1 should return True once, and not be called again. 26 | # - x2 should return True twice, and not be called again. 27 | # - x3 should return True five times, and not be called again. 28 | # i.e. 8 True returns in the 5-cycle workflow run. 29 | """ 30 | [runtime] 31 | [[root]] 32 | script = sleep 5 33 | [[foo, bar, cat, dog, qux]] 34 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore= 3 | ; module level import not at top of file 4 | E402, 5 | ; line break before binary operator 6 | W503, 7 | ; line break after binary operator 8 | W504 9 | exclude= 10 | .git, 11 | __pycache__, 12 | .tox, 13 | src/conf.py, 14 | doc/ 15 | --------------------------------------------------------------------------------