├── .dockerignore ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .pyautotest ├── .readthedocs.yaml ├── CODEOWNERS ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── OWNERS ├── README.md ├── bin ├── generate_tron_tab_completion_cache ├── tronctl ├── tronctl_tabcomplete.sh ├── trond ├── tronfig ├── tronrepl ├── tronview └── tronview_tabcomplete.sh ├── cluster_itests └── docker-compose.yml ├── contrib ├── migration_script.py ├── mock_patch_checker.py ├── namespace_cleanup.sh ├── patch-config-loggers.diff ├── sync-from-yelp-prod.sh └── sync_namespaces_jobs.py ├── debian ├── changelog ├── compat ├── control ├── copyright ├── docs ├── install ├── pycompat ├── pyversions ├── rules ├── tron.conffiles ├── tron.default ├── tron.dirs ├── tron.example ├── tron.links ├── tron.manpages ├── tron.postinst ├── tron.service ├── tron.upstart └── watch ├── dev ├── config │ ├── MASTER.yaml │ └── _manifest.yaml ├── logging.conf └── tron.lock ├── docs └── source │ ├── _static │ └── nature.css │ ├── command_context.rst │ ├── conf.py │ ├── config.rst │ ├── developing.rst │ ├── generated │ ├── modules.rst │ ├── tron.actioncommand.rst │ ├── tron.api.adapter.rst │ ├── tron.api.async_resource.rst │ ├── tron.api.auth.rst │ ├── tron.api.controller.rst │ ├── tron.api.requestargs.rst │ ├── tron.api.resource.rst │ ├── tron.api.rst │ ├── tron.command_context.rst │ ├── tron.commands.authentication.rst │ ├── tron.commands.backfill.rst │ ├── tron.commands.client.rst │ ├── tron.commands.cmd_utils.rst │ ├── tron.commands.display.rst │ ├── tron.commands.retry.rst │ ├── tron.commands.rst │ ├── tron.config.config_parse.rst │ ├── tron.config.config_utils.rst │ ├── tron.config.manager.rst │ ├── tron.config.rst │ ├── tron.config.schedule_parse.rst │ ├── tron.config.schema.rst │ ├── tron.config.static_config.rst │ ├── tron.core.action.rst │ ├── tron.core.actiongraph.rst │ ├── tron.core.actionrun.rst │ ├── tron.core.job.rst │ ├── tron.core.job_collection.rst │ ├── tron.core.job_scheduler.rst │ ├── tron.core.jobgraph.rst │ ├── tron.core.jobrun.rst │ ├── tron.core.recovery.rst │ ├── tron.core.rst │ ├── tron.eventbus.rst │ ├── tron.kubernetes.rst │ ├── tron.manhole.rst │ ├── tron.mcp.rst │ ├── tron.mesos.rst │ ├── tron.metrics.rst │ ├── tron.node.rst │ ├── tron.prom_metrics.rst │ ├── tron.rst │ ├── tron.scheduler.rst │ ├── tron.serialize.filehandler.rst │ ├── tron.serialize.rst │ ├── tron.serialize.runstate.dynamodb_state_store.rst │ ├── tron.serialize.runstate.rst │ ├── tron.serialize.runstate.shelvestore.rst │ ├── tron.serialize.runstate.statemanager.rst │ ├── tron.serialize.runstate.yamlstore.rst │ ├── tron.ssh.rst │ ├── tron.trondaemon.rst │ ├── tron.utils.collections.rst │ ├── tron.utils.crontab.rst │ ├── tron.utils.exitcode.rst │ ├── tron.utils.logreader.rst │ ├── tron.utils.observer.rst │ ├── tron.utils.persistable.rst │ ├── tron.utils.proxy.rst │ ├── tron.utils.queue.rst │ ├── tron.utils.rst │ ├── tron.utils.state.rst │ ├── tron.utils.timeutils.rst │ ├── tron.utils.trontimespec.rst │ ├── tron.utils.twistedutils.rst │ └── tron.yaml.rst │ ├── index.rst │ ├── jobs.rst │ ├── man │ ├── tronctl.1 │ ├── trond.8 │ ├── tronfig.1 │ └── tronview.1 │ ├── man_tronctl.rst │ ├── man_trond.rst │ ├── man_tronfig.rst │ ├── man_tronview.rst │ ├── overview.rst │ ├── sample_config.yaml │ ├── tools.rst │ ├── tron.yaml │ ├── tronweb.rst │ ├── tutorial.rst │ └── whats-new.rst ├── itest.sh ├── mypy.ini ├── osx-bdb.sh ├── package.json ├── pyproject.toml ├── requirements-dev-minimal.txt ├── requirements-dev.txt ├── requirements-docs.txt ├── requirements-minimal.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── testfiles └── MASTER.yaml ├── testifycompat ├── __init__.py ├── assertions.py ├── bin │ ├── __init__.py │ └── migrate.py └── fixtures.py ├── tests ├── __init__.py ├── actioncommand_test.py ├── api │ ├── __init__.py │ ├── adapter_test.py │ ├── auth_test.py │ ├── controller_test.py │ ├── requestargs_test.py │ └── resource_test.py ├── assertions.py ├── bin │ ├── __init__.py │ ├── action_runner_test.py │ ├── action_status_test.py │ ├── check_tron_jobs_test.py │ ├── get_tron_metrics_test.py │ └── recover_batch_test.py ├── command_context_test.py ├── commands │ ├── __init__.py │ ├── backfill_test.py │ ├── client_test.py │ ├── cmd_utils_test.py │ ├── display_test.py │ └── retry_test.py ├── config │ ├── __init__.py │ ├── config_parse_test.py │ ├── config_utils_test.py │ ├── manager_test.py │ └── schedule_parse_test.py ├── core │ ├── __init__.py │ ├── action_test.py │ ├── actiongraph_test.py │ ├── actionrun_test.py │ ├── job_collection_test.py │ ├── job_scheduler_test.py │ ├── job_test.py │ ├── jobgraph_test.py │ ├── jobrun_test.py │ └── recovery_test.py ├── data │ ├── logging.conf │ └── test_config.yaml ├── eventbus_test.py ├── kubernetes_test.py ├── mcp_reconfigure_test.py ├── mcp_test.py ├── mesos_test.py ├── metrics_test.py ├── mocks.py ├── node_test.py ├── sandbox.py ├── scheduler_test.py ├── serialize │ ├── __init__.py │ ├── filehandler_test.py │ └── runstate │ │ ├── __init__.py │ │ ├── dynamodb_state_store_test.py │ │ ├── shelvestore_test.py │ │ ├── statemanager_test.py │ │ └── yamlstore_test.py ├── ssh_test.py ├── test_id_rsa ├── test_id_rsa.pub ├── testingutils.py ├── tools │ └── sync_tron_state_from_k8s_test.py ├── trond_test.py ├── trondaemon_test.py └── utils │ ├── __init__.py │ ├── collections_test.py │ ├── crontab_test.py │ ├── logreader_test.py │ ├── observer_test.py │ ├── proxy_test.py │ ├── shortOutputTest.txt │ ├── state_test.py │ ├── timeutils_test.py │ └── trontimespec_test.py ├── tools ├── action_dag_diagram.py ├── inspect_serialized_state.py ├── migration │ ├── migrate_config_0.2_to_0.3.py │ ├── migrate_config_0.5.1_to_0.5.2.py │ ├── migrate_state.py │ └── migrate_state_1.3.15_to_1.4.0.py ├── pickles_to_json.py └── sync_tron_state_from_k8s.py ├── tox.ini ├── tron ├── __init__.py ├── actioncommand.py ├── api │ ├── __init__.py │ ├── adapter.py │ ├── async_resource.py │ ├── auth.py │ ├── controller.py │ ├── requestargs.py │ └── resource.py ├── bin │ ├── action_runner.py │ ├── action_status.py │ ├── check_tron_datastore_staleness.py │ ├── check_tron_jobs.py │ ├── get_tron_metrics.py │ └── recover_batch.py ├── command_context.py ├── commands │ ├── __init__.py │ ├── authentication.py │ ├── backfill.py │ ├── client.py │ ├── cmd_utils.py │ ├── display.py │ └── retry.py ├── config │ ├── __init__.py │ ├── config_parse.py │ ├── config_utils.py │ ├── manager.py │ ├── schedule_parse.py │ ├── schema.py │ ├── static_config.py │ └── tronfig_schema.json ├── core │ ├── __init__.py │ ├── action.py │ ├── actiongraph.py │ ├── actionrun.py │ ├── job.py │ ├── job_collection.py │ ├── job_scheduler.py │ ├── jobgraph.py │ ├── jobrun.py │ └── recovery.py ├── default_config.yaml ├── eventbus.py ├── kubernetes.py ├── logging.conf ├── manhole.py ├── mcp.py ├── mesos.py ├── metrics.py ├── node.py ├── prom_metrics.py ├── scheduler.py ├── serialize │ ├── __init__.py │ ├── filehandler.py │ └── runstate │ │ ├── __init__.py │ │ ├── dynamodb_state_store.py │ │ ├── shelvestore.py │ │ ├── statemanager.py │ │ └── yamlstore.py ├── ssh.py ├── trondaemon.py ├── utils │ ├── __init__.py │ ├── collections.py │ ├── crontab.py │ ├── exitcode.py │ ├── logreader.py │ ├── observer.py │ ├── persistable.py │ ├── proxy.py │ ├── queue.py │ ├── state.py │ ├── timeutils.py │ ├── trontimespec.py │ └── twistedutils.py └── yaml.py ├── tronweb ├── coffee │ ├── actionrun.coffee │ ├── config.coffee │ ├── dashboard.coffee │ ├── graph.coffee │ ├── job.coffee │ ├── models.coffee │ ├── navbar.coffee │ ├── nodes.coffee │ ├── routes.coffee │ ├── timeline.coffee │ └── views.coffee ├── css │ ├── bootstrap-responsive.min.css │ ├── bootstrap.min.css │ ├── codemirror.css │ ├── tronweb.less │ └── whhg.css ├── fonts │ ├── SIL OFL Font License WebHostingHub Glyphs.txt │ └── webhostinghub-glyphs.ttf ├── img │ ├── ui-bg_diagonals-small_10_555_40x40.png │ └── ui-bg_dots-medium_100_eee_4x4.png ├── index.html ├── js │ ├── backbone-min.js │ ├── bootstrap.min.js │ ├── codemirror.js │ ├── cytoscape-dagre.min.js │ ├── cytoscape.min.js │ ├── d3.v7.min.js │ ├── dagre.min.js │ ├── less-1.3.3.min.js │ ├── moment-timezone-with-data-10-year-range.min.js │ ├── moment.min.js │ ├── plugins.js │ ├── underscore-min.js │ ├── underscore.extra.js │ ├── underscore.string.js │ └── yaml.js └── tronweb.ico ├── tronweb2 ├── .eslintrc.js ├── .gitignore ├── package.json ├── public │ ├── bootstrap.min.css │ ├── bootstrap.min.css.map │ ├── fonts │ │ ├── SIL OFL Font License WebHostingHub Glyphs.txt │ │ └── webhostinghub-glyphs.ttf │ ├── index.html │ ├── manifest.json │ ├── tronweb.ico │ └── whhg.css └── src │ ├── App.css │ ├── App.js │ ├── components │ ├── ActionGraph │ │ ├── ActionGraph.css │ │ ├── ActionGraph.js │ │ └── index.js │ ├── Job │ │ ├── Job.js │ │ └── index.js │ ├── JobScheduler │ │ ├── JobScheduler.js │ │ └── index.js │ ├── JobSettings │ │ ├── JobSettings.js │ │ └── index.js │ ├── JobsDashboard │ │ ├── JobsDashboard.css │ │ ├── JobsDashboard.js │ │ └── index.js │ ├── NavBar │ │ ├── NavBar.css │ │ ├── NavBar.js │ │ └── index.js │ └── index.js │ ├── index.js │ └── utils │ └── utils.js ├── tronweb_tests ├── SpecRunner.html ├── spec │ └── README └── tests │ ├── actionrun_test.coffee │ ├── dashboard_test.coffee │ ├── navbar_test.coffee │ ├── routes_test.coffee │ └── timeline_test.coffee └── yelp_package ├── bionic └── Dockerfile ├── extra_requirements_yelp.txt ├── itest_dockerfiles ├── mesos │ ├── Dockerfile │ ├── mesos-secrets │ └── mesos-slave-secret ├── tronmaster │ └── Dockerfile └── zookeeper │ └── Dockerfile └── jammy └── Dockerfile /.dockerignore: -------------------------------------------------------------------------------- 1 | .tox 2 | .git 3 | .idea 4 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: tron-ci 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags: 8 | - v*.* 9 | pull_request: 10 | release: 11 | jobs: 12 | tox: 13 | runs-on: ubuntu-22.04 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | toxenv: 18 | - py38,docs 19 | steps: 20 | - uses: actions/checkout@v2 21 | - uses: actions/setup-python@v2 22 | with: 23 | python-version: 3.8 24 | # GHA won't setup tox for us and we use tox-pip-extensions for venv-update 25 | - run: pip install tox==3.2 tox-pip-extensions==1.3.0 26 | # there are no pre-built wheels for bsddb3, so we need to install 27 | # its dpkg dependencies so that we can build a wheel when we're 28 | # creating our env. Once we get rid of bsddb3 as a Python dependency, 29 | # then we can also get rid of this dpkg 30 | - run: sudo apt-get install --quiet --assume-yes libdb5.3-dev 31 | # we explictly attempt to import the C extensions for some PyYAML 32 | # functionality, so we need the LibYAML bindings provided by this 33 | # package 34 | - run: sudo apt-get install --quiet --assume-yes libyaml-dev 35 | - run: tox -e ${{ matrix.toxenv }} 36 | build_debs: 37 | runs-on: ubuntu-22.04 38 | strategy: 39 | fail-fast: false 40 | matrix: 41 | dist: [bionic, jammy] 42 | steps: 43 | - uses: actions/checkout@v2 44 | - uses: actions/setup-python@v2 45 | with: 46 | python-version: 3.8 47 | # Update package lists to ensure we have the latest information 48 | - run: sudo apt-get update 49 | # the container provided by GitHub doesn't include utilities 50 | # needed for dpkg building, so we need to install `devscripts` 51 | # to bring those in 52 | - run: sudo apt-get install --quiet --assume-yes devscripts 53 | - run: make itest_${{ matrix.dist }} 54 | - uses: actions/upload-artifact@v4 55 | with: 56 | name: deb-${{ matrix.dist }} 57 | path: dist/tron_*.deb 58 | cut_release: 59 | runs-on: ubuntu-22.04 60 | needs: build_debs 61 | steps: 62 | - uses: actions/checkout@v2 63 | - run: mkdir -p dist/ 64 | - uses: actions/download-artifact@v4 65 | with: 66 | name: deb-bionic 67 | path: dist/ 68 | - uses: actions/download-artifact@v4 69 | with: 70 | name: deb-jammy 71 | path: dist/ 72 | - name: Release 73 | uses: softprops/action-gh-release@v1 74 | if: startsWith(github.ref, 'refs/tags/v') 75 | with: 76 | generate_release_notes: true 77 | files: | 78 | dist/tron_*.deb 79 | fail_on_unmatched_files: true 80 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | build 3 | MANIFEST 4 | tron.egg-info 5 | *.pyc 6 | *._* 7 | *.swp 8 | *.swo 9 | docs/_build/ 10 | .idea 11 | .vscode 12 | .fleet 13 | tron.iml 14 | docs/images/ 15 | *.dot 16 | tronweb/js/cs/*.js 17 | yarn.lock 18 | tronweb_tests/spec/*.js 19 | tronweb_tests/lib/ 20 | .tox 21 | .tox-indocker 22 | tron.iml 23 | __pycache__/ 24 | .pytest_cache/ 25 | tron_state 26 | tron.lock 27 | manhole.sock 28 | manhole.sock.lock 29 | node_modules/ 30 | 31 | # Example cluster 32 | example-cluster/config 33 | example-cluster/MASTER.* 34 | example-cluster/tron-repl.lock 35 | example-cluster/tron_state* 36 | example-cluster/manhole.sock* 37 | example-cluster/_events/ 38 | *.stdout 39 | *.stderr 40 | dev/manhole.sock.lock 41 | dev/tron.pid 42 | dev/_events/ 43 | 44 | # Generated debian artifacts 45 | debian/.debhelper/ 46 | debian/debhelper-build-stamp 47 | debian/files 48 | debian/tron 49 | debian/tron.debhelper.log 50 | debian/tron.postinst.debhelper 51 | debian/tron.postrm.debhelper 52 | debian/tron.preinst.debhelper 53 | debian/tron.prerm.debhelper 54 | debian/tron.substvars 55 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | default_language_version: 3 | python: python3.8 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v2.5.0 7 | hooks: 8 | - id: trailing-whitespace 9 | - id: end-of-file-fixer 10 | exclude: CHANGELOG.md 11 | - id: check-docstring-first 12 | - id: check-json 13 | - id: check-yaml 14 | - id: requirements-txt-fixer 15 | - id: fix-encoding-pragma 16 | args: [--remove] 17 | - id: pretty-format-json 18 | args: [--autofix, --indent, '4', --no-sort-keys] 19 | - repo: https://github.com/PyCQA/flake8 20 | rev: 5.0.4 21 | hooks: 22 | - id: flake8 23 | exclude: ^docs/source/conf.py$ 24 | - repo: https://github.com/asottile/reorder_python_imports 25 | rev: v1.9.0 26 | hooks: 27 | - id: reorder-python-imports 28 | args: [--py3-plus] 29 | - repo: https://github.com/asottile/pyupgrade 30 | rev: v3.3.1 31 | hooks: 32 | - id: pyupgrade 33 | args: [--py38-plus] 34 | - repo: local 35 | hooks: 36 | - id: patch-enforce-autospec 37 | name: mock.patch enforce autospec 38 | description: | 39 | This hook ensures all mock.patch invocations specify an autospec 40 | entry: contrib/mock_patch_checker.py 41 | language: script 42 | files: ^tests/.*\.py$ 43 | - repo: http://github.com/psf/black 44 | rev: 22.3.0 45 | hooks: 46 | - id: black 47 | args: [--target-version, py38] 48 | -------------------------------------------------------------------------------- /.pyautotest: -------------------------------------------------------------------------------- 1 | test_runner_name: "testify" 2 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # RTD defaults as of 2023-11-08 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.8" 12 | # You can also specify other tool versions: 13 | # nodejs: "20" 14 | # rust: "1.70" 15 | # golang: "1.20" 16 | 17 | # Also provide downloadable zip 18 | formats: [htmlzip] 19 | 20 | # Build documentation in the "docs/" directory with Sphinx 21 | sphinx: 22 | configuration: docs/source/conf.py 23 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 24 | # builder: "dirhtml" 25 | # Fail on all warnings to avoid broken references 26 | # fail_on_warning: true 27 | 28 | # Optionally build your docs in additional formats such as PDF and ePub 29 | # formats: 30 | # - pdf 31 | # - epub 32 | 33 | # Optional but recommended, declare the Python requirements required 34 | # to build your documentation 35 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 36 | python: 37 | install: 38 | - requirements: requirements-docs.txt 39 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # NOTE: "we" in this file will refer to the Compute Infrastructure team at Yelp 2 | * @Yelp/paasta 3 | # 4 | # prevent cheeky modifications :) 5 | CODEOWNERS @Yelp/paasta 6 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2010-2012 Yelp 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | include *.md 3 | include Makefile 4 | include tron/default_config.yaml 5 | include tron/logging.conf 6 | include tron/named_config_template.yaml 7 | recursive-include tests *.py *.yaml 8 | recursive-include docs *.rst *.yaml *.1 *.8 9 | recursive-include tronweb * 10 | recursive-exclude tronweb *.coffee 11 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | --- 2 | teams: 3 | - Compute Infra 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Tron - Batch Scheduling System 2 | ============================== 3 | 4 | [![Build Status](https://github.com/Yelp/Tron/actions/workflows/ci.yml/badge.svg?query=branch%3Amaster)](https://github.com/Yelp/Tron/actions/workflows/ci.yml) 5 | [![Documentation Status](https://readthedocs.org/projects/tron/badge/?version=latest)](http://tron.readthedocs.io/en/latest/?badge=latest) 6 | 7 | Tron is a centralized system for managing periodic batch processes 8 | across a cluster. If you find [cron](http://en.wikipedia.org/wiki/Cron) or 9 | [fcron](http://fcron.free.fr/) to be insufficient for managing complex work 10 | flows across multiple computers, Tron might be for you. 11 | 12 | Install with: 13 | 14 | > sudo pip install tron 15 | 16 | Or look at the [tutorial](http://tron.readthedocs.io/en/latest/tutorial.html). 17 | 18 | The full documentation is available [on ReadTheDocs](http://tron.readthedocs.io/en/latest/). 19 | 20 | Versions / Roadmap 21 | ------------------ 22 | 23 | Tron is changing and under active development. 24 | 25 | It is being transformed from an ssh-based execution engine to be compatible with running on [Kubernetes 26 | ](https://kubernetes.io/docs/concepts/overview/). 27 | 28 | Tron development is specifically targeting Yelp's needs and not designed to be 29 | a general solution for other companies. 30 | 31 | 32 | Contributing 33 | ------------ 34 | 35 | Read [Working on Tron](http://tron.readthedocs.io/en/latest/developing.html) and 36 | start sending pull requests! 37 | 38 | Any issues should be posted [on Github](http://github.com/Yelp/Tron/issues). 39 | 40 | BerkeleyDB on Mac OS X 41 | ---------------------- 42 | 43 | $ brew install berkeley-db 44 | $ export BERKELEYDB_DIR=$(brew --cellar)/berkeley-db/ 45 | $ export YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1 46 | -------------------------------------------------------------------------------- /bin/generate_tron_tab_completion_cache: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | print a list of all the tron jobs, to be saved as a cache for tab completion 4 | """ 5 | import argcomplete 6 | 7 | from tron.commands import cmd_utils 8 | from tron.commands.client import Client 9 | 10 | 11 | def main(): 12 | parser = cmd_utils.build_option_parser() 13 | argcomplete.autocomplete(parser) 14 | args = parser.parse_args() 15 | cmd_utils.load_config(args) 16 | 17 | client = Client(args.server) 18 | for job in client.jobs(include_job_runs=True, include_action_runs=True): 19 | print(job["name"]) 20 | for run in job["runs"]: 21 | print(run["id"]) 22 | for action in run["runs"]: 23 | print(action["id"]) 24 | 25 | 26 | if __name__ == "__main__": 27 | main() 28 | -------------------------------------------------------------------------------- /bin/tronctl_tabcomplete.sh: -------------------------------------------------------------------------------- 1 | if [[ -n ${ZSH_VERSION-} ]]; then 2 | autoload -U +X bashcompinit && bashcompinit 3 | fi 4 | 5 | # This magic eval enables tab-completion for tron commands 6 | # http://argcomplete.readthedocs.io/en/latest/index.html#synopsis 7 | eval "$(/opt/venvs/tron/bin/register-python-argcomplete tronctl)" 8 | -------------------------------------------------------------------------------- /bin/tronview_tabcomplete.sh: -------------------------------------------------------------------------------- 1 | if [[ -n ${ZSH_VERSION-} ]]; then 2 | autoload -U +X bashcompinit && bashcompinit 3 | fi 4 | 5 | # This magic eval enables tab-completion for tron commands 6 | # http://argcomplete.readthedocs.io/en/latest/index.html#synopsis 7 | eval "$(/opt/venvs/tron/bin/register-python-argcomplete tronview)" 8 | -------------------------------------------------------------------------------- /cluster_itests/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/cluster_itests/docker-compose.yml -------------------------------------------------------------------------------- /contrib/mock_patch_checker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.8 2 | import ast 3 | import sys 4 | 5 | 6 | class MockChecker(ast.NodeVisitor): 7 | def __init__(self): 8 | self.errors = 0 9 | self.init_module_imports() 10 | 11 | def init_module_imports(self): 12 | self.imported_patch = False 13 | self.imported_mock = False 14 | 15 | def check_files(self, files): 16 | for file in files: 17 | self.check_file(file) 18 | 19 | def check_file(self, filename): 20 | self.current_filename = filename 21 | try: 22 | with open(filename) as fd: 23 | try: 24 | file_ast = ast.parse(fd.read()) 25 | except SyntaxError as error: 26 | print("SyntaxError on file %s:%d" % (filename, error.lineno)) 27 | return 28 | except OSError: 29 | print("Error opening filename: %s" % filename) 30 | return 31 | self.init_module_imports() 32 | self.visit(file_ast) 33 | 34 | def _call_uses_patch(self, node): 35 | try: 36 | return node.func.id == "patch" 37 | except AttributeError: 38 | return False 39 | 40 | def _call_uses_mock_patch(self, node): 41 | try: 42 | return node.func.value.id == "mock" and node.func.attr == "patch" 43 | except AttributeError: 44 | return False 45 | 46 | def visit_Import(self, node): 47 | if [name for name in node.names if "mock" == name.name]: 48 | self.imported_mock = True 49 | 50 | def visit_ImportFrom(self, node): 51 | if node.module == "mock" and (name for name in node.names if "patch" == name.name): 52 | self.imported_patch = True 53 | 54 | def visit_Call(self, node): 55 | try: 56 | if (self.imported_patch and self._call_uses_patch(node)) or ( 57 | self.imported_mock and self._call_uses_mock_patch(node) 58 | ): 59 | if not any([keyword for keyword in node.keywords if keyword.arg == "autospec"]): 60 | print("%s:%d: Found a mock without an autospec!" % (self.current_filename, node.lineno)) 61 | self.errors += 1 62 | except AttributeError: 63 | pass 64 | self.generic_visit(node) 65 | 66 | 67 | def main(filenames): 68 | checker = MockChecker() 69 | checker.check_files(filenames) 70 | if checker.errors == 0: 71 | sys.exit(0) 72 | else: 73 | print("You probably meant to specify 'autospec=True' in these tests.") 74 | print("If you really don't want to, specify 'autospec=None'") 75 | sys.exit(1) 76 | 77 | 78 | if __name__ == "__main__": 79 | main(sys.argv[1:]) 80 | -------------------------------------------------------------------------------- /contrib/namespace_cleanup.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | ecosystem="stagef" 4 | 5 | read -p "Are you at tron-$ecosystem (y/n)?" RES 6 | echo 7 | if [ $RES = "y" ]; then 8 | #load namespace from _manifest.yaml 9 | for namespace in $(cat /nail/tron/config/_manifest.yaml | uq | jq -r 'keys[]') 10 | do 11 | file=$(cat /nail/tron/config/_manifest.yaml | uq | jq -r .\"$namespace\") 12 | filename=$(basename $file) 13 | if [ -f "/nail/etc/services/tron/$ecosystem/$filename" ]; then 14 | echo "$namespace is up to date" 15 | elif [ $namespace == "MASTER" ]; then 16 | echo "It is MASTER namepsace" 17 | else 18 | num_job=$(cat /nail/tron/config/$filename | uq | jq -r ".jobs | length") 19 | echo "========= $filename =========" 20 | cat /nail/tron/config/$filename 21 | echo "=============================" 22 | if [ $num_job == 0 ]; then 23 | echo "$namespace is left behind, deleting the namespace" 24 | tronfig -d $namespace 25 | else 26 | echo "Can't remove the namespace since it is not empty." 27 | fi 28 | fi 29 | done 30 | else 31 | echo "Please change the ecosystem variable in this script or execute this script at tron-$ecosystem" 32 | fi 33 | -------------------------------------------------------------------------------- /contrib/sync-from-yelp-prod.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rsync --exclude=.stderr --exclude=.stdout -aPv tron-prod:/nail/tron/* example-cluster/ 3 | git checkout example-cluster/logging.conf 4 | 5 | echo "" 6 | echo "Now Run:" 7 | echo "" 8 | echo " tox -e example-cluster" 9 | echo " ./example-cluster/start.sh" 10 | -------------------------------------------------------------------------------- /contrib/sync_namespaces_jobs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ This script is for load testing of Tron 3 | 4 | Historically, Tronview and Tronweb were (are) slow. To better understand the performance 5 | bottleneck of Tron, we could use this script to generate the fake namespaces and 6 | jobs as many as we want to perform load testing. Ticket TRON-70 tracks the progress 7 | of speeding up Tronview and Tronweb. 8 | """ 9 | import argparse 10 | import os 11 | 12 | from tron import yaml 13 | 14 | 15 | def parse_args(): 16 | parser = argparse.ArgumentParser( 17 | description="Creating namespaces and jobs configuration for load testing", 18 | ) 19 | parser.add_argument( 20 | "--multiple", 21 | type=int, 22 | default=1, 23 | help="multiple workload of namespaces and jobs from source directory", 24 | ) 25 | parser.add_argument( 26 | "--src", 27 | default="/nail/etc/services/tron/prod", 28 | help="Directory to get Tron configuration files", 29 | ) 30 | parser.add_argument( 31 | "--dest", 32 | default="/tmp/tron-servdir", 33 | help="Directory to put Tron configuration files for load testing", 34 | ) 35 | args = parser.parse_args() 36 | return args 37 | 38 | 39 | def main(): 40 | args = parse_args() 41 | for filename in os.listdir(args.src): 42 | print(f"filename = {filename}") 43 | filepath = os.path.join(args.src, filename) 44 | if os.path.isfile(filepath) and filepath.endswith(".yaml"): 45 | with open(filepath) as f: 46 | config = yaml.load(f) 47 | 48 | if filename == "MASTER.yaml": 49 | for key in list(config): 50 | if key != "jobs": 51 | del config[key] 52 | 53 | jobs = config.get("jobs", []) 54 | if jobs is not None: 55 | for job in jobs: 56 | job["node"] = "localhost" 57 | if "monitoring" in job: 58 | del job["monitoring"] 59 | for action in job.get("actions", []): 60 | action["command"] = "sleep 10s" 61 | if "node" in action: 62 | action["node"] = "localhost" 63 | for i in range(args.multiple): 64 | out_filepath = os.path.join( 65 | args.dest, 66 | "load_testing_" + str(i) + "-" + filename, 67 | ) 68 | with open(out_filepath, "w") as outf: 69 | yaml.dump(config, outf, default_flow_style=False) 70 | 71 | 72 | if __name__ == "__main__": 73 | main() 74 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: tron 2 | Section: admin 3 | Priority: optional 4 | Maintainer: Daniel Nephin 5 | Build-Depends: debhelper (>= 7), python3.8-dev, libdb5.3-dev, libyaml-dev, libssl-dev, libffi-dev, dh-virtualenv 6 | Standards-Version: 3.8.3 7 | 8 | Package: tron 9 | Architecture: all 10 | Homepage: http://github.com/yelp/Tron 11 | Depends: bsdutils, python3.8, libdb5.3, libyaml-0-2, ${shlibs:Depends}, ${misc:Depends} 12 | Description: Tron is a job scheduling, running and monitoring package. 13 | Designed to replace Cron for complex scheduling and dependencies. 14 | Provides: 15 | Centralized configuration for running jobs across multiple machines 16 | Dependencies on jobs and resources 17 | Monitoring of jobs 18 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | This package was debianized by Steve Johnson 2 | on Sat, 26 Nov 2011 15:13:00 -0400. 3 | 4 | It was downloaded from http://github.com/yelp/Tron 5 | 6 | Upstream Author: 7 | 8 | Rhett Garber 9 | Matt Tytel 10 | 11 | Copyright: 12 | 13 | Copyright 2010 Yelp 14 | 15 | License: 16 | 17 | Licensed under the Apache License, Version 2.0 (the "License"); 18 | you may not use this file except in compliance with the License. 19 | You may obtain a copy of the License at 20 | 21 | http://www.apache.org/licenses/LICENSE-2.0 22 | 23 | Unless required by applicable law or agreed to in writing, software 24 | distributed under the License is distributed on an "AS IS" BASIS, 25 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 26 | See the License for the specific language governing permissions and 27 | limitations under the License. 28 | -------------------------------------------------------------------------------- /debian/docs: -------------------------------------------------------------------------------- 1 | LICENSE.txt 2 | README.md 3 | -------------------------------------------------------------------------------- /debian/install: -------------------------------------------------------------------------------- 1 | tron/logging.conf var/lib/tron 2 | tronweb/ opt/venvs/tron/ 3 | -------------------------------------------------------------------------------- /debian/pycompat: -------------------------------------------------------------------------------- 1 | 2 2 | -------------------------------------------------------------------------------- /debian/pyversions: -------------------------------------------------------------------------------- 1 | 2.5-2.6 2 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | DH_VERBOSE := 1 5 | 6 | %: 7 | dh $@ --with python-virtualenv 8 | 9 | # do not call `make clean` as part of packaging 10 | override_dh_auto_clean: 11 | true 12 | 13 | override_dh_auto_build: 14 | true 15 | 16 | # do not call `make test` as part of packaging 17 | override_dh_auto_test: 18 | true 19 | 20 | override_dh_virtualenv: 21 | echo $(PIP_INDEX_URL) 22 | dh_virtualenv --index-url $(PIP_INDEX_URL) \ 23 | --extra-pip-arg --trusted-host=169.254.255.254 \ 24 | --python=/usr/bin/python3.8 \ 25 | --preinstall cython==0.29.36 \ 26 | --preinstall pip==18.1 \ 27 | --preinstall setuptools==46.1.3 28 | @echo patching k8s client lib for configuration class 29 | patch debian/tron/opt/venvs/tron/lib/python3.8/site-packages/kubernetes/client/configuration.py contrib/patch-config-loggers.diff 30 | override_dh_installinit: 31 | dh_installinit --noscripts 32 | -------------------------------------------------------------------------------- /debian/tron.conffiles: -------------------------------------------------------------------------------- 1 | /var/lib/tron/logging.conf 2 | -------------------------------------------------------------------------------- /debian/tron.default: -------------------------------------------------------------------------------- 1 | # Defaults for tron initscript 2 | # sourced by /etc/init.d/tron 3 | # installed at /etc/default/tron by the maintainer scripts 4 | 5 | # 6 | # This is a POSIX shell fragment 7 | # 8 | 9 | # Additional options that are passed to the Daemon. 10 | DAEMON_OPTS="--log-conf /var/lib/tron/logging.conf" 11 | 12 | LISTEN_HOST="0.0.0.0" 13 | LISTEN_PORT="8089" 14 | 15 | # User the daemon will run as. Needs to have appropriate credentials to SSH into your working nodes. 16 | # You should take care in setting permissions appropriately for log and working directories on /var 17 | DAEMONUSER="" 18 | 19 | # Enable this when you have configured tron to your liking. 20 | RUN="no" 21 | -------------------------------------------------------------------------------- /debian/tron.dirs: -------------------------------------------------------------------------------- 1 | var/lib/tron/ 2 | var/log/tron/ 3 | -------------------------------------------------------------------------------- /debian/tron.example: -------------------------------------------------------------------------------- 1 | sample_config.yaml 2 | -------------------------------------------------------------------------------- /debian/tron.links: -------------------------------------------------------------------------------- 1 | opt/venvs/tron/bin/check_tron_jobs usr/bin/check_tron_jobs.py 2 | opt/venvs/tron/bin/tronctl usr/bin/tronctl 3 | opt/venvs/tron/bin/trond usr/bin/trond 4 | opt/venvs/tron/bin/tronfig usr/bin/tronfig 5 | opt/venvs/tron/bin/tronview usr/bin/tronview 6 | opt/venvs/tron/bin/generate_tron_tab_completion_cache usr/bin/generate_tron_tab_completion_cache 7 | opt/venvs/tron/bin/tronctl_tabcomplete.sh usr/share/bash-completion/completions/tronctl 8 | opt/venvs/tron/bin/tronview_tabcomplete.sh usr/share/bash-completion/completions/tronview 9 | -------------------------------------------------------------------------------- /debian/tron.manpages: -------------------------------------------------------------------------------- 1 | docs/source/man/tronctl.1 2 | docs/source/man/trond.8 3 | docs/source/man/tronview.1 4 | docs/source/man/tronfig.1 5 | -------------------------------------------------------------------------------- /debian/tron.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | # 3 | # Post-installation script for tron 4 | 5 | #DEBHELPER# 6 | 7 | exit 0 8 | -------------------------------------------------------------------------------- /debian/tron.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=trond 3 | After=network.target 4 | 5 | [Service] 6 | User=tron 7 | EnvironmentFile=/etc/default/tron 8 | ExecStart=/usr/bin/zk-flock tron_master_${CLUSTER_NAME} "/usr/bin/trond --lock-file=${LOCKFILE:-$PIDFILE} --working-dir=${WORKINGDIR} --host ${LISTEN_HOST} --port ${LISTEN_PORT} ${DAEMON_OPTS}" 9 | ExecStopPost=/usr/bin/logger -t tron_exit_status "SERVICE_RESULT:${SERVICE_RESULT} EXIT_CODE:${EXIT_CODE} EXIT_STATUS:${EXIT_STATUS}" 10 | TimeoutStopSec=20 11 | Restart=always 12 | LimitNOFILE=100000 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /debian/tron.upstart: -------------------------------------------------------------------------------- 1 | description "trond" 2 | 3 | start on filesystem and (started networking) 4 | stop on shutdown 5 | 6 | respawn 7 | kill timeout 20 8 | 9 | script 10 | set -a 11 | if [ -f /etc/default/tron ] ; then 12 | . /etc/default/tron 13 | fi 14 | if [ "x$RUN" != "xyes" ]; then 15 | log_failure_msg "$NAME disabled, please adjust the configuration to your needs " 16 | log_failure_msg "and then set RUN to 'yes' in /etc/default/$NAME to enable it." 17 | exit 0 18 | fi 19 | exec start-stop-daemon --start -c $DAEMONUSER --exec /usr/bin/trond -- $DAEMON_OPTS 20 | end script 21 | -------------------------------------------------------------------------------- /debian/watch: -------------------------------------------------------------------------------- 1 | # Example watch control file for uscan 2 | # Rename this file to "watch" and then you can run the "uscan" command 3 | # to check for upstream updates and more. 4 | # See uscan(1) for format 5 | 6 | # Compulsory line, this is a version 3 file 7 | version=3 8 | 9 | http://githubredir.debian.net/github/Yelp/Tron 10 | -------------------------------------------------------------------------------- /dev/config/MASTER.yaml: -------------------------------------------------------------------------------- 1 | # Please visit y/tron-development for a guide on how to setup Tron for local development 2 | state_persistence: 3 | name: "tron_state" 4 | table_name: "tmp-tron-state" 5 | store_type: "dynamodb" 6 | buffer_size: 1 7 | dynamodb_region: us-west-1 8 | 9 | eventbus_enabled: True 10 | ssh_options: 11 | agent: True 12 | 13 | nodes: 14 | - hostname: localhost 15 | 16 | # Replace this with the path relative to your home dir to use 17 | # action_runner: 18 | # runner_type: "subprocess" 19 | # remote_status_path: "pg/tron/status" 20 | # remote_exec_path: "pg/tron/.tox/py38/bin" 21 | 22 | jobs: 23 | testjob0: 24 | enabled: true 25 | node: localhost 26 | schedule: "cron * * * * *" 27 | run_limit: 5 28 | actions: 29 | zeroth: 30 | command: env 31 | trigger_downstreams: 32 | minutely: "{ymdhm}" 33 | cpus: 1 34 | mem: 100 35 | 36 | testjob1: 37 | enabled: false 38 | node: localhost 39 | schedule: "cron * * * * *" 40 | actions: 41 | first: 42 | command: "sleep 5" 43 | cpus: 1 44 | mem: 100 45 | second: 46 | command: "echo 'hello world'" 47 | requires: [first] 48 | triggered_by: 49 | - "MASTER.testjob0.zeroth.minutely.{ymdhm}" 50 | trigger_downstreams: 51 | minutely: "{ymdhm}" 52 | cpus: 1 53 | mem: 100 54 | 55 | testjob2: 56 | enabled: false 57 | node: localhost 58 | schedule: "cron * * * * *" 59 | actions: 60 | first: 61 | command: "echo 'goodbye, world'" 62 | cpus: 1 63 | mem: 100 64 | triggered_by: 65 | - "MASTER.testjob1.second.minutely.{ymdhm}" 66 | 67 | retrier: 68 | node: localhost 69 | schedule: "cron 0 0 1 1 *" 70 | actions: 71 | failing: 72 | command: exit 1 73 | retries: 1 74 | retries_delay: 5m 75 | -------------------------------------------------------------------------------- /dev/config/_manifest.yaml: -------------------------------------------------------------------------------- 1 | MASTER: config/MASTER.yaml 2 | -------------------------------------------------------------------------------- /dev/logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root, twisted, tron, tron.serialize.runstate.statemanager, tron.api.www.access, task_processing, tron.mesos.task_output, pymesos 3 | 4 | [handlers] 5 | keys=stdoutHandler, accessHandler, nullHandler 6 | 7 | [formatters] 8 | keys=defaultFormatter, accessFormatter 9 | 10 | [logger_root] 11 | level=WARN 12 | handlers=stdoutHandler 13 | 14 | [logger_twisted] 15 | level=WARN 16 | handlers=stdoutHandler 17 | qualname=twisted 18 | propagate=0 19 | 20 | [logger_tron] 21 | level=DEBUG 22 | handlers=stdoutHandler 23 | qualname=tron 24 | propagate=0 25 | 26 | [logger_tron.api.www.access] 27 | level=DEBUG 28 | handlers=accessHandler 29 | qualname=tron.api.www.access 30 | propagate=0 31 | 32 | [logger_tron.serialize.runstate.statemanager] 33 | level=DEBUG 34 | handlers=stdoutHandler 35 | qualname=tron.serialize.runstate.statemanager 36 | propagate=0 37 | 38 | [logger_task_processing] 39 | level=INFO 40 | handlers=stdoutHandler 41 | qualname=task_processing 42 | propagate=0 43 | 44 | [logger_pymesos] 45 | level=DEBUG 46 | handlers=stdoutHandler 47 | qualname=pymesos 48 | propagate=0 49 | 50 | [logger_tron.mesos.task_output] 51 | level=INFO 52 | handlers=nullHandler 53 | qualname=tron.mesos.task_output 54 | propagate=0 55 | 56 | [handler_stdoutHandler] 57 | class=logging.StreamHandler 58 | level=DEBUG 59 | formatter=defaultFormatter 60 | args=() 61 | 62 | [handler_nullHandler] 63 | class=logging.NullHandler 64 | level=DEBUG 65 | args=() 66 | 67 | [handler_accessHandler] 68 | class=logging.StreamHandler 69 | level=DEBUG 70 | formatter=accessFormatter 71 | args=() 72 | 73 | [formatter_defaultFormatter] 74 | format=%(asctime)s %(name)s %(levelname)s %(message)s 75 | 76 | [formatter_accessFormatter] 77 | format=%(message)s 78 | -------------------------------------------------------------------------------- /dev/tron.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/dev/tron.lock -------------------------------------------------------------------------------- /docs/source/developing.rst: -------------------------------------------------------------------------------- 1 | .. _developing: 2 | 3 | Contributing to Tron 4 | ==================== 5 | 6 | Tron is an open source project and welcomes contributions from the community. 7 | The source and issue tracker are hosted on github at 8 | http://github.com/yelp/Tron. 9 | 10 | Setting Up an Environment 11 | ------------------------- 12 | 13 | Tron works well with `virtualenv `_, which can be 14 | setup using `virtualenvwrapper 15 | `_:: 16 | 17 | $ mkvirtualenv tron --distribute --no-site-packages 18 | $ pip install -r dev/req_dev.txt 19 | 20 | ``req_dev.txt`` contains a list of packages required for development, 21 | to run the tests, and `Sphinx `_ to build the documentation. 22 | 23 | Coding Standards 24 | ---------------- 25 | 26 | All code should be `PEP8 `_ compliant, 27 | and should pass pyflakes without warnings. All new code should include full 28 | test coverage, and bug fixes should include a test which reproduces the 29 | reported issue. 30 | 31 | This documentation must also be kept up to date with any changes in functionality. 32 | 33 | 34 | Running Tron in a Sandbox 35 | ------------------------- 36 | 37 | The source package includes a development logging.conf and a 38 | sample configuration file with a few test cases. To run a development instance 39 | of Tron create a working directory and start 40 | :command:`trond` using the following:: 41 | 42 | $ make dev 43 | 44 | 45 | Running the Tests 46 | ----------------- 47 | Run the tests using ``make test``. 48 | 49 | Contributing 50 | ------------ 51 | 52 | There should be a github issue created prior to all pull requests. Pull requests 53 | should be made to the ``Yelp:development`` branch, and should include additions to 54 | ``CHANGES.txt`` which describe what has changed. 55 | -------------------------------------------------------------------------------- /docs/source/generated/modules.rst: -------------------------------------------------------------------------------- 1 | tron 2 | ==== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | tron 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.actioncommand.rst: -------------------------------------------------------------------------------- 1 | tron.actioncommand module 2 | ========================= 3 | 4 | .. automodule:: tron.actioncommand 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.api.adapter.rst: -------------------------------------------------------------------------------- 1 | tron.api.adapter module 2 | ======================= 3 | 4 | .. automodule:: tron.api.adapter 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.api.async_resource.rst: -------------------------------------------------------------------------------- 1 | tron.api.async\_resource module 2 | =============================== 3 | 4 | .. automodule:: tron.api.async_resource 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.api.auth.rst: -------------------------------------------------------------------------------- 1 | tron.api.auth module 2 | ==================== 3 | 4 | .. automodule:: tron.api.auth 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.api.controller.rst: -------------------------------------------------------------------------------- 1 | tron.api.controller module 2 | ========================== 3 | 4 | .. automodule:: tron.api.controller 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.api.requestargs.rst: -------------------------------------------------------------------------------- 1 | tron.api.requestargs module 2 | =========================== 3 | 4 | .. automodule:: tron.api.requestargs 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.api.resource.rst: -------------------------------------------------------------------------------- 1 | tron.api.resource module 2 | ======================== 3 | 4 | .. automodule:: tron.api.resource 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.api.rst: -------------------------------------------------------------------------------- 1 | tron.api package 2 | ================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | tron.api.adapter 11 | tron.api.async_resource 12 | tron.api.auth 13 | tron.api.controller 14 | tron.api.requestargs 15 | tron.api.resource 16 | 17 | Module contents 18 | --------------- 19 | 20 | .. automodule:: tron.api 21 | :members: 22 | :undoc-members: 23 | :show-inheritance: 24 | -------------------------------------------------------------------------------- /docs/source/generated/tron.command_context.rst: -------------------------------------------------------------------------------- 1 | tron.command\_context module 2 | ============================ 3 | 4 | .. automodule:: tron.command_context 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.commands.authentication.rst: -------------------------------------------------------------------------------- 1 | tron.commands.authentication module 2 | =================================== 3 | 4 | .. automodule:: tron.commands.authentication 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.commands.backfill.rst: -------------------------------------------------------------------------------- 1 | tron.commands.backfill module 2 | ============================= 3 | 4 | .. automodule:: tron.commands.backfill 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.commands.client.rst: -------------------------------------------------------------------------------- 1 | tron.commands.client module 2 | =========================== 3 | 4 | .. automodule:: tron.commands.client 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.commands.cmd_utils.rst: -------------------------------------------------------------------------------- 1 | tron.commands.cmd\_utils module 2 | =============================== 3 | 4 | .. automodule:: tron.commands.cmd_utils 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.commands.display.rst: -------------------------------------------------------------------------------- 1 | tron.commands.display module 2 | ============================ 3 | 4 | .. automodule:: tron.commands.display 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.commands.retry.rst: -------------------------------------------------------------------------------- 1 | tron.commands.retry module 2 | ========================== 3 | 4 | .. automodule:: tron.commands.retry 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.commands.rst: -------------------------------------------------------------------------------- 1 | tron.commands package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | tron.commands.authentication 11 | tron.commands.backfill 12 | tron.commands.client 13 | tron.commands.cmd_utils 14 | tron.commands.display 15 | tron.commands.retry 16 | 17 | Module contents 18 | --------------- 19 | 20 | .. automodule:: tron.commands 21 | :members: 22 | :undoc-members: 23 | :show-inheritance: 24 | -------------------------------------------------------------------------------- /docs/source/generated/tron.config.config_parse.rst: -------------------------------------------------------------------------------- 1 | tron.config.config\_parse module 2 | ================================ 3 | 4 | .. automodule:: tron.config.config_parse 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.config.config_utils.rst: -------------------------------------------------------------------------------- 1 | tron.config.config\_utils module 2 | ================================ 3 | 4 | .. automodule:: tron.config.config_utils 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.config.manager.rst: -------------------------------------------------------------------------------- 1 | tron.config.manager module 2 | ========================== 3 | 4 | .. automodule:: tron.config.manager 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.config.rst: -------------------------------------------------------------------------------- 1 | tron.config package 2 | =================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | tron.config.config_parse 11 | tron.config.config_utils 12 | tron.config.manager 13 | tron.config.schedule_parse 14 | tron.config.schema 15 | tron.config.static_config 16 | 17 | Module contents 18 | --------------- 19 | 20 | .. automodule:: tron.config 21 | :members: 22 | :undoc-members: 23 | :show-inheritance: 24 | -------------------------------------------------------------------------------- /docs/source/generated/tron.config.schedule_parse.rst: -------------------------------------------------------------------------------- 1 | tron.config.schedule\_parse module 2 | ================================== 3 | 4 | .. automodule:: tron.config.schedule_parse 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.config.schema.rst: -------------------------------------------------------------------------------- 1 | tron.config.schema module 2 | ========================= 3 | 4 | .. automodule:: tron.config.schema 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.config.static_config.rst: -------------------------------------------------------------------------------- 1 | tron.config.static\_config module 2 | ================================= 3 | 4 | .. automodule:: tron.config.static_config 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.action.rst: -------------------------------------------------------------------------------- 1 | tron.core.action module 2 | ======================= 3 | 4 | .. automodule:: tron.core.action 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.actiongraph.rst: -------------------------------------------------------------------------------- 1 | tron.core.actiongraph module 2 | ============================ 3 | 4 | .. automodule:: tron.core.actiongraph 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.actionrun.rst: -------------------------------------------------------------------------------- 1 | tron.core.actionrun module 2 | ========================== 3 | 4 | .. automodule:: tron.core.actionrun 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.job.rst: -------------------------------------------------------------------------------- 1 | tron.core.job module 2 | ==================== 3 | 4 | .. automodule:: tron.core.job 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.job_collection.rst: -------------------------------------------------------------------------------- 1 | tron.core.job\_collection module 2 | ================================ 3 | 4 | .. automodule:: tron.core.job_collection 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.job_scheduler.rst: -------------------------------------------------------------------------------- 1 | tron.core.job\_scheduler module 2 | =============================== 3 | 4 | .. automodule:: tron.core.job_scheduler 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.jobgraph.rst: -------------------------------------------------------------------------------- 1 | tron.core.jobgraph module 2 | ========================= 3 | 4 | .. automodule:: tron.core.jobgraph 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.jobrun.rst: -------------------------------------------------------------------------------- 1 | tron.core.jobrun module 2 | ======================= 3 | 4 | .. automodule:: tron.core.jobrun 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.recovery.rst: -------------------------------------------------------------------------------- 1 | tron.core.recovery module 2 | ========================= 3 | 4 | .. automodule:: tron.core.recovery 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.core.rst: -------------------------------------------------------------------------------- 1 | tron.core package 2 | ================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | tron.core.action 11 | tron.core.actiongraph 12 | tron.core.actionrun 13 | tron.core.job 14 | tron.core.job_collection 15 | tron.core.job_scheduler 16 | tron.core.jobgraph 17 | tron.core.jobrun 18 | tron.core.recovery 19 | 20 | Module contents 21 | --------------- 22 | 23 | .. automodule:: tron.core 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | -------------------------------------------------------------------------------- /docs/source/generated/tron.eventbus.rst: -------------------------------------------------------------------------------- 1 | tron.eventbus module 2 | ==================== 3 | 4 | .. automodule:: tron.eventbus 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.kubernetes.rst: -------------------------------------------------------------------------------- 1 | tron.kubernetes module 2 | ====================== 3 | 4 | .. automodule:: tron.kubernetes 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.manhole.rst: -------------------------------------------------------------------------------- 1 | tron.manhole module 2 | =================== 3 | 4 | .. automodule:: tron.manhole 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.mcp.rst: -------------------------------------------------------------------------------- 1 | tron.mcp module 2 | =============== 3 | 4 | .. automodule:: tron.mcp 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.mesos.rst: -------------------------------------------------------------------------------- 1 | tron.mesos module 2 | ================= 3 | 4 | .. automodule:: tron.mesos 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.metrics.rst: -------------------------------------------------------------------------------- 1 | tron.metrics module 2 | =================== 3 | 4 | .. automodule:: tron.metrics 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.node.rst: -------------------------------------------------------------------------------- 1 | tron.node module 2 | ================ 3 | 4 | .. automodule:: tron.node 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.prom_metrics.rst: -------------------------------------------------------------------------------- 1 | tron.prom\_metrics module 2 | ========================= 3 | 4 | .. automodule:: tron.prom_metrics 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.rst: -------------------------------------------------------------------------------- 1 | tron package 2 | ============ 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | tron.api 11 | tron.commands 12 | tron.config 13 | tron.core 14 | tron.serialize 15 | tron.utils 16 | 17 | Submodules 18 | ---------- 19 | 20 | .. toctree:: 21 | :maxdepth: 4 22 | 23 | tron.actioncommand 24 | tron.command_context 25 | tron.eventbus 26 | tron.kubernetes 27 | tron.manhole 28 | tron.mcp 29 | tron.mesos 30 | tron.metrics 31 | tron.node 32 | tron.prom_metrics 33 | tron.scheduler 34 | tron.ssh 35 | tron.trondaemon 36 | tron.yaml 37 | 38 | Module contents 39 | --------------- 40 | 41 | .. automodule:: tron 42 | :members: 43 | :undoc-members: 44 | :show-inheritance: 45 | -------------------------------------------------------------------------------- /docs/source/generated/tron.scheduler.rst: -------------------------------------------------------------------------------- 1 | tron.scheduler module 2 | ===================== 3 | 4 | .. automodule:: tron.scheduler 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.serialize.filehandler.rst: -------------------------------------------------------------------------------- 1 | tron.serialize.filehandler module 2 | ================================= 3 | 4 | .. automodule:: tron.serialize.filehandler 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.serialize.rst: -------------------------------------------------------------------------------- 1 | tron.serialize package 2 | ====================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | tron.serialize.runstate 11 | 12 | Submodules 13 | ---------- 14 | 15 | .. toctree:: 16 | :maxdepth: 4 17 | 18 | tron.serialize.filehandler 19 | 20 | Module contents 21 | --------------- 22 | 23 | .. automodule:: tron.serialize 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | -------------------------------------------------------------------------------- /docs/source/generated/tron.serialize.runstate.dynamodb_state_store.rst: -------------------------------------------------------------------------------- 1 | tron.serialize.runstate.dynamodb\_state\_store module 2 | ===================================================== 3 | 4 | .. automodule:: tron.serialize.runstate.dynamodb_state_store 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.serialize.runstate.rst: -------------------------------------------------------------------------------- 1 | tron.serialize.runstate package 2 | =============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | tron.serialize.runstate.dynamodb_state_store 11 | tron.serialize.runstate.shelvestore 12 | tron.serialize.runstate.statemanager 13 | tron.serialize.runstate.yamlstore 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: tron.serialize.runstate 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /docs/source/generated/tron.serialize.runstate.shelvestore.rst: -------------------------------------------------------------------------------- 1 | tron.serialize.runstate.shelvestore module 2 | ========================================== 3 | 4 | .. automodule:: tron.serialize.runstate.shelvestore 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.serialize.runstate.statemanager.rst: -------------------------------------------------------------------------------- 1 | tron.serialize.runstate.statemanager module 2 | =========================================== 3 | 4 | .. automodule:: tron.serialize.runstate.statemanager 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.serialize.runstate.yamlstore.rst: -------------------------------------------------------------------------------- 1 | tron.serialize.runstate.yamlstore module 2 | ======================================== 3 | 4 | .. automodule:: tron.serialize.runstate.yamlstore 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.ssh.rst: -------------------------------------------------------------------------------- 1 | tron.ssh module 2 | =============== 3 | 4 | .. automodule:: tron.ssh 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.trondaemon.rst: -------------------------------------------------------------------------------- 1 | tron.trondaemon module 2 | ====================== 3 | 4 | .. automodule:: tron.trondaemon 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.collections.rst: -------------------------------------------------------------------------------- 1 | tron.utils.collections module 2 | ============================= 3 | 4 | .. automodule:: tron.utils.collections 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.crontab.rst: -------------------------------------------------------------------------------- 1 | tron.utils.crontab module 2 | ========================= 3 | 4 | .. automodule:: tron.utils.crontab 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.exitcode.rst: -------------------------------------------------------------------------------- 1 | tron.utils.exitcode module 2 | ========================== 3 | 4 | .. automodule:: tron.utils.exitcode 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.logreader.rst: -------------------------------------------------------------------------------- 1 | tron.utils.logreader module 2 | =========================== 3 | 4 | .. automodule:: tron.utils.logreader 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.observer.rst: -------------------------------------------------------------------------------- 1 | tron.utils.observer module 2 | ========================== 3 | 4 | .. automodule:: tron.utils.observer 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.persistable.rst: -------------------------------------------------------------------------------- 1 | tron.utils.persistable module 2 | ============================= 3 | 4 | .. automodule:: tron.utils.persistable 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.proxy.rst: -------------------------------------------------------------------------------- 1 | tron.utils.proxy module 2 | ======================= 3 | 4 | .. automodule:: tron.utils.proxy 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.queue.rst: -------------------------------------------------------------------------------- 1 | tron.utils.queue module 2 | ======================= 3 | 4 | .. automodule:: tron.utils.queue 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.rst: -------------------------------------------------------------------------------- 1 | tron.utils package 2 | ================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | tron.utils.collections 11 | tron.utils.crontab 12 | tron.utils.exitcode 13 | tron.utils.logreader 14 | tron.utils.observer 15 | tron.utils.persistable 16 | tron.utils.proxy 17 | tron.utils.queue 18 | tron.utils.state 19 | tron.utils.timeutils 20 | tron.utils.trontimespec 21 | tron.utils.twistedutils 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: tron.utils 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.state.rst: -------------------------------------------------------------------------------- 1 | tron.utils.state module 2 | ======================= 3 | 4 | .. automodule:: tron.utils.state 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.timeutils.rst: -------------------------------------------------------------------------------- 1 | tron.utils.timeutils module 2 | =========================== 3 | 4 | .. automodule:: tron.utils.timeutils 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.trontimespec.rst: -------------------------------------------------------------------------------- 1 | tron.utils.trontimespec module 2 | ============================== 3 | 4 | .. automodule:: tron.utils.trontimespec 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.utils.twistedutils.rst: -------------------------------------------------------------------------------- 1 | tron.utils.twistedutils module 2 | ============================== 3 | 4 | .. automodule:: tron.utils.twistedutils 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/generated/tron.yaml.rst: -------------------------------------------------------------------------------- 1 | tron.yaml module 2 | ================ 3 | 4 | .. automodule:: tron.yaml 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Tron 2 | ==== 3 | 4 | Tron is a centralized system for managing periodic batch processes 5 | across a cluster. If this is your first time using Tron, read :doc:`tutorial` 6 | and :doc:`overview` to get a better idea of what it is, how it works, and how 7 | to use it. 8 | 9 | .. note:: 10 | 11 | Please report bugs in the documentation at `our Github issue tracker 12 | `_. 13 | 14 | Table of Contents 15 | ----------------- 16 | 17 | .. toctree:: 18 | :maxdepth: 2 19 | 20 | whats-new.rst 21 | tutorial.rst 22 | overview.rst 23 | config.rst 24 | jobs.rst 25 | command_context.rst 26 | tronweb.rst 27 | tools.rst 28 | developing.rst 29 | 30 | Generated Docs 31 | ~~~~~~~~~~~~~~ 32 | 33 | .. toctree:: 34 | :maxdepth: 1 35 | 36 | generated/modules 37 | 38 | Indices and tables 39 | ================== 40 | 41 | * :ref:`genindex` 42 | * :ref:`search` 43 | -------------------------------------------------------------------------------- /docs/source/man/tronfig.1: -------------------------------------------------------------------------------- 1 | .TH "TRONFIG" "1" "April 24, 2013" "0.6" "Tron" 2 | .SH NAME 3 | tronfig \- tronfig documentation 4 | . 5 | .nr rst2man-indent-level 0 6 | . 7 | .de1 rstReportMargin 8 | \\$1 \\n[an-margin] 9 | level \\n[rst2man-indent-level] 10 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 11 | - 12 | \\n[rst2man-indent0] 13 | \\n[rst2man-indent1] 14 | \\n[rst2man-indent2] 15 | .. 16 | .de1 INDENT 17 | .\" .rstReportMargin pre: 18 | . RS \\$1 19 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 20 | . nr rst2man-indent-level +1 21 | .\" .rstReportMargin post: 22 | .. 23 | .de UNINDENT 24 | . RE 25 | .\" indent \\n[an-margin] 26 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 27 | .nr rst2man-indent-level -1 28 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 30 | .. 31 | .\" Man page generated from reStructeredText. 32 | . 33 | .SH SYNOPSYS 34 | .sp 35 | \fBtronfig [\-\-server server_name ] [\-\-verbose | \-v] [] [\-p] [\-]\fP 36 | .SH DESCRIPTION 37 | .sp 38 | \fBtronfig\fP allows live editing of the Tron configuration. It retrieves 39 | the configuration file for local editing, verifies the configuration, 40 | and sends it back to the tron server. The configuration is applied 41 | immediately. 42 | .SH OPTIONS 43 | .INDENT 0.0 44 | .TP 45 | .B \fB\-\-server \fP 46 | The server the tron instance is running on 47 | .TP 48 | .B \fB\-\-verbose\fP 49 | Displays status messages along the way 50 | .TP 51 | .B \fB\-\-version\fP 52 | Displays version string 53 | .TP 54 | .B \fB\-p\fP 55 | Print the configuration 56 | .TP 57 | .B \fBnamespace\fP 58 | The configuration namespace to edit. Defaults to MASTER 59 | .TP 60 | .B \fB\-\fP 61 | Read new config from \fBstdin\fP. 62 | .UNINDENT 63 | .SH CONFIGURATION 64 | .sp 65 | By default tron will run with a blank configuration file. The config file is 66 | saved to \fB/config/\fP by default. See the full documentation at 67 | \fI\%http://tron.readthedocs.io/en/latest/config.html\fP. 68 | .SH BUGS 69 | .sp 70 | Post bugs to \fI\%http://www.github.com/yelp/tron/issues\fP. 71 | .SH SEE ALSO 72 | .sp 73 | \fBtrond\fP (8), \fBtronctl\fP (1), \fBtronview\fP (1), 74 | .SH AUTHOR 75 | Yelp, Inc. 76 | .SH COPYRIGHT 77 | 2011, Yelp, Inc. 78 | .\" Generated by docutils manpage writer. 79 | .\" 80 | . 81 | -------------------------------------------------------------------------------- /docs/source/man_trond.rst: -------------------------------------------------------------------------------- 1 | .. _trond: 2 | 3 | trond 4 | ===== 5 | 6 | Synopsis 7 | -------- 8 | 9 | ``trond [--working-dir=] [--verbose] [--debug]`` 10 | 11 | Description 12 | ----------- 13 | 14 | **trond** is the tron daemon that manages all jobs. 15 | 16 | Options 17 | ------- 18 | 19 | ``--version`` 20 | show program's version number and exit 21 | 22 | ``-h, --help`` 23 | show this help message and exit 24 | 25 | ``--working-dir=WORKING_DIR`` 26 | Directory where tron's state and output is stored (default /var/lib/tron/) 27 | 28 | ``-l LOG_CONF, --log-conf=LOG_CONF`` 29 | Logging configuration file to setup python logger 30 | 31 | ``-c CONFIG_FILE, --config-file=CONFIG_FILE`` 32 | Configuration file to load (default in working dir) 33 | 34 | ``-v, --verbose`` 35 | Verbose logging 36 | 37 | ``--debug`` 38 | Debug mode, extra error reporting, no daemonizing 39 | 40 | ``--nodaemon`` 41 | [DEPRECATED in 0.9.4] Indicates we should not fork and daemonize the process (default False) 42 | 43 | ``--lock-file=LOCKFILE`` 44 | Where to store the lock file of the executing process (default /var/run/tron.lock) 45 | 46 | ``-P LISTEN_PORT, --port=LISTEN_PORT`` 47 | What port to listen on, defaults 8089 48 | 49 | ``-H LISTEN_HOST, --host=LISTEN_HOST`` 50 | What host to listen on defaults to localhost 51 | 52 | Files 53 | ----- 54 | 55 | Working directory 56 | The directory where state and saved output of processes are stored. 57 | 58 | Lock file 59 | Ensures only one daemon runs at a time. 60 | 61 | Log File 62 | trond error log, configured from logging.conf 63 | 64 | 65 | Signals 66 | ------- 67 | 68 | `SIGINT` 69 | Graceful shutdown. Waits for running jobs to complete. 70 | 71 | `SIGTERM` 72 | Does some cleanup before shutting down. 73 | 74 | `SIGHUP` 75 | Reload the configuration file. 76 | 77 | `SIGUSR1` 78 | Will drop into an ipdb debugging prompt. 79 | 80 | Logging 81 | ------- 82 | 83 | Tron uses Python's standard logging and by default uses a rotating log file 84 | handler that rotates files each day. Logs go to ``/var/log/tron/tron.log``. 85 | 86 | To configure logging pass -l to trond. You can modify the 87 | default logging.conf by coping it from tron/logging.conf. See 88 | http://docs.python.org/howto/logging.html#configuring-logging 89 | 90 | 91 | Bugs 92 | ---- 93 | 94 | trond has issues around daylight savings time and may run jobs an hour early 95 | at the boundary. 96 | 97 | Post further bugs to http://www.github.com/yelp/tron/issues. 98 | 99 | See Also 100 | -------- 101 | 102 | **tronctl** (1), **tronfig** (1), **tronview** (1), 103 | -------------------------------------------------------------------------------- /docs/source/man_tronfig.rst: -------------------------------------------------------------------------------- 1 | .. _tronfig: 2 | 3 | tronfig 4 | ======= 5 | 6 | Synopsis 7 | -------- 8 | 9 | ``tronfig [--server server_name ] [--verbose | -v] [] [-p] [-]`` 10 | 11 | Description 12 | ----------- 13 | 14 | **tronfig** allows live editing of the Tron configuration. It retrieves 15 | the configuration file for local editing, verifies the configuration, 16 | and sends it back to the tron server. The configuration is applied 17 | immediately. 18 | 19 | Options 20 | ------- 21 | 22 | ``--server `` 23 | The server the tron instance is running on 24 | 25 | ``--verbose`` 26 | Displays status messages along the way 27 | 28 | ``--version`` 29 | Displays version string 30 | 31 | ``-p`` 32 | Print the configuration 33 | 34 | ``namespace`` 35 | The configuration namespace to edit. Defaults to MASTER 36 | 37 | ``-`` 38 | Read new config from ``stdin``. 39 | 40 | Configuration 41 | ------------- 42 | 43 | By default tron will run with a blank configuration file. The config file is 44 | saved to ``/config/`` by default. See the full documentation at 45 | http://tron.readthedocs.io/en/latest/config.html. 46 | 47 | 48 | Bugs 49 | ---- 50 | 51 | Post bugs to http://www.github.com/yelp/tron/issues. 52 | 53 | See Also 54 | -------- 55 | 56 | **trond** (8), **tronctl** (1), **tronview** (1), 57 | -------------------------------------------------------------------------------- /docs/source/man_tronview.rst: -------------------------------------------------------------------------------- 1 | .. _tronview: 2 | 3 | tronview 4 | ======== 5 | 6 | Synopsis 7 | -------- 8 | 9 | ``tronview [-n ] [--server ] [--verbose] [ | | ]`` 10 | 11 | Description 12 | ----------- 13 | 14 | **tronview** displays the status of tron scheduled jobs. 15 | 16 | tronview 17 | Show all configured jobs 18 | 19 | tronview 20 | Shows details for a job. Ex:: 21 | 22 | $ tronview my_job 23 | 24 | tronview 25 | Show details for specific run or instance. Ex:: 26 | 27 | $ tronview my_job.0 28 | 29 | tronview 30 | Show details for specific action run. Ex:: 31 | 32 | $ tronview my_job.0.my_action 33 | 34 | Options 35 | ------- 36 | 37 | ``--version`` 38 | show program's version number and exit 39 | 40 | ``-h, --help`` 41 | show this help message and exit 42 | 43 | ``-v, --verbose`` 44 | Verbose logging 45 | 46 | ``-n NUM_DISPLAYS, --numshown=NUM_DISPLAYS`` 47 | The maximum number of job runs or lines of output to display(0 for show 48 | all). Does not affect the display of all jobs and the display of actions 49 | for given job. 50 | 51 | ``--server=SERVER`` 52 | Server URL to connect to 53 | 54 | ``-c, --color`` 55 | Display in color 56 | 57 | ``--nocolor`` 58 | Display without color 59 | 60 | ``-o, --stdout`` 61 | Solely displays stdout 62 | 63 | ``-e, --stderr`` 64 | Solely displays stderr 65 | 66 | ``-s, --save`` 67 | Save server and color options to client config file (~/.tron) 68 | 69 | 70 | States 71 | ---------- 72 | For complete list of states with a diagram of valid transitions see 73 | http://packages.python.org/tron/jobs.html#states 74 | 75 | 76 | Bugs 77 | ---- 78 | 79 | Post bugs to http://www.github.com/yelp/tron/issues. 80 | 81 | See Also 82 | -------- 83 | 84 | **trond** (8), **tronctl** (1), **tronfig** (1), 85 | -------------------------------------------------------------------------------- /docs/source/sample_config.yaml: -------------------------------------------------------------------------------- 1 | # optional and settable from the command line 2 | working_dir: './working' 3 | 4 | # optional 5 | ssh_options: 6 | agent: true # default False 7 | identities: # default [] 8 | - "/home/batch/.ssh/id_dsa-nopasswd" 9 | 10 | command_context: 11 | PYTHON: /usr/bin/python 12 | TMPDIR: /tmp 13 | 14 | # required 15 | nodes: 16 | - name: node1 17 | hostname: 'batch1' 18 | username: 'tronuser' 19 | - name: node2 20 | hostname: 'batch2' 21 | username: 'tronuser' 22 | 23 | node_pools: 24 | - name: pool 25 | nodes: [node1, node2] 26 | 27 | jobs: 28 | "job0": 29 | node: pool 30 | all_nodes: True 31 | schedule: 32 | start_time: 04:00:00 33 | queueing: False 34 | actions: 35 | "verify_logs_present": 36 | command: > 37 | ls /var/log/app/log_{shortdate-1}.txt 38 | "convert_logs": 39 | command: > 40 | convert_logs /var/log/app/log_{shortdate-1}.txt \ 41 | /var/log/app_converted/log_{shortdate-1}.txt 42 | requires: [verify_logs_present] 43 | # this will run when the job succeeds or fails 44 | cleanup_action: 45 | command: "rm /{TMPDIR}/random_temp_file" 46 | 47 | "job1": 48 | node: node 49 | schedule: "every monday at 09:00" 50 | queueing: False 51 | actions: 52 | "actionAlone": 53 | command: "cat big.txt; sleep 10" 54 | -------------------------------------------------------------------------------- /docs/source/tools.rst: -------------------------------------------------------------------------------- 1 | Man Pages 2 | ========= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | man_tronctl.rst 8 | man_trond.rst 9 | man_tronfig.rst 10 | man_tronview.rst 11 | -------------------------------------------------------------------------------- /docs/source/tron.yaml: -------------------------------------------------------------------------------- 1 | ssh_options: 2 | agent: true 3 | 4 | nodes: 5 | - name: node0 6 | hostname: 'localhost' 7 | 8 | jobs: 9 | "uptime_job": 10 | node: node0 11 | schedule: "cron */10 * * * *" 12 | actions: 13 | "uptimer": 14 | command: "uptime" 15 | -------------------------------------------------------------------------------- /docs/source/tronweb.rst: -------------------------------------------------------------------------------- 1 | .. _tronweb: 2 | 3 | tronweb 4 | ======== 5 | 6 | tronweb is the web-based UI for tron. 7 | 8 | See http://localhost:8089/web/ 9 | -------------------------------------------------------------------------------- /docs/source/whats-new.rst: -------------------------------------------------------------------------------- 1 | What's New 2 | ========== 3 | 4 | See the `CHANGELOG `_. 5 | -------------------------------------------------------------------------------- /itest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | export DEBIAN_FRONTEND=noninteractive 6 | 7 | apt-get update 8 | apt-get install -y software-properties-common gdebi-core curl 9 | add-apt-repository -y ppa:deadsnakes/ppa 10 | apt-get update 11 | 12 | gdebi --non-interactive /work/dist/*.deb 13 | 14 | # TODO: change default MASTER config to not require ssh agent 15 | apt-get install -y ssh 16 | service ssh start 17 | eval $(ssh-agent) 18 | 19 | trond --help 20 | tronfig --help 21 | 22 | 23 | /opt/venvs/tron/bin/python - </dev/null; then 37 | break 38 | fi 39 | if [ "$i" == "5" ]; then 40 | echo "Failed to start" 41 | kill -9 $TRON_PID 42 | exit 1 43 | fi 44 | sleep 1 45 | done 46 | kill -0 $TRON_PID 47 | 48 | curl localhost:8089/api/status | grep -qi alive 49 | 50 | tronfig -p MASTER 51 | tronfig -n MASTER /work/testfiles/MASTER.yaml 52 | tronfig /work/testfiles/MASTER.yaml 53 | cat /work/testfiles/MASTER.yaml | tronfig -n MASTER - 54 | 55 | if test -L /opt/venvs/tron/lib/python3.8/encodings/punycode.py; then 56 | echo "Whoa, the tron package shouldn't have an encoding symlink!" 57 | echo "Check out https://github.com/spotify/dh-virtualenv/issues/272" 58 | exit 1 59 | fi 60 | 61 | kill -SIGTERM $TRON_PID 62 | wait $TRON_PID || true 63 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version = 3.8 3 | 4 | show_column_numbers = True 5 | show_error_context = True 6 | 7 | warn_incomplete_stub = True 8 | warn_redundant_casts = True 9 | warn_return_any = True 10 | warn_unreachable = True 11 | warn_unused_ignores = True 12 | 13 | [mypy-clusterman_metrics.*] 14 | ignore_missing_imports = True 15 | 16 | [mypy-logreader.*] 17 | ignore_missing_imports = True 18 | -------------------------------------------------------------------------------- /osx-bdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export BERKELEYDB_DIR=$(brew --prefix berkeley-db) 4 | export YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "homepage": "./", 4 | "dependencies": {}, 5 | "browserslist": { 6 | "production": [ 7 | ">0.2%", 8 | "not dead", 9 | "not op_mini all" 10 | ], 11 | "development": [ 12 | "last 1 chrome version", 13 | "last 1 firefox version", 14 | "last 1 safari version" 15 | ] 16 | }, 17 | "devDependencies": { 18 | "eslint": "^6.6.0", 19 | "eslint-config-airbnb": "^18.2.0", 20 | "eslint-plugin-import": "^2.22.0", 21 | "eslint-plugin-jsx-a11y": "^6.3.1", 22 | "eslint-plugin-react": "^7.20.6", 23 | "eslint-plugin-react-hooks": "^4.1.0" 24 | }, 25 | "resolutions": { 26 | "axe-core": "4.7.0" 27 | }, 28 | "engines": { 29 | "node-version-shim": "10.x", 30 | "node": ">=10" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 120 3 | target_version = ['py38'] 4 | -------------------------------------------------------------------------------- /requirements-dev-minimal.txt: -------------------------------------------------------------------------------- 1 | asynctest 2 | botocore-stubs 3 | debugpy 4 | flake8 5 | mock 6 | mypy 7 | pre-commit 8 | pylint 9 | pytest 10 | pytest-asyncio 11 | requirements-tools 12 | types-pytz 13 | types-PyYAML 14 | types-requests<2.31.0.7 # newer types-requests requires urllib3>=2 15 | types-simplejson 16 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | astroid==2.13.3 2 | asynctest==0.12.0 3 | botocore-stubs==1.38.19 4 | cfgv==2.0.1 5 | debugpy==1.8.1 6 | dill==0.3.6 7 | distlib==0.3.6 8 | filelock==3.4.1 9 | flake8==5.0.4 10 | identify==2.5.5 11 | iniconfig==1.1.1 12 | isort==4.3.18 13 | lazy-object-proxy==1.9.0 14 | mccabe==0.7.0 15 | mypy==1.9.0 16 | mypy-extensions==1.0.0 17 | nodeenv==1.3.3 18 | packaging==19.2 19 | platformdirs==2.5.2 20 | pluggy==0.13.0 21 | pre-commit==2.20.0 22 | py==1.10.0 23 | pycodestyle==2.9.0 24 | pyflakes==2.5.0 25 | pylint==2.15.10 26 | pyparsing==2.4.2 27 | pytest==6.2.2 28 | pytest-asyncio==0.14.0 29 | requirements-tools==1.2.1 30 | toml==0.10.2 31 | tomli==2.0.1 32 | tomlkit==0.11.6 33 | types-awscrt==0.27.2 34 | types-pytz==2024.2.0.20240913 35 | types-PyYAML==6.0.12 36 | types-requests==2.31.0.5 37 | types-simplejson==3.19.0.20240310 38 | types-urllib3==1.26.25.14 39 | virtualenv==20.17.1 40 | -------------------------------------------------------------------------------- /requirements-docs.txt: -------------------------------------------------------------------------------- 1 | Jinja2==3.1.2 2 | markupsafe==2.1.1 3 | mock==3.0.5 4 | Pygments==2.13.0 5 | Sphinx==6.1.3 6 | Sphinx-PyPI-upload==0.2.1 7 | -------------------------------------------------------------------------------- /requirements-minimal.txt: -------------------------------------------------------------------------------- 1 | addict # not sure why check-requirements is not picking this up from task_processing[mesos_executor] 2 | argcomplete 3 | boto3 4 | bsddb3 5 | cryptography 6 | dataclasses 7 | ecdsa>=0.13.3 8 | http-parser # not sure why check-requirements is not picking this up from task_processing[mesos_executor] 9 | humanize 10 | ipdb 11 | ipython 12 | Jinja2>=3.1.2 13 | lockfile 14 | moto 15 | prometheus-client 16 | psutil 17 | py-bcrypt 18 | pyasn1 19 | pyformance 20 | pymesos # not sure why check-requirements is not picking this up from task_processing[mesos_executor] 21 | pysensu-yelp 22 | PyStaticConfiguration 23 | pytimeparse 24 | pytz 25 | PyYAML>=5.1 26 | requests 27 | task_processing[mesos_executor,k8s]>=1.2.0 28 | Twisted>=19.7.0 29 | urllib3>=1.24.2 30 | Werkzeug>=0.15.3 31 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | addict==2.2.1 2 | argcomplete==1.9.5 3 | asttokens==2.2.1 4 | attrs==19.3.0 5 | Automat==20.2.0 6 | aws-sam-translator==1.15.1 7 | aws-xray-sdk==2.4.2 8 | backcall==0.1.0 9 | boto==2.49.0 10 | boto3==1.34.80 11 | botocore==1.34.80 12 | bsddb3==6.2.7 13 | cachetools==4.2.1 14 | certifi==2022.12.7 15 | cffi==1.12.3 16 | cfn-lint==0.24.4 17 | charset-normalizer==2.0.12 18 | constantly==15.1.0 19 | cryptography==39.0.1 20 | dataclasses==0.6 21 | DateTime==4.3 22 | decorator==4.4.0 23 | docker==4.1.0 24 | ecdsa==0.13.3 25 | executing==1.2.0 26 | future==0.18.3 27 | google-auth==1.23.0 28 | http-parser==0.9.0 29 | humanize==0.5.1 30 | hyperlink==19.0.0 31 | idna==2.8 32 | incremental==22.10.0 33 | ipdb==0.13.2 34 | ipython==8.10.0 35 | ipython-genutils==0.2.0 36 | jedi==0.16.0 37 | Jinja2==3.1.2 38 | jmespath==0.9.4 39 | jsondiff==1.1.2 40 | jsonpatch==1.24 41 | jsonpickle==1.2 42 | jsonpointer==2.0 43 | jsonschema==3.2.0 44 | kubernetes==26.1.0 45 | lockfile==0.12.2 46 | MarkupSafe==2.1.1 47 | matplotlib-inline==0.1.3 48 | mock==3.0.5 49 | moto==1.3.13 50 | oauthlib==3.1.0 51 | parso==0.7.0 52 | pexpect==4.7.0 53 | pickleshare==0.7.5 54 | prometheus-client==0.21.1 55 | prompt-toolkit==3.0.38 56 | psutil==5.6.6 57 | ptyprocess==0.6.0 58 | pure-eval==0.2.2 59 | py-bcrypt==0.4 60 | pyasn1==0.4.7 61 | pyasn1-modules==0.2.8 62 | pycparser==2.19 63 | pyformance==0.4 64 | Pygments==2.13.0 65 | pymesos==0.3.9 66 | pyrsistent==0.15.4 67 | pysensu-yelp==0.4.4 68 | PyStaticConfiguration==0.10.4 69 | python-dateutil==2.8.1 70 | python-jose==3.0.1 71 | pytimeparse==1.1.8 72 | pytz==2019.3 73 | PyYAML==6.0.1 74 | requests==2.27.1 75 | requests-oauthlib==1.2.0 76 | responses==0.10.6 77 | rsa==4.9 78 | s3transfer==0.10.1 79 | setuptools==65.5.1 80 | six==1.15.0 81 | sshpubkeys==3.1.0 82 | stack-data==0.6.2 83 | task-processing==1.3.5 84 | traitlets==5.0.0 85 | Twisted==22.10.0 86 | typing-extensions==4.5.0 87 | urllib3==1.25.10 88 | wcwidth==0.1.7 89 | websocket-client==0.56.0 90 | Werkzeug==2.2.3 91 | wrapt==1.11.2 92 | xmltodict==0.12.0 93 | zope.interface==5.1.0 94 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_docs] 2 | source-dir = docs/ 3 | build-dir = docs/_build 4 | all_files = 1 5 | 6 | [upload_docs] 7 | upload_dir = docs/_build/html 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | try: 2 | from setuptools import setup, find_packages 3 | 4 | assert setup 5 | except ImportError: 6 | from distutils.core import setup 7 | 8 | import glob 9 | import tron 10 | 11 | setup( 12 | name="tron", 13 | version=tron.__version__, 14 | provides=["tron"], 15 | author="Yelp", 16 | author_email="yelplabs@yelp.com", 17 | url="http://github.com/Yelp/Tron", 18 | description="Job scheduling and monitoring system", 19 | classifiers=[ 20 | "Programming Language :: Python", 21 | "Programming Language :: Python :: 3.8", 22 | "Operating System :: OS Independent", 23 | "License :: OSI Approved :: Apache Software License", 24 | "Topic :: System :: Monitoring", 25 | "Topic :: System :: Systems Administration", 26 | "Intended Audience :: Developers", 27 | "Intended Audience :: System Administrators", 28 | "Development Status :: 4 - Beta", 29 | ], 30 | packages=find_packages( 31 | exclude=["tests.*", "tests", "example-cluster"], 32 | ) 33 | + ["tronweb"], 34 | scripts=glob.glob("bin/*") + glob.glob("tron/bin/*.py"), 35 | include_package_data=True, 36 | long_description=""" 37 | Tron is a centralized system for managing periodic batch processes across a 38 | cluster. If you find cron or fcron to be insufficient for managing complex work 39 | flows across multiple computers, Tron might be for you. 40 | 41 | For more information, look at the 42 | `tutorial `_ or the 43 | `full documentation `_. 44 | """, 45 | ) 46 | -------------------------------------------------------------------------------- /testfiles/MASTER.yaml: -------------------------------------------------------------------------------- 1 | eventbus_enabled: true 2 | 3 | state_persistence: 4 | name: "/nail/tron/tron_state" 5 | store_type: "shelve" 6 | buffer_size: 10 7 | 8 | ssh_options: 9 | agent: False 10 | identities: 11 | - /work/example-cluster/insecure_key 12 | 13 | action_runner: 14 | runner_type: "subprocess" 15 | remote_status_path: "/tmp/tron" 16 | remote_exec_path: "/work/tron/bin/" 17 | 18 | nodes: 19 | - hostname: localhost 20 | username: root 21 | 22 | time_zone: US/Eastern 23 | 24 | jobs: 25 | one: 26 | node: localhost 27 | schedule: "cron */5 * * * *" 28 | actions: 29 | one: 30 | command: exit 1 31 | retries: 3 32 | retries_delay: 1m 33 | trigger_downstreams: {ymdhm: "{ymdhm}"} 34 | two: 35 | node: localhost 36 | schedule: "cron */5 * * * *" 37 | actions: 38 | two: 39 | command: sleep 10 && date 40 | triggered_by: ["MASTER.one.one.ymdhm.{ymdhm}"] 41 | trigger_timeout: 30s 42 | -------------------------------------------------------------------------------- /testifycompat/__init__.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase # noqa: F401 2 | 3 | from testifycompat.assertions import * # noqa: F401, F403 4 | from testifycompat.fixtures import * # noqa: F401, F403 5 | 6 | 7 | version = "0.1.2" 8 | 9 | 10 | def run(): 11 | raise AssertionError( 12 | "Oops, you tried to use testifycompat.run(). This function doesn't " 13 | "do anything, it only exists as backwards compatibility with testify. " 14 | "You should remove it from your code.", 15 | ) 16 | -------------------------------------------------------------------------------- /testifycompat/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/testifycompat/bin/__init__.py -------------------------------------------------------------------------------- /testifycompat/bin/migrate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | 4 | .. warning:: 5 | 6 | This script is still very experimental. Use at your own risk. It will 7 | be replaced over time with lib2to3 fixers. 8 | 9 | 10 | Usage: 11 | 12 | ``python -m testifycompat.bin.migrate `` 13 | 14 | Example: 15 | 16 | ``find tests -name *.py | xargs python migrate.py`` 17 | 18 | 19 | """ 20 | import functools 21 | import re 22 | import sys 23 | 24 | 25 | def replace(pattern, repl): 26 | return functools.partial(re.sub, pattern, repl) 27 | 28 | 29 | replaces = [ 30 | # Replace imports 31 | replace(r"^from testify import ", "from testifycompat import "), 32 | replace(r"^from testify.assertions import ", "from testifycompat import "), 33 | replace(r"^import testify as T", "import testifycompat as T"), 34 | # Replace test classes 35 | replace( 36 | r"^class (?:Test)?(\w+)(?:Test|TestCase)\((?:T\.)?TestCase\):$", 37 | "class Test\\1(object):", 38 | ), 39 | replace( 40 | r"^class (?:Test)?(\w+)(?:Test|TestCase)(\(\w+TestCase\)):$", 41 | "class Test\\1\\2:", 42 | ), 43 | # Replace some old assertions 44 | replace(r"self.assert_\((.*)\)", "assert \\1"), 45 | ] 46 | 47 | 48 | def run_replacement(contents): 49 | for line in contents: 50 | for replacement in replaces: 51 | line = replacement(line) 52 | yield line 53 | 54 | 55 | def strip_if_main_run(contents): 56 | if len(contents) < 2: 57 | return contents 58 | if "run()" in contents[-1] and "if __name__ == " in contents[-2]: 59 | return contents[:-2] 60 | return contents 61 | 62 | 63 | def run_migration_on_file(filename): 64 | with open(filename) as fh: 65 | lines = fh.read().split("\n") 66 | 67 | lines = list(run_replacement(lines)) 68 | lines = strip_if_main_run(lines) 69 | 70 | with open(filename, "w") as fh: 71 | fh.write("\n".join(lines)) 72 | 73 | 74 | if __name__ == "__main__": 75 | for filename in sys.argv[1:]: 76 | run_migration_on_file(filename) 77 | -------------------------------------------------------------------------------- /testifycompat/fixtures.py: -------------------------------------------------------------------------------- 1 | """ 2 | Compatibility fixtures for migrating code from testify to py.test 3 | 4 | .. note:: 5 | 6 | ``class_`` fixtures must be applied to @classmethods. py.test will not run 7 | a class_* fixture that is not attached to a class-level method, so your 8 | tests will probably fail. 9 | """ 10 | import pytest 11 | 12 | 13 | def setup(func): 14 | return pytest.fixture(autouse=True)(func) 15 | 16 | 17 | def setup_teardown(func): 18 | return pytest.yield_fixture(autouse=True)(func) 19 | 20 | 21 | def teardown(func): 22 | def teardown_(*args, **kwargs): 23 | yield 24 | func(*args, **kwargs) 25 | 26 | return pytest.yield_fixture(autouse=True)(teardown_) 27 | 28 | 29 | def class_setup(func): 30 | return pytest.fixture(autouse=True, scope="class")(func) 31 | 32 | 33 | def class_setup_teardown(func): 34 | return pytest.yield_fixture(autouse=True, scope="class")(func) 35 | 36 | 37 | def class_teardown(func): 38 | def teardown_(*args, **kwargs): 39 | yield 40 | func(*args, **kwargs) 41 | 42 | return pytest.yield_fixture(autouse=True, scope="class")(teardown_) 43 | 44 | 45 | def suite(name, reason=None): 46 | """Translate a :func:`testify.suite` decorator into the appropriate 47 | :mod:`pytest.mark` call. For the disabled suite this results in a 48 | skipped test. For other suites it will return a `pytest.mark.` 49 | decorator. 50 | """ 51 | if name == "disabled": 52 | return pytest.mark.skipif(True, reason=reason) 53 | 54 | return getattr(pytest.mark, name) 55 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | from twisted.python import log 2 | 3 | observer = log.PythonLoggingObserver() 4 | observer.start() 5 | -------------------------------------------------------------------------------- /tests/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tests/api/__init__.py -------------------------------------------------------------------------------- /tests/api/requestargs_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from unittest.mock import MagicMock 3 | 4 | from testifycompat import assert_equal 5 | from testifycompat import run 6 | from testifycompat import setup 7 | from testifycompat import TestCase 8 | from tron.api.requestargs import get_bool 9 | from tron.api.requestargs import get_datetime 10 | from tron.api.requestargs import get_integer 11 | from tron.api.requestargs import get_string 12 | 13 | 14 | class TestRequestArgs(TestCase): 15 | @setup 16 | def setup_args(self): 17 | self.args = { 18 | b"number": [b"123"], 19 | b"string": [b"astring"], 20 | b"boolean": [b"1"], 21 | b"datetime": [b"2012-03-14 15:09:26"], 22 | } 23 | self.datetime = datetime.datetime(2012, 3, 14, 15, 9, 26) 24 | self.request = MagicMock(args=self.args) 25 | 26 | def _add_arg(self, name, value): 27 | name = name.encode() 28 | value = value.encode() 29 | if name not in self.args: 30 | self.args[name] = [] 31 | self.args[name].append(value) 32 | 33 | def test_get_integer_valid_int(self): 34 | self._add_arg("number", "5") 35 | assert_equal(get_integer(self.request, "number"), 123) 36 | 37 | def test_get_integer_invalid_int(self): 38 | self._add_arg("nan", "beez") 39 | assert not get_integer(self.request, "nan") 40 | 41 | def test_get_integer_missing(self): 42 | assert not get_integer(self.request, "missing") 43 | 44 | def test_get_string(self): 45 | self._add_arg("string", "bogus") 46 | assert_equal(get_string(self.request, "string"), "astring") 47 | 48 | def test_get_string_missing(self): 49 | assert not get_string(self.request, "missing") 50 | 51 | def test_get_bool(self): 52 | assert get_bool(self.request, "boolean") 53 | 54 | def test_get_bool_false(self): 55 | self._add_arg("false", "0") 56 | assert not get_bool(self.request, "false") 57 | 58 | def test_get_bool_missing(self): 59 | assert not get_bool(self.request, "missing") 60 | 61 | def test_get_datetime_valid(self): 62 | assert_equal(get_datetime(self.request, "datetime"), self.datetime) 63 | 64 | def test_get_datetime_invalid(self): 65 | self._add_arg("nope", "2012-333-4") 66 | assert not get_datetime(self.request, "nope") 67 | 68 | def test_get_datetime_missing(self): 69 | assert not get_datetime(self.request, "missing") 70 | 71 | 72 | if __name__ == "__main__": 73 | run() 74 | -------------------------------------------------------------------------------- /tests/assertions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Assertions for testify. 3 | """ 4 | from testifycompat import assert_in 5 | from testifycompat import assert_not_reached 6 | 7 | 8 | def assert_raises(expected_exception_class, callable_obj, *args, **kwargs): 9 | """Returns the exception if the callable raises expected_exception_class""" 10 | try: 11 | callable_obj(*args, **kwargs) 12 | except expected_exception_class as e: 13 | # we got the expected exception 14 | return e 15 | assert_not_reached( 16 | "No exception was raised (expected %s)" % expected_exception_class, 17 | ) 18 | 19 | 20 | def assert_length(sequence, expected, msg=None): 21 | """Assert that a sequence or iterable has an expected length.""" 22 | msg = msg or "%(sequence)s has length %(length)s expected %(expected)s" 23 | length = len(list(sequence)) 24 | assert length == expected, msg % locals() 25 | 26 | 27 | def assert_call(mock, call_idx, *args, **kwargs): 28 | """Assert that a function was called on mock with the correct args.""" 29 | actual = mock.mock_calls[call_idx] if mock.mock_calls else None 30 | msg = f"Call {call_idx} expected {(args, kwargs)}, was {actual}" 31 | assert actual == (args, kwargs), msg 32 | 33 | 34 | def assert_mock_calls(expected, mock_calls): 35 | """Assert that all expected calls are in the list of mock_calls.""" 36 | for expected_call in expected: 37 | assert_in(expected_call, mock_calls) 38 | -------------------------------------------------------------------------------- /tests/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tests/bin/__init__.py -------------------------------------------------------------------------------- /tests/bin/action_status_test.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import tempfile 3 | from unittest import mock 4 | 5 | from testifycompat import setup_teardown 6 | from testifycompat import TestCase 7 | from tron import yaml 8 | from tron.bin import action_status 9 | 10 | 11 | class TestActionStatus(TestCase): 12 | @setup_teardown 13 | def setup_status_file(self): 14 | self.status_file = tempfile.NamedTemporaryFile(mode="r+") 15 | self.status_content = { 16 | "pid": 1234, 17 | "return_code": None, 18 | "run_id": "MASTER.foo.bar.1234", 19 | } 20 | self.status_file.write(yaml.safe_dump(self.status_content)) 21 | self.status_file.flush() 22 | self.status_file.seek(0) 23 | yield 24 | self.status_file.close() 25 | 26 | @mock.patch("tron.bin.action_status.os.killpg", autospec=True) 27 | @mock.patch( 28 | "tron.bin.action_status.os.getpgid", 29 | autospec=True, 30 | return_value=42, 31 | ) 32 | def test_send_signal(self, mock_getpgid, mock_kill): 33 | action_status.send_signal(signal.SIGKILL, self.status_file) 34 | mock_getpgid.assert_called_with(self.status_content["pid"]) 35 | mock_kill.assert_called_with(42, signal.SIGKILL) 36 | 37 | def test_get_field_retrieves_last_entry(self): 38 | self.status_file.seek(0, 2) 39 | additional_status_content = { 40 | "pid": 1234, 41 | "return_code": 0, 42 | "run_id": "MASTER.foo.bar.1234", 43 | "command": "echo " + "really_long" * 100, 44 | } 45 | self.status_file.write( 46 | yaml.safe_dump(additional_status_content, explicit_start=True), 47 | ) 48 | self.status_file.flush() 49 | self.status_file.seek(0) 50 | assert action_status.get_field("return_code", self.status_file) == 0 51 | 52 | def test_get_field_none(self): 53 | assert action_status.get_field("return_code", self.status_file) is None 54 | -------------------------------------------------------------------------------- /tests/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tests/commands/__init__.py -------------------------------------------------------------------------------- /tests/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tests/config/__init__.py -------------------------------------------------------------------------------- /tests/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tests/core/__init__.py -------------------------------------------------------------------------------- /tests/data/logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root, twisted, tron 3 | 4 | [handlers] 5 | keys=fileHandler 6 | 7 | [formatters] 8 | keys=defaultFormatter 9 | 10 | [logger_root] 11 | level=WARN 12 | handlers=fileHandler 13 | 14 | [logger_twisted] 15 | level=WARN 16 | handlers=fileHandler 17 | qualname=twisted 18 | propagate=0 19 | 20 | [logger_tron] 21 | level=WARN 22 | handlers=fileHandler 23 | qualname=tron 24 | propagate=0 25 | 26 | [handler_fileHandler] 27 | class=logging.FileHandler 28 | level=WARN 29 | formatter=defaultFormatter 30 | args=('{0}',) 31 | 32 | [formatter_defaultFormatter] 33 | format=%(asctime)s %(name)s %(levelname)s %(message)s 34 | -------------------------------------------------------------------------------- /tests/serialize/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tests/serialize/__init__.py -------------------------------------------------------------------------------- /tests/serialize/runstate/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tests/serialize/runstate/__init__.py -------------------------------------------------------------------------------- /tests/serialize/runstate/shelvestore_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import tempfile 4 | 5 | from testifycompat import assert_equal 6 | from testifycompat import run 7 | from testifycompat import setup 8 | from testifycompat import teardown 9 | from testifycompat import TestCase 10 | from tron.serialize.runstate.shelvestore import Py2Shelf 11 | from tron.serialize.runstate.shelvestore import ShelveKey 12 | from tron.serialize.runstate.shelvestore import ShelveStateStore 13 | 14 | 15 | class TestShelveStateStore(TestCase): 16 | @setup 17 | def setup_store(self): 18 | self.tmpdir = tempfile.mkdtemp() 19 | self.filename = os.path.join(self.tmpdir, "state") 20 | self.store = ShelveStateStore(self.filename) 21 | 22 | @teardown 23 | def teardown_store(self): 24 | shutil.rmtree(self.tmpdir) 25 | 26 | def test__init__(self): 27 | assert_equal(self.filename, self.store.filename) 28 | 29 | def test_save(self): 30 | key_value_pairs = [ 31 | ( 32 | ShelveKey("one", "two"), 33 | { 34 | "this": "data", 35 | }, 36 | ), 37 | ( 38 | ShelveKey("three", "four"), 39 | { 40 | "this": "data2", 41 | }, 42 | ), 43 | ] 44 | self.store.save(key_value_pairs) 45 | self.store.cleanup() 46 | 47 | stored_data = Py2Shelf(self.filename) 48 | for key, value in key_value_pairs: 49 | assert_equal(stored_data[str(key.key)], value) 50 | stored_data.close() 51 | 52 | def test_delete(self): 53 | key_value_pairs = [ 54 | ( 55 | ShelveKey("one", "two"), 56 | { 57 | "this": "data", 58 | }, 59 | ), 60 | ( 61 | ShelveKey("three", "four"), 62 | { 63 | "this": "data2", 64 | }, 65 | ), 66 | # Delete first key 67 | ( 68 | ShelveKey("one", "two"), 69 | None, 70 | ), 71 | ] 72 | self.store.save(key_value_pairs) 73 | self.store.cleanup() 74 | 75 | stored_data = Py2Shelf(self.filename) 76 | assert stored_data == { 77 | str(ShelveKey("three", "four").key): {"this": "data2"}, 78 | } 79 | stored_data.close() 80 | 81 | def test_restore(self): 82 | self.store.cleanup() 83 | keys = [ShelveKey("thing", i) for i in range(5)] 84 | value = {"this": "data"} 85 | store = Py2Shelf(self.filename) 86 | for key in keys: 87 | store[str(key.key)] = value 88 | store.close() 89 | 90 | self.store.shelve = Py2Shelf(self.filename) 91 | retrieved_data = self.store.restore(keys) 92 | for key in keys: 93 | assert_equal(retrieved_data[key], value) 94 | 95 | 96 | if __name__ == "__main__": 97 | run() 98 | -------------------------------------------------------------------------------- /tests/test_id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAz+pjFOAHRLcQc6X51SyysJurwBTben3OWCG46CB81faxTVrC 3 | gcVEM3HCHz5MU8jI0Wb+DK9AXU229yQ8OCRPt3CrzxyI031ZrzuagVNRb/hmiBN+ 4 | +SNmIFQl97/bqht5DXKykUJQmP31crLz3+G0rGXONeJjZAqFFIA1NfMMAnRZMRMo 5 | 417Xf/p3yYXCV0AbWqbMWYA8aQKWFs4EY36vOzUJyftJI1cGttvskcCd3dce4DpR 6 | 8JEnI/rkRKOR19eCLCtftV6CmC3igLoSF1JDeQgkVZRdVG/pcwfV1a/SEAlxAO0e 7 | RcZccMVoPi/Ans2CxIeukQ6ThFyRMcaD+iku6QIBIwKCAQBw3lMLVQtCj0Nx+wP3 8 | YWhRPpBvlktCfs8Z5muxNjUjsc32yt6eN+Mx3qs1iDgQOcwZ53P4QeEctSjPTi9R 9 | rU/YnEACt7f9x7RXz+Yo8rcuJ8KhpC78Rmqj1ei5sUtcWA6DpKoUVzMROWf8b8Y3 10 | tQpO9W/xXaOrVir8gBzixcSw3xgBiobkxWeiwCrD8LsPQ7qrP27nf252wrIa8BJx 11 | 7qFrRSa05I6Hinj3jMwXUiUPVXBG5m+t8Z4c9WITvnr91uwHuHZc9TV6HedqznmA 12 | 0n6iq2rPHPj4D0gWZFKvTFXN06KD+ldAiyWWGg3nob7gzFkGZkbAk2KLUN7PLuv3 13 | 0KRPAoGBAPg8DkyznKNVgenaKt5Jc9YdLCVd4uvvjuTgmBiezvE4iwVRkIm6Q0jq 14 | FLgN4c4fUd10Fa4QPwIFXQBK5UYTwoklvDttZ+LnwY5SEzGeaXHdi8suOPlcDvDp 15 | L5Fpbi6wGJk04MXcApIwLIQCQp+pJeIHbKoq7c2ABMuqp+QPbUwrAoGBANZrcXfo 16 | p2fIxbhrdwS7yMT0jInX6HeDvDM6e67X3TZYAjZuj9PNtlg6qOT6mFWUzOvcEFzj 17 | RDfA2d53DpGfpALdQ8OBxyI7QrdcbeW0Br/pGCicW+ovbT3Z3Bfl+GOhav1EuHRR 18 | L0GlW2xN3h+oZRexUSPAA80ep9o/XGaAjGM7AoGBAOoMvQZ9dm4da9x9PlyOZeci 19 | 0doWsWIcYihBeXZMlzs1T+BxeaZtymlRu8N6zZZ1TS/ietdRJXbvHSwpW9RbxgxI 20 | JoEso8dPipwhf8+yne8EFhdXd4wGVzrqfU6WmxYTv2vhZjblYYKFMUlD9ax66TQy 21 | 4sxUXI6OpW+SRoaSM9oZAoGBAKVo0+AogSQtKtAYYyDoojGJc7rLIQu9ZUwXLDZs 22 | AmvAPDiebvPZNOT6DULtM6+77ooQKeFBmwZwMwqznYZH840uWNil8WOMzREbariD 23 | kC2lL+TQZCmvjslQSrNZolt8hbwQcQlF8UFFDAMXf3eB55XvL/cB1wvzE8Wel7zJ 24 | kN7VAoGBAIISjKvjU3VpKNX9CEQ7V6eb7OuUKo3gUTCMaunWwpVYZ4pR5UiOtib+ 25 | lcDpTQybKQOqSgHKQ13L/7Nu0GY9ILlXhfNhRlflSNUNaGcwMykxm0tzh3TsQ3vv 26 | 8cQYV+W+24bwK39tSzl+n/nnuDdly7aTS2nDrhwWIsL45DU1QXK8 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /tests/test_id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAz+pjFOAHRLcQc6X51SyysJurwBTben3OWCG46CB81faxTVrCgcVEM3HCHz5MU8jI0Wb+DK9AXU229yQ8OCRPt3CrzxyI031ZrzuagVNRb/hmiBN++SNmIFQl97/bqht5DXKykUJQmP31crLz3+G0rGXONeJjZAqFFIA1NfMMAnRZMRMo417Xf/p3yYXCV0AbWqbMWYA8aQKWFs4EY36vOzUJyftJI1cGttvskcCd3dce4DpR8JEnI/rkRKOR19eCLCtftV6CmC3igLoSF1JDeQgkVZRdVG/pcwfV1a/SEAlxAO0eRcZccMVoPi/Ans2CxIeukQ6ThFyRMcaD+iku6Q== rhettg@devvm1 2 | -------------------------------------------------------------------------------- /tests/testingutils.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import logging 3 | import time 4 | from unittest import mock 5 | 6 | from testifycompat import class_setup 7 | from testifycompat import class_teardown 8 | from testifycompat import setup 9 | from testifycompat import teardown 10 | from testifycompat import TestCase 11 | from tron.utils import timeutils 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | # TODO: remove when replaced with tron.eventloop 16 | 17 | 18 | class MockReactorTestCase(TestCase): 19 | """Patch the reactor to a MockReactor.""" 20 | 21 | # Override this in subclasses 22 | module_to_mock = None 23 | 24 | @class_setup 25 | def class_setup_patched_reactor(self): 26 | msg = "%s must set a module_to_mock field" % self.__class__ 27 | assert self.module_to_mock, msg 28 | self.old_reactor = getattr(self.module_to_mock, "reactor") 29 | 30 | @class_teardown 31 | def teardown_patched_reactor(self): 32 | setattr(self.module_to_mock, "reactor", self.old_reactor) 33 | 34 | @setup 35 | def setup_mock_reactor(self): 36 | self.reactor = mock.MagicMock() 37 | setattr(self.module_to_mock, "reactor", self.reactor) 38 | 39 | 40 | # TODO: remove 41 | class MockTimeTestCase(TestCase): 42 | 43 | now = None 44 | 45 | @setup 46 | def setup_current_time(self): 47 | assert self.now, "%s must set a now field" % self.__class__ 48 | self.old_current_time = timeutils.current_time 49 | timeutils.current_time = lambda tz=None: self.now 50 | 51 | @teardown 52 | def teardown_current_time(self): 53 | timeutils.current_time = self.old_current_time 54 | # Reset 'now' back to what was set on the class because some test may 55 | # have changed it 56 | self.now = self.__class__.now 57 | 58 | 59 | def retry(max_tries=3, delay=0.1, exceptions=(KeyError, IndexError)): 60 | """A function decorator for re-trying an operation. Useful for MongoDB 61 | which is only eventually consistent. 62 | """ 63 | 64 | def wrapper(f): 65 | @functools.wraps(f) 66 | def wrap(*args, **kwargs): 67 | for _ in range(max_tries): 68 | try: 69 | return f(*args, **kwargs) 70 | except exceptions: 71 | time.sleep(delay) 72 | raise 73 | 74 | return wrap 75 | 76 | return wrapper 77 | 78 | 79 | def autospec_method(method, *args, **kwargs): 80 | """create an autospec for an instance method.""" 81 | mocked_method = mock.create_autospec(method, *args, **kwargs) 82 | setattr(method.__self__, method.__name__, mocked_method) 83 | -------------------------------------------------------------------------------- /tests/trondaemon_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import time 4 | from unittest import mock 5 | 6 | from testifycompat import setup 7 | from testifycompat import teardown 8 | from testifycompat import TestCase 9 | from tron.trondaemon import TronDaemon 10 | 11 | 12 | class TronDaemonTestCase(TestCase): 13 | @setup 14 | @mock.patch("tron.trondaemon.setup_logging", mock.Mock(), autospec=None) 15 | @mock.patch("signal.signal", mock.Mock(), autospec=None) 16 | def setup(self): 17 | self.tmpdir = tempfile.TemporaryDirectory() 18 | trond_opts = mock.Mock() 19 | trond_opts.working_dir = self.tmpdir.name 20 | trond_opts.lock_file = os.path.join(self.tmpdir.name, "lockfile") 21 | self.trond = TronDaemon(trond_opts) 22 | 23 | @teardown 24 | def teardown(self): 25 | self.tmpdir.cleanup() 26 | 27 | @mock.patch("tron.trondaemon.setup_logging", mock.Mock(), autospec=None) 28 | @mock.patch("signal.signal", mock.Mock(), autospec=None) 29 | def test_init(self): 30 | daemon = TronDaemon.__new__(TronDaemon) # skip __init__ 31 | options = mock.Mock() 32 | 33 | with mock.patch( 34 | "tron.utils.flock", 35 | autospec=True, 36 | ) as mock_flock: 37 | daemon.__init__(options) 38 | assert mock_flock.call_count == 0 39 | 40 | def test_run_uses_context(self): 41 | with mock.patch("tron.trondaemon.setup_logging", mock.Mock(), autospec=None,), mock.patch( 42 | "tron.trondaemon.no_daemon_context", 43 | mock.Mock(), 44 | autospec=None, 45 | ) as ndc: 46 | ndc.return_value = mock.MagicMock() 47 | boot_time = time.time() 48 | ndc.return_value.__enter__.side_effect = RuntimeError() 49 | daemon = TronDaemon(mock.Mock()) 50 | try: 51 | daemon.run(boot_time) 52 | except Exception: 53 | pass 54 | assert ndc.call_count == 1 55 | 56 | def test_run_manhole_new_manhole(self): 57 | with open(self.trond.manhole_sock, "w+"): 58 | pass 59 | 60 | with mock.patch( 61 | "twisted.internet.reactor.listenUNIX", 62 | autospec=True, 63 | ) as mock_listenUNIX: 64 | self.trond._run_manhole() 65 | 66 | assert mock_listenUNIX.call_count == 1 67 | # _run_manhole will remove the old manhole.sock but not recreate 68 | # it because we mocked out listenUNIX 69 | assert not os.path.exists(self.trond.manhole_sock) 70 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tests/utils/__init__.py -------------------------------------------------------------------------------- /tests/utils/collections_test.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | from testifycompat import assert_equal 4 | from testifycompat import assert_in 5 | from testifycompat import assert_not_in 6 | from testifycompat import assert_raises 7 | from testifycompat import setup 8 | from testifycompat import TestCase 9 | from tests.assertions import assert_mock_calls 10 | from tests.testingutils import autospec_method 11 | from tron.utils import collections 12 | 13 | 14 | class TestMappingCollections(TestCase): 15 | @setup 16 | def setup_collection(self): 17 | self.name = "some_name" 18 | self.collection = collections.MappingCollection(self.name) 19 | 20 | def test_filter_by_name(self): 21 | autospec_method(self.collection.remove) 22 | self.collection.update(dict.fromkeys(["c", "d", "e"])) 23 | self.collection.filter_by_name(["a", "c"]) 24 | expected = [mock.call(name) for name in ["d", "e"]] 25 | assert_mock_calls(expected, self.collection.remove.mock_calls) 26 | 27 | def test_remove_missing(self): 28 | assert_raises(ValueError, self.collection.remove, "name") 29 | 30 | def test_remove(self): 31 | name = "the_name" 32 | self.collection[name] = item = mock.Mock() 33 | self.collection.remove(name) 34 | assert_not_in(name, self.collection) 35 | item.disable.assert_called_with() 36 | 37 | def test_contains_item_false(self): 38 | mock_item, mock_func = mock.Mock(), mock.Mock() 39 | assert not self.collection.contains_item(mock_item, mock_func) 40 | assert not mock_func.mock_calls 41 | 42 | def test_contains_item_not_equal(self): 43 | mock_item, mock_func = mock.Mock(), mock.Mock() 44 | self.collection[mock_item.get_name()] = "other item" 45 | result = self.collection.contains_item(mock_item, mock_func) 46 | assert_equal(result, mock_func.return_value) 47 | mock_func.assert_called_with(mock_item) 48 | 49 | def test_contains_item_true(self): 50 | mock_item, mock_func = mock.Mock(), mock.Mock() 51 | self.collection[mock_item.get_name()] = mock_item 52 | assert self.collection.contains_item(mock_item, mock_func) 53 | 54 | def test_add_contains(self): 55 | autospec_method(self.collection.contains_item) 56 | item, update_func = mock.Mock(), mock.Mock() 57 | assert not self.collection.add(item, update_func) 58 | assert_not_in(item.get_name(), self.collection) 59 | 60 | def test_add_new(self): 61 | autospec_method(self.collection.contains_item, return_value=False) 62 | item, update_func = mock.Mock(), mock.Mock() 63 | assert self.collection.add(item, update_func) 64 | assert_in(item.get_name(), self.collection) 65 | 66 | def test_replace(self): 67 | autospec_method(self.collection.add) 68 | item = mock.Mock() 69 | self.collection.replace(item) 70 | self.collection.add.assert_called_with( 71 | item, 72 | self.collection.remove_item, 73 | ) 74 | -------------------------------------------------------------------------------- /tests/utils/proxy_test.py: -------------------------------------------------------------------------------- 1 | from testifycompat import assert_equal 2 | from testifycompat import assert_in 3 | from testifycompat import assert_raises 4 | from testifycompat import run 5 | from testifycompat import setup 6 | from testifycompat import TestCase 7 | from tron.utils.proxy import AttributeProxy 8 | from tron.utils.proxy import CollectionProxy 9 | 10 | 11 | class DummyTarget: 12 | def __init__(self, v): 13 | self.v = v 14 | 15 | def foo(self): 16 | return self.v 17 | 18 | @property 19 | def not_foo(self): 20 | return not self.v 21 | 22 | def equals(self, b, sometimes=False): 23 | if sometimes: 24 | return "sometimes" 25 | return self.v == b 26 | 27 | 28 | class DummyObject: 29 | def __init__(self, proxy): 30 | self.proxy = proxy 31 | 32 | def __getattr__(self, item): 33 | return self.proxy.perform(item) 34 | 35 | 36 | class TestCollectionProxy(TestCase): 37 | @setup 38 | def setup_proxy(self): 39 | self.target_list = [DummyTarget(1), DummyTarget(2), DummyTarget(0)] 40 | self.proxy = CollectionProxy( 41 | lambda: self.target_list, 42 | [ 43 | ("foo", any, True), 44 | ("not_foo", all, False), 45 | ("equals", lambda a: list(a), True), 46 | ], 47 | ) 48 | self.dummy = DummyObject(self.proxy) 49 | 50 | def test_add(self): 51 | self.proxy.add("foo", any, True) 52 | assert_equal(self.proxy._defs["foo"], (any, True)) 53 | 54 | def test_perform(self): 55 | assert self.dummy.foo() 56 | assert not self.dummy.not_foo 57 | 58 | def test_perform_not_defined(self): 59 | assert_raises(AttributeError, self.dummy.proxy.perform, "bar") 60 | 61 | def test_perform_with_params(self): 62 | assert_equal(self.proxy.perform("equals")(2), [False, True, False]) 63 | sometimes = ["sometimes"] * 3 64 | assert_equal( 65 | self.proxy.perform("equals")(3, sometimes=True), 66 | sometimes, 67 | ) 68 | 69 | 70 | class TestAttributeProxy(TestCase): 71 | @setup 72 | def setup_proxy(self): 73 | self.target = DummyTarget(1) 74 | self.proxy = AttributeProxy(self.target, ["foo", "not_foo"]) 75 | self.dummy = DummyObject(self.proxy) 76 | 77 | def test_add(self): 78 | self.proxy.add("bar") 79 | assert_in("bar", self.proxy._attributes) 80 | 81 | def test_perform(self): 82 | assert_equal(self.dummy.foo(), 1) 83 | assert_equal(self.dummy.not_foo, False) 84 | 85 | def test_perform_not_defined(self): 86 | assert_raises(AttributeError, self.dummy.proxy.perform, "zzz") 87 | 88 | 89 | if __name__ == "__main__": 90 | run() 91 | -------------------------------------------------------------------------------- /tests/utils/state_test.py: -------------------------------------------------------------------------------- 1 | from testifycompat import assert_equal 2 | from testifycompat import setup 3 | from testifycompat import TestCase 4 | from tron.utils import state 5 | 6 | 7 | class TestStateMachineSimple(TestCase): 8 | @setup 9 | def build_machine(self): 10 | self.state_green = "green" 11 | self.state_red = "red" 12 | 13 | self.machine = state.Machine(self.state_red, red=dict(true="green")) 14 | 15 | def test_transition_many(self): 16 | # Stay the same 17 | assert not self.machine.transition("missing") 18 | assert_equal(self.machine.state, self.state_red) 19 | 20 | # Traffic has arrived 21 | self.machine.transition("true") 22 | assert_equal(self.machine.state, self.state_green) 23 | 24 | # Still traffic 25 | self.machine.transition("true") 26 | assert_equal(self.machine.state, self.state_green) 27 | 28 | def test_check(self): 29 | assert not self.machine.check(False) 30 | assert_equal(self.machine.check("true"), self.state_green) 31 | assert_equal(self.machine.state, self.state_red) 32 | 33 | 34 | class TestStateMachineMultiOption(TestCase): 35 | @setup 36 | def build_machine(self): 37 | # Generalized rules of a conversation 38 | # If they are talking, we should listen 39 | # If they are listening, we should talk 40 | # If they are ignoring us we should get angry 41 | self.machine = state.Machine( 42 | "listening", 43 | listening=dict(listening="talking"), 44 | talking=dict(ignoring="angry", talking="listening"), 45 | ) 46 | 47 | def test_transition_many(self): 48 | # Talking, we should listen 49 | self.machine.transition("talking") 50 | assert_equal(self.machine.state, "listening") 51 | 52 | # Now be polite 53 | self.machine.transition("listening") 54 | assert_equal(self.machine.state, "talking") 55 | 56 | self.machine.transition("listening") 57 | assert_equal(self.machine.state, "talking") 58 | 59 | # But they are tired of us... 60 | self.machine.transition("ignoring") 61 | assert_equal(self.machine.state, "angry") 62 | 63 | def test_transition_set(self): 64 | expected = {"listening", "talking", "ignoring"} 65 | assert_equal(set(self.machine.transition_names), expected) 66 | -------------------------------------------------------------------------------- /tools/action_dag_diagram.py: -------------------------------------------------------------------------------- 1 | """ 2 | Create a graphviz diagram from a Tron Job configuration. 3 | 4 | Usage: 5 | python tools/action_dag_diagram.py -c -n 6 | 7 | This will create a file named .dot 8 | You can create a diagram using: 9 | dot -Tpng -o .png .dot 10 | """ 11 | import optparse 12 | 13 | from tron.config import manager 14 | from tron.config import schema 15 | 16 | 17 | def parse_args(): 18 | parser = optparse.OptionParser() 19 | parser.add_option("-c", "--config", help="Tron configuration path.") 20 | parser.add_option( 21 | "-n", 22 | "--name", 23 | help="Job name to graph. Also used as output filename.", 24 | ) 25 | parser.add_option( 26 | "--namespace", 27 | default=schema.MASTER_NAMESPACE, 28 | help="Configuration namespace which contains the job.", 29 | ) 30 | opts, _ = parser.parse_args() 31 | 32 | if not opts.config: 33 | parser.error("A config filename is required.") 34 | if not opts.name: 35 | parser.error("A Job name is required.") 36 | return opts 37 | 38 | 39 | def build_diagram(job_config): 40 | edges, nodes = [], [] 41 | 42 | for action in job_config.actions.values(): 43 | shape = "invhouse" if not action.requires else "rect" 44 | nodes.append(f"node [shape = {shape}]; {action.name}") 45 | for required_action in action.requires: 46 | edges.append(f"{required_action} -> {action.name}") 47 | 48 | return "digraph g{{{}\n{}}}".format("\n".join(nodes), "\n".join(edges)) 49 | 50 | 51 | def get_job(config_container, namespace, job_name): 52 | if namespace not in config_container: 53 | raise ValueError("Unknown namespace: %s" % namespace) 54 | 55 | config = config_container[opts.namespace] 56 | if job_name not in config.jobs: 57 | raise ValueError("Could not find Job %s" % job_name) 58 | 59 | return config.jobs[job_name] 60 | 61 | 62 | if __name__ == "__main__": 63 | opts = parse_args() 64 | 65 | config_manager = manager.ConfigManager(opts.config) 66 | container = config_manager.load() 67 | job_config = get_job(container, opts.namespace, opts.name) 68 | graph = build_diagram(job_config) 69 | 70 | with open("%s.dot" % opts.name, "w") as fh: 71 | fh.write(graph) 72 | -------------------------------------------------------------------------------- /tools/inspect_serialized_state.py: -------------------------------------------------------------------------------- 1 | """Read a state file or db and create a report which summarizes it's contents. 2 | 3 | Displays: 4 | State configuration 5 | Count of jobs 6 | 7 | Table of Jobs with start date of last run 8 | 9 | """ 10 | import optparse 11 | 12 | from tron.config import manager 13 | from tron.serialize.runstate import statemanager 14 | from tron.utils import chdir 15 | 16 | 17 | def parse_options(): 18 | parser = optparse.OptionParser() 19 | parser.add_option("-c", "--config-path", help="Path to the configuration.") 20 | parser.add_option( 21 | "-w", 22 | "--working-dir", 23 | default=".", 24 | help="Working directory to resolve relative paths.", 25 | ) 26 | opts, _ = parser.parse_args() 27 | 28 | if not opts.config_path: 29 | parser.error("A --config-path is required.") 30 | return opts 31 | 32 | 33 | def get_container(config_path): 34 | config_manager = manager.ConfigManager(config_path) 35 | return config_manager.load() 36 | 37 | 38 | def get_state(container): 39 | config = container.get_master().state_persistence 40 | state_manager = statemanager.PersistenceManagerFactory.from_config(config) 41 | names = container.get_job_names() 42 | return state_manager.restore(*names) 43 | 44 | 45 | def format_date(date_string): 46 | return date_string.strftime("%Y-%m-%d %H:%M:%S") if date_string else None 47 | 48 | 49 | def format_jobs(job_states): 50 | format = "%-30s %-8s %-5s %s\n" 51 | header = format % ("Name", "Enabled", "Runs", "Last Update") 52 | 53 | def max_run(item): 54 | start_time = filter(None, (run["start_time"] for run in item)) 55 | return max(start_time) if start_time else None 56 | 57 | def build(name, job): 58 | start_times = (max_run(job_run["runs"]) for job_run in job["runs"]) 59 | start_times = filter(None, start_times) 60 | last_run = format_date(max(start_times)) if start_times else None 61 | return format % (name, job["enabled"], len(job["runs"]), last_run) 62 | 63 | seq = sorted(build(*item) for item in job_states.items()) 64 | return header + "".join(seq) 65 | 66 | 67 | def display_report(state_config, job_states): 68 | print("State Config: %s" % str(state_config)) 69 | print("Total Jobs: %s" % len(job_states)) 70 | 71 | print("\n%s" % format_jobs(job_states)) 72 | 73 | 74 | def main(config_path, working_dir): 75 | container = get_container(config_path) 76 | config = container.get_master().state_persistence 77 | with chdir(working_dir): 78 | display_report(config, *get_state(container)) 79 | 80 | 81 | if __name__ == "__main__": 82 | opts = parse_options() 83 | main(opts.config_path, opts.working_dir) 84 | -------------------------------------------------------------------------------- /tools/migration/migrate_config_0.5.1_to_0.5.2.py: -------------------------------------------------------------------------------- 1 | """Migrate a single configuration file (tron 0.5.1) to the new 0.5.2 2 | multi-file format. 3 | 4 | Usage: 5 | 6 | python tools/migration/migrate_config_0.5.1_to_0.5.2.py \ 7 | --source old_config_filename \ 8 | --dest new_config_dir 9 | """ 10 | import optparse 11 | import os 12 | 13 | from tron.config import manager 14 | 15 | 16 | def parse_options(): 17 | parser = optparse.OptionParser() 18 | parser.add_option("-s", "--source", help="Path to old configuration file.") 19 | parser.add_option( 20 | "-d", 21 | "--dest", 22 | help="Path to new configuration directory.", 23 | ) 24 | opts, _ = parser.parse_args() 25 | 26 | if not opts.source: 27 | parser.error("--source is required") 28 | if not opts.dest: 29 | parser.error("--dest is required") 30 | return opts 31 | 32 | 33 | def main(source, dest): 34 | dest = os.path.abspath(dest) 35 | if not os.path.isfile(source): 36 | raise SystemExit("Error: Source (%s) is not a file" % source) 37 | if os.path.exists(dest): 38 | raise SystemExit("Error: Destination path (%s) already exists" % dest) 39 | old_config = manager.read_raw(source) 40 | manager.create_new_config(dest, old_config) 41 | 42 | 43 | if __name__ == "__main__": 44 | opts = parse_options() 45 | main(opts.source, opts.dest) 46 | -------------------------------------------------------------------------------- /tools/migration/migrate_state_1.3.15_to_1.4.0.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | 4 | from tron.config import manager 5 | from tron.serialize import runstate 6 | from tron.serialize.runstate.statemanager import PersistenceManagerFactory 7 | from tron.utils import chdir 8 | 9 | 10 | def parse_args(): 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument( 13 | "--back", 14 | help="Flag to migrate back from new state back to old state", 15 | action="store_true", 16 | default=False, 17 | ) 18 | parser.add_argument( 19 | "--working-dir", 20 | help="Working directory for the Tron daemon", 21 | required=True, 22 | ) 23 | parser.add_argument( 24 | "--config-path", 25 | help="Path in working dir with configs", 26 | required=True, 27 | ) 28 | return parser.parse_args() 29 | 30 | 31 | def create_job_runs_for_job(state_manager, job_name, job_state): 32 | for run in job_state["runs"]: 33 | run_num = run["run_num"] 34 | state_manager.save(runstate.JOB_RUN_STATE, f"{job_name}.{run_num}", run) 35 | run_nums = [run["run_num"] for run in job_state["runs"]] 36 | job_state["run_nums"] = run_nums 37 | # Note: not removing 'runs' from job_state for safety. 38 | # If Tron starts up correctly after the state migration, it will update the job state 39 | # and remove 'runs'. 40 | state_manager.save(runstate.JOB_STATE, job_name, job_state) 41 | 42 | 43 | def move_job_runs_to_job(state_manager, job_name, job_state): 44 | runs = state_manager._restore_runs_for_job(job_name, job_state) 45 | job_state["runs"] = runs 46 | state_manager.save(runstate.JOB_STATE, job_name, job_state) 47 | for run in runs: 48 | state_manager.delete(runstate.JOB_RUN_STATE, f'{job_name}.{run["run_num"]}') 49 | 50 | 51 | def update_state(state_manager, job_names, back): 52 | jobs = state_manager._restore_dicts(runstate.JOB_STATE, job_names) 53 | for job_name, job_state in jobs.items(): 54 | if back: 55 | move_job_runs_to_job(state_manager, job_name, job_state) 56 | else: 57 | create_job_runs_for_job(state_manager, job_name, job_state) 58 | 59 | 60 | def migrate_state(config_path, working_dir, back): 61 | with chdir(working_dir): 62 | config_manager = manager.ConfigManager(config_path) 63 | config_container = config_manager.load() 64 | job_names = config_container.get_job_names() 65 | state_config = config_container.get_master().state_persistence 66 | state_manager = PersistenceManagerFactory.from_config(state_config) 67 | update_state(state_manager, job_names, back) 68 | state_manager.cleanup() 69 | 70 | 71 | if __name__ == "__main__": 72 | # INFO for boto, DEBUG for all tron-related state logs 73 | logging.basicConfig(level=logging.INFO) 74 | logging.getLogger("tron").setLevel(logging.DEBUG) 75 | 76 | args = parse_args() 77 | migrate_state(args.config_path, args.working_dir, args.back) 78 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py38 3 | 4 | [testenv] 5 | basepython = python3.8 6 | deps = 7 | --requirement={toxinidir}/requirements.txt 8 | --requirement={toxinidir}/requirements-dev.txt 9 | usedevelop = true 10 | passenv = USER PIP_INDEX_URL 11 | setenv = 12 | YARN_REGISTRY = {env:NPM_CONFIG_REGISTRY:https://registry.npmjs.org/} 13 | whitelist_externals= 14 | yarn 15 | commands = 16 | # we have a pre-commit hook that runs eslint over our FE files, but our 17 | # current setup means that the necessary binaries won't be installed 18 | # automatically, so we manually invoke yarn to install things 19 | yarn install 20 | pre-commit install -f --install-hooks 21 | pre-commit run --all-files 22 | # tron has been around for a while, so we'll need to slowly add types or make an effort 23 | # to get it mypy-clean in one shot - until then, let's only check files that we've added types to 24 | mypy --pretty tron/kubernetes.py tron/commands/backfill.py bin/tronctl tron/utils/logreader.py 25 | check-requirements 26 | # optionally install yelpy requirements - this is after check-requirements since 27 | # check-requirements doesn't understand these extra requirements 28 | -pip install -r yelp_package/extra_requirements_yelp.txt 29 | # we then run tests at the very end so that we can run tests with yelpy requirements 30 | py.test -s {posargs:tests} 31 | 32 | [flake8] 33 | ignore = E501,E265,E241,E704,E251,W504,E231,W503,E203 34 | 35 | [testenv:docs] 36 | deps = 37 | --requirement={toxinidir}/requirements-docs.txt 38 | --requirement={toxinidir}/requirements.txt 39 | whitelist_externals= 40 | mkdir 41 | commands= 42 | /bin/rm -rf docs/source/generated/ 43 | # The last arg to apidoc is a list of excluded paths 44 | sphinx-apidoc -f -e -o docs/source/generated/ tron 45 | mkdir -p docs 46 | sphinx-build -b html -d docs/_build docs/source docs/_build/html 47 | 48 | [testenv:example-cluster] 49 | whitelist_externals=docker-compose 50 | deps= 51 | docker-compose 52 | commands= 53 | docker-compose -f example-cluster/docker-compose.yml build playground 54 | docker-compose -f example-cluster/docker-compose.yml run -p 8089:8089 playground 55 | docker-compose -f example-cluster/docker-compose.yml down 56 | 57 | [testenv:itest] 58 | commands = 59 | make deb_bionic 60 | make _itest_bionic 61 | make deb_jammy 62 | make _itest_jammy 63 | -------------------------------------------------------------------------------- /tron/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015-2016 Yelp Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # It is imperative that this file not contain any imports from our 16 | # dependencies. Since this file is imported from setup.py in the 17 | # setup phase, the dependencies may not exist on disk yet. 18 | # 19 | # Don't bump version manually. See `make release` docs in ./Makefile 20 | __version__ = "3.6.1" 21 | -------------------------------------------------------------------------------- /tron/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tron/api/__init__.py -------------------------------------------------------------------------------- /tron/api/async_resource.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | from twisted.internet import threads 5 | from twisted.web import server 6 | 7 | from tron.metrics import timer 8 | 9 | 10 | def report_resource_request(resource, request, duration_ms): 11 | timer( 12 | name=f"tron.api.{resource.__class__.__name__}", 13 | delta=duration_ms, 14 | dimensions={"method": request.method.decode()}, 15 | ) 16 | 17 | 18 | class AsyncResource: 19 | capacity = 10 20 | semaphore = threading.Semaphore(value=capacity) 21 | lock = threading.Lock() 22 | 23 | @staticmethod 24 | def finish(result, request, resource): 25 | result, duration_ms = result 26 | request.write(result) 27 | request.finish() 28 | report_resource_request(resource, request, duration_ms) 29 | 30 | @staticmethod 31 | def process(fn, resource, request): 32 | start = time.time() 33 | with AsyncResource.semaphore: 34 | result = fn(resource, request) 35 | duration_ms = 1000 * (time.time() - start) 36 | return result, duration_ms 37 | 38 | @staticmethod 39 | def bounded(fn): 40 | def wrapper(resource, request): 41 | d = threads.deferToThread( 42 | AsyncResource.process, 43 | fn, 44 | resource, 45 | request, 46 | ) 47 | d.addCallback(AsyncResource.finish, request, resource) 48 | d.addErrback(request.processingFailed) 49 | return server.NOT_DONE_YET 50 | 51 | return wrapper 52 | 53 | @staticmethod 54 | def exclusive(fn): 55 | def wrapper(resource, request): 56 | # ensures only one exclusive request starts consuming the semaphore 57 | start = time.time() 58 | with AsyncResource.lock: 59 | # this will wait until all bounded requests finished processing 60 | for _ in range(AsyncResource.capacity): 61 | AsyncResource.semaphore.acquire() 62 | try: 63 | return fn(resource, request) 64 | finally: 65 | for _ in range(AsyncResource.capacity): 66 | AsyncResource.semaphore.release() 67 | duration_ms = 1000 * (time.time() - start) 68 | report_resource_request(resource, request, duration_ms) 69 | 70 | return wrapper 71 | -------------------------------------------------------------------------------- /tron/api/requestargs.py: -------------------------------------------------------------------------------- 1 | """Functions for returning validated values from a twisted.web.Request object. 2 | """ 3 | import datetime 4 | 5 | DATE_FORMAT = "%Y-%m-%d %H:%M:%S" 6 | 7 | 8 | def get_integer(request, key): 9 | """Returns the first value in the request args for the given key, if that 10 | value is an integer. Otherwise returns None. 11 | """ 12 | value = get_string(request, key) 13 | if value is None or not value.isdigit(): 14 | return None 15 | 16 | return int(value) 17 | 18 | 19 | def get_string(request, key): 20 | """Returns the first value in the request args for a given key.""" 21 | if not request.args: 22 | return None 23 | 24 | if type(key) is not bytes: 25 | key = key.encode() 26 | 27 | if key not in request.args: 28 | return None 29 | 30 | val = request.args[key][0] 31 | if val is not None and type(val) is bytes: 32 | val = val.decode() 33 | 34 | return val 35 | 36 | 37 | def get_bool(request, key, default=None): 38 | """Returns True if the key exists and is truthy in the request args.""" 39 | int_value = get_integer(request, key) 40 | if int_value is None: 41 | return default 42 | 43 | return bool(int_value) 44 | 45 | 46 | def get_datetime(request, key): 47 | """Returns the first value in the request args for a given key. Casts to 48 | a datetime. Returns None if the value cannot be converted to datetime. 49 | """ 50 | val = get_string(request, key) 51 | if not val: 52 | return None 53 | 54 | try: 55 | return datetime.datetime.strptime(val, DATE_FORMAT) 56 | except ValueError: 57 | return None 58 | -------------------------------------------------------------------------------- /tron/bin/action_status.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.8 2 | import argparse 3 | import logging 4 | import os 5 | import signal 6 | 7 | from tron import yaml 8 | 9 | log = logging.getLogger("tron.action_status") 10 | 11 | STATUS_FILE = "status" 12 | 13 | 14 | def get_field(field, status_file): 15 | docs = yaml.load_all(status_file.read()) 16 | content = list(docs)[-1] 17 | return content.get(field) 18 | 19 | 20 | def print_status_file(status_file): 21 | for line in status_file.readlines(): 22 | print(yaml.load(line)) 23 | 24 | 25 | def send_signal(signal_num, status_file): 26 | pid = get_field("pid", status_file) 27 | if pid: 28 | try: 29 | os.killpg(os.getpgid(pid), signal_num) 30 | except OSError as e: 31 | msg = "Failed to signal %s with %s: %s" 32 | raise SystemExit(msg % (pid, signal_num, e)) 33 | 34 | 35 | commands = { 36 | "print": print_status_file, 37 | "pid": lambda statusfile: print(get_field("pid", statusfile)), 38 | "return_code": lambda statusfile: print(get_field("return_code", statusfile)), 39 | "terminate": lambda statusfile: send_signal(signal.SIGTERM, statusfile), 40 | "kill": lambda statusfile: send_signal(signal.SIGKILL, statusfile), 41 | } 42 | 43 | 44 | def parse_args(): 45 | parser = argparse.ArgumentParser(description="Action Status for Tron") 46 | parser.add_argument( 47 | "output_dir", 48 | help="The directory where the state of the action run is", 49 | ) 50 | parser.add_argument( 51 | "command", 52 | help="the command to run", 53 | ) 54 | parser.add_argument( 55 | "run_id", 56 | help="run_id of the action", 57 | ) 58 | return parser.parse_args() 59 | 60 | 61 | def run_command(command, status_file): 62 | commands[command](status_file) 63 | 64 | 65 | def main(): 66 | logging.basicConfig() 67 | args = parse_args() 68 | with open(os.path.join(args.output_dir, STATUS_FILE)) as f: 69 | run_command(args.command, f) 70 | 71 | 72 | if __name__ == "__main__": 73 | main() 74 | -------------------------------------------------------------------------------- /tron/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tron/commands/__init__.py -------------------------------------------------------------------------------- /tron/commands/authentication.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import cast 3 | from typing import Optional 4 | 5 | from tron.commands.cmd_utils import get_client_config 6 | 7 | try: 8 | from vault_tools.oidc import get_instance_oidc_identity_token # type: ignore # library lacks py.typed marker 9 | from okta_auth import get_and_cache_jwt_default # type: ignore # library lacks py.typed marker 10 | except ImportError: 11 | 12 | def get_instance_oidc_identity_token(role: str, ecosystem: Optional[str] = None) -> str: 13 | return "" 14 | 15 | def get_and_cache_jwt_default(client_id: str) -> str: 16 | return "" 17 | 18 | 19 | def get_sso_auth_token() -> str: 20 | """Generate an authentication token for the calling user from the Single Sign On provider, if configured""" 21 | client_id = get_client_config().get("auth_sso_oidc_client_id") 22 | return cast(str, get_and_cache_jwt_default(client_id, refreshable=True)) if client_id else "" 23 | 24 | 25 | def get_vault_auth_token() -> str: 26 | """Generate an authentication token for the underlying instance via Vault""" 27 | vault_role = get_client_config().get("vault_api_auth_role", "service_authz") 28 | return cast(str, get_instance_oidc_identity_token(vault_role)) 29 | 30 | 31 | def get_auth_token() -> str: 32 | """Generate authentication token via Vault or Okta""" 33 | return get_vault_auth_token() if os.getenv("TRONCTL_VAULT_AUTH") else get_sso_auth_token() 34 | -------------------------------------------------------------------------------- /tron/config/__init__.py: -------------------------------------------------------------------------------- 1 | class ConfigError(Exception): 2 | """Generic exception class for errors with config validation""" 3 | 4 | pass 5 | -------------------------------------------------------------------------------- /tron/config/static_config.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | import staticconf # type: ignore 4 | from staticconf import config 5 | 6 | FILENAME = "/nail/srv/configs/tron.yaml" 7 | NAMESPACE = "tron" 8 | 9 | 10 | def load_yaml_file() -> None: 11 | staticconf.YamlConfiguration(filename=FILENAME, namespace=NAMESPACE) 12 | 13 | 14 | def build_configuration_watcher(filename: str, namespace: str) -> config.ConfigurationWatcher: 15 | config_loader = partial(staticconf.YamlConfiguration, filename, namespace=namespace) 16 | reloader = config.ReloadCallbackChain(namespace) 17 | return config.ConfigurationWatcher(config_loader, filename, min_interval=0, reloader=reloader) 18 | 19 | 20 | # Load configuration from 'tron.yaml' into namespace 'tron' 21 | def get_config_watcher() -> config.ConfigurationWatcher: 22 | load_yaml_file() 23 | return build_configuration_watcher(FILENAME, NAMESPACE) 24 | -------------------------------------------------------------------------------- /tron/core/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | tron.core contains all the core objects for running and scheduling jobs. 3 | """ 4 | -------------------------------------------------------------------------------- /tron/core/actiongraph.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections import namedtuple 3 | from typing import Mapping 4 | 5 | from tron.core.action import Action 6 | from tron.utils.timeutils import delta_total_seconds 7 | 8 | log = logging.getLogger(__name__) 9 | Trigger = namedtuple("Trigger", ["name", "command"]) 10 | 11 | 12 | class ActionGraph: 13 | """A directed graph of actions and their requirements for a specific job.""" 14 | 15 | def __init__(self, action_map: Mapping[str, Action], required_actions, required_triggers): 16 | self.action_map = action_map 17 | self.required_actions = required_actions 18 | self.required_triggers = required_triggers 19 | self.all_triggers = set(self.required_triggers) 20 | for action_triggers in self.required_triggers.values(): 21 | self.all_triggers |= action_triggers 22 | self.all_triggers -= set(self.action_map) 23 | 24 | def get_dependencies(self, action_name, include_triggers=False): 25 | """Given an Action's name return the Actions (and optionally, Triggers) 26 | required to run before that Action. 27 | """ 28 | if action_name not in set(self.action_map) | self.all_triggers: 29 | return [] 30 | 31 | dependencies = [self.action_map[action] for action in self.required_actions[action_name]] 32 | if include_triggers: 33 | dependencies += [self[trigger_name] for trigger_name in self.required_triggers[action_name]] 34 | return dependencies 35 | 36 | def names(self, include_triggers=False): 37 | names = set(self.action_map) 38 | if include_triggers: 39 | names |= self.all_triggers 40 | return names 41 | 42 | @property 43 | def expected_runtime(self): 44 | return {name: delta_total_seconds(self.action_map[name].expected_runtime) for name in self.action_map.keys()} 45 | 46 | def __getitem__(self, name): 47 | if name in self.action_map: 48 | return self.action_map[name] 49 | elif name in self.all_triggers: 50 | # we don't have the Trigger config to know what the real command is, 51 | # so we just fill in the command with 'TRIGGER' 52 | return Trigger(name, "TRIGGER") 53 | else: 54 | raise KeyError(f"{name} is not a valid action") 55 | 56 | def __eq__(self, other): 57 | return ( 58 | self.action_map == other.action_map 59 | and self.required_actions == other.required_actions 60 | and self.required_triggers == other.required_triggers 61 | ) 62 | 63 | def __ne__(self, other): 64 | return not self == other 65 | -------------------------------------------------------------------------------- /tron/core/recovery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from tron.core.actionrun import ActionRun 4 | from tron.core.actionrun import KubernetesActionRun 5 | from tron.core.actionrun import MesosActionRun 6 | from tron.core.actionrun import SSHActionRun 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | def filter_action_runs_needing_recovery(action_runs): 12 | ssh_runs = [] 13 | mesos_runs = [] 14 | kubernetes_runs = [] 15 | for action_run in action_runs: 16 | if isinstance(action_run, SSHActionRun): 17 | if action_run.state == ActionRun.UNKNOWN: 18 | ssh_runs.append(action_run) 19 | elif isinstance(action_run, MesosActionRun): 20 | if action_run.state == ActionRun.UNKNOWN and action_run.end_time is None: 21 | mesos_runs.append(action_run) 22 | elif isinstance(action_run, KubernetesActionRun): 23 | if action_run.state == ActionRun.UNKNOWN and action_run.end_time is None: 24 | kubernetes_runs.append(action_run) 25 | return ssh_runs, mesos_runs, kubernetes_runs 26 | 27 | 28 | def launch_recovery_actionruns_for_job_runs(job_runs, master_action_runner): 29 | for run in job_runs: 30 | if not run._action_runs: 31 | log.info(f"Skipping recovery of {run} with no action runs (may have been cleaned up)") 32 | continue 33 | 34 | # TODO: Why do we do this separately if we just need to call recover() 35 | ssh_runs, mesos_runs, kubernetes_runs = filter_action_runs_needing_recovery(run._action_runs) 36 | for action_run in ssh_runs: 37 | action_run.recover() 38 | 39 | for action_run in mesos_runs: 40 | action_run.recover() 41 | 42 | for action_run in kubernetes_runs: 43 | action_run.recover() 44 | -------------------------------------------------------------------------------- /tron/default_config.yaml: -------------------------------------------------------------------------------- 1 | ssh_options: 2 | ## Tron needs SSH keys to allow the effective user to login to each of the 3 | ## nodes specified in the "nodes" section. You can choose to use either an 4 | ## SSH agent or list 5 | # identities: 6 | # - /home/tron/.ssh/id_dsa 7 | agent: false 8 | 9 | ## Directory used to store stdout/stderr from jobs. Defaults 10 | ## to the working directory 11 | # output_stream_dir: /tmp/tron/streams/ 12 | 13 | #state_persistence: 14 | ## Configuration for how to store Tron state data 15 | # name: 'shelve' 16 | # store_type: 'tron_State.shelve' 17 | # buffer_size: 18 | 19 | nodes: 20 | ## You'll need to list out all the available nodes for doing work. 21 | # - name: "node" 22 | # hostname: 'localhost' 23 | # username: 'tronuser' 24 | 25 | ## Optionally you can list 'pools' of nodes where selection of a node will 26 | ## be randomly determined or jobs can be configured to be run on all nodes 27 | ## in the pool 28 | # node_pools: 29 | # - name: NodePool 30 | # nodes: [node] 31 | 32 | command_context: 33 | # Variable subsitution 34 | # There are some built-in values such as 'node', 'runid', 'actionname' and 35 | # run-time based variables such as 'shortdate'. (See tronfig.1 for 36 | # reference.) You can specify whatever else you want similiar to 37 | # environment variables: 38 | # PYTHON: "/usr/bin/python" 39 | 40 | jobs: 41 | ## Configure your jobs here by specifing a name, node, schedule and the 42 | ## work flow that should executed. 43 | # - name: "sample_job" 44 | # node: node 45 | # schedule: "daily" 46 | # actions: 47 | # - name: "uname" 48 | # command: "uname -a" 49 | # cleanup_action: 50 | # command: "rm -rf /tmp/sample_job_scratch" 51 | -------------------------------------------------------------------------------- /tron/logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root, twisted, tron, tron.serialize, task_processing, tron.mesos.task_output, pymesos 3 | 4 | [handlers] 5 | keys=timedRotatingFileHandler, syslogHandler, nullHandler 6 | 7 | [formatters] 8 | keys=defaultFormatter, syslogFormatter 9 | 10 | [logger_root] 11 | level=WARNING 12 | handlers=timedRotatingFileHandler 13 | 14 | [logger_twisted] 15 | level=WARNING 16 | handlers=timedRotatingFileHandler 17 | qualname=twisted 18 | propagate=0 19 | 20 | [logger_tron] 21 | level=WARNING 22 | handlers=timedRotatingFileHandler 23 | qualname=tron 24 | propagate=0 25 | 26 | [logger_tron.serialize] 27 | level=CRITICAL 28 | handlers=timedRotatingFileHandler 29 | qualname=tron 30 | propagate=0 31 | 32 | [logger_task_processing] 33 | level=WARNING 34 | handlers=timedRotatingFileHandler 35 | qualname=task_processing 36 | propagate=0 37 | 38 | [logger_pymesos] 39 | level=DEBUG 40 | handlers=syslogHandler 41 | qualname=pymesos 42 | propagate=0 43 | 44 | [logger_tron.mesos.task_output] 45 | level=INFO 46 | handlers=nullHandler 47 | qualname=tron.mesos.task_output 48 | propagate=0 49 | 50 | [handler_timedRotatingFileHandler] 51 | class=logging.handlers.TimedRotatingFileHandler 52 | level=INFO 53 | formatter=defaultFormatter 54 | args=('/var/log/tron/tron.log', 'D') 55 | 56 | [handler_syslogHandler] 57 | class=logging.handlers.SysLogHandler 58 | level=WARNING 59 | formatter=syslogFormatter 60 | args=('/dev/log',) 61 | 62 | [handler_nullHandler] 63 | class=logging.NullHandler 64 | level=DEBUG 65 | args=() 66 | 67 | [formatter_defaultFormatter] 68 | format=%(asctime)s %(name)s %(levelname)s %(message)s 69 | 70 | [formatter_syslogFormatter] 71 | format=tron[%(process)d]: %(message)s 72 | -------------------------------------------------------------------------------- /tron/manhole.py: -------------------------------------------------------------------------------- 1 | from twisted.conch.insults import insults 2 | from twisted.conch.manhole import ColoredManhole 3 | from twisted.conch.telnet import TelnetBootstrapProtocol 4 | from twisted.conch.telnet import TelnetTransport 5 | from twisted.internet import protocol 6 | 7 | 8 | def make_manhole(namespace): 9 | f = protocol.ServerFactory() 10 | f.protocol = lambda: TelnetTransport( 11 | TelnetBootstrapProtocol, 12 | insults.ServerProtocol, 13 | ColoredManhole, 14 | namespace, 15 | ) 16 | return f 17 | -------------------------------------------------------------------------------- /tron/metrics.py: -------------------------------------------------------------------------------- 1 | from pyformance.meters import Counter # type: ignore 2 | from pyformance.meters import Histogram 3 | from pyformance.meters import Meter 4 | from pyformance.meters import SimpleGauge 5 | from pyformance.meters import Timer 6 | 7 | all_metrics = {} # type: ignore 8 | 9 | 10 | def get_metric(metric_type, name, dimensions, default): 11 | global all_metrics 12 | dimensions = tuple(sorted(dimensions.items())) if dimensions else () 13 | key = (metric_type, name, dimensions) 14 | return all_metrics.setdefault(key, default) 15 | 16 | 17 | def timer(name, delta, dimensions=None): 18 | timer = get_metric("timer", name, dimensions, Timer()) 19 | timer._update(delta) 20 | 21 | 22 | def count(name, inc=1, dimensions=None): 23 | counter = get_metric("counter", name, dimensions, Counter()) 24 | counter.inc(inc) 25 | 26 | 27 | def meter(name, dimensions=None): 28 | meter = get_metric("meter", name, dimensions, Meter()) 29 | meter.mark() 30 | 31 | 32 | def gauge(name, value, dimensions=None): 33 | gauge = get_metric("gauge", name, dimensions, SimpleGauge()) 34 | gauge.set_value(value) 35 | 36 | 37 | def histogram(name, value, dimensions=None): 38 | histogram = get_metric("histogram", name, dimensions, Histogram()) 39 | histogram.add(value) 40 | 41 | 42 | def view_timer(timer): 43 | data = view_meter(timer) 44 | data.update(view_histogram(timer)) 45 | return data 46 | 47 | 48 | def view_counter(counter): 49 | return {"count": counter.get_count()} 50 | 51 | 52 | def view_meter(meter): 53 | return { 54 | "count": meter.get_count(), 55 | "m1_rate": meter.get_one_minute_rate(), 56 | "m5_rate": meter.get_five_minute_rate(), 57 | "m15_rate": meter.get_fifteen_minute_rate(), 58 | } 59 | 60 | 61 | def view_gauge(gauge): 62 | return {"value": gauge.get_value()} 63 | 64 | 65 | def view_histogram(histogram): 66 | snapshot = histogram.get_snapshot() 67 | return { 68 | "count": histogram.get_count(), 69 | "mean": histogram.get_mean(), 70 | "min": histogram.get_min(), 71 | "max": histogram.get_max(), 72 | "p50": snapshot.get_median(), 73 | "p75": snapshot.get_75th_percentile(), 74 | "p95": snapshot.get_95th_percentile(), 75 | "p99": snapshot.get_99th_percentile(), 76 | } 77 | 78 | 79 | metrics_to_viewers = { 80 | "counter": view_counter, 81 | "gauge": view_gauge, 82 | "histogram": view_histogram, 83 | "meter": view_meter, 84 | "timer": view_timer, 85 | } 86 | 87 | 88 | def view_all_metrics(): 89 | all_data = {metric_type: [] for metric_type in metrics_to_viewers} 90 | for (metric_type, name, dims), metric in all_metrics.items(): 91 | data = {"name": name, **metrics_to_viewers[metric_type](metric)} 92 | if dims: 93 | data.update({"dimensions": dict(dims)}) 94 | all_data[metric_type].append(data) 95 | return all_data 96 | -------------------------------------------------------------------------------- /tron/serialize/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tron/serialize/__init__.py -------------------------------------------------------------------------------- /tron/serialize/runstate/__init__.py: -------------------------------------------------------------------------------- 1 | # State types 2 | JOB_STATE = "job_state" 3 | JOB_RUN_STATE = "job_run_state" 4 | MESOS_STATE = "mesos_state" 5 | -------------------------------------------------------------------------------- /tron/serialize/runstate/yamlstore.py: -------------------------------------------------------------------------------- 1 | """Store state in a local YAML file. 2 | 3 | WARNING: Using this store is NOT recommended. It will be far too slow for 4 | anything but the most trivial setups. It should only be used with a high 5 | buffer size (10+), and a low run_limit (< 10). 6 | """ 7 | import operator 8 | import os 9 | from collections import namedtuple 10 | 11 | from tron import yaml 12 | from tron.serialize import runstate 13 | 14 | YamlKey = namedtuple("YamlKey", ["type", "iden"]) 15 | 16 | TYPE_MAPPING = { 17 | runstate.JOB_STATE: "jobs", 18 | } 19 | 20 | 21 | class YamlStateStore: 22 | def __init__(self, filename): 23 | self.filename = filename 24 | self.buffer = {} 25 | 26 | def build_key(self, type, iden): 27 | return YamlKey(TYPE_MAPPING[type], iden) 28 | 29 | def restore(self, keys): 30 | if not os.path.exists(self.filename): 31 | return {} 32 | 33 | with open(self.filename) as fh: 34 | self.buffer = yaml.load(fh) 35 | 36 | items = (self.buffer.get(key.type, {}).get(key.iden) for key in keys) 37 | key_item_pairs = zip(keys, items) 38 | return dict(filter(operator.itemgetter(1), key_item_pairs)) 39 | 40 | def save(self, key_value_pairs): 41 | for key, state_data in key_value_pairs: 42 | if state_data is None: 43 | self._delete_from_buffer(key) 44 | else: 45 | self.buffer.setdefault(key.type, {})[key.iden] = state_data 46 | self._write_buffer() 47 | 48 | def _delete_from_buffer(self, key): 49 | data_for_type = self.buffer.get(key.type, {}) 50 | if data_for_type.get(key.iden): 51 | del data_for_type[key.iden] 52 | if not data_for_type: # No remaining data for this type 53 | del self.buffer[key.type] 54 | 55 | def _write_buffer(self): 56 | with open(self.filename, "w") as fh: 57 | yaml.dump(self.buffer, fh) 58 | 59 | def cleanup(self): 60 | pass 61 | 62 | def __repr__(self): 63 | return "YamlStateStore('%s')" % self.filename 64 | -------------------------------------------------------------------------------- /tron/utils/__init__.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import fcntl 3 | import logging 4 | import os 5 | import signal 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it. 11 | def maybe_decode(maybe_string): 12 | if type(maybe_string) is bytes: 13 | return maybe_string.decode() 14 | return maybe_string 15 | 16 | 17 | # TODO: TRON-2293 maybe_encode is a relic of Python2->Python3 migration. Remove it. 18 | def maybe_encode(maybe_bytes): 19 | if type(maybe_bytes) is not bytes: 20 | return maybe_bytes.encode() 21 | return maybe_bytes 22 | 23 | 24 | def next_or_none(iterable): 25 | try: 26 | return next(iterable) 27 | except StopIteration: 28 | pass 29 | 30 | 31 | @contextlib.contextmanager 32 | def flock(fd): 33 | close = False 34 | if isinstance(fd, str): 35 | fd = open(fd, "a") 36 | close = True 37 | 38 | try: 39 | fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) 40 | except BlockingIOError as e: # locked by someone else 41 | log.debug(f"Locked by another process: {fd}") 42 | raise e 43 | 44 | try: 45 | yield 46 | finally: 47 | fcntl.lockf(fd, fcntl.LOCK_UN) 48 | if close: 49 | fd.close() 50 | 51 | 52 | @contextlib.contextmanager 53 | def chdir(path): 54 | cwd = os.getcwd() 55 | os.chdir(path) 56 | try: 57 | yield 58 | finally: 59 | os.chdir(cwd) 60 | 61 | 62 | @contextlib.contextmanager 63 | def signals(signal_map): 64 | orig_map = {} 65 | for signum, handler in signal_map.items(): 66 | orig_map[signum] = signal.signal(signum, handler) 67 | 68 | try: 69 | yield 70 | finally: 71 | for signum, handler in orig_map.items(): 72 | signal.signal(signum, handler) 73 | -------------------------------------------------------------------------------- /tron/utils/collections.py: -------------------------------------------------------------------------------- 1 | """Utilities for working with collections.""" 2 | import logging 3 | 4 | log = logging.getLogger(__name__) 5 | 6 | 7 | class MappingCollection(dict): 8 | """Dictionary like object for managing collections of items. Item is 9 | expected to support the following interface, and should be hashable. 10 | 11 | class Item(object): 12 | 13 | def get_name(self): ... 14 | 15 | def restore_state(self, state_data): ... 16 | 17 | def disable(self): ... 18 | 19 | def __eq__(self, other): ... 20 | 21 | """ 22 | 23 | def __init__(self, item_name): 24 | dict.__init__(self) 25 | self.item_name = item_name 26 | 27 | def filter_by_name(self, names): 28 | for name in set(self) - set(names): 29 | self.remove(name) 30 | 31 | def remove(self, name): 32 | if name not in self: 33 | raise ValueError(f"{self.item_name} {name} unknown") 34 | 35 | log.info("Removing %s %s", self.item_name, name) 36 | self.pop(name).disable() 37 | 38 | def contains_item(self, item, handle_update_func): 39 | if item == self.get(item.get_name()): 40 | return True 41 | 42 | return handle_update_func(item) if item.get_name() in self else False 43 | 44 | def add(self, item, update_func): 45 | if self.contains_item(item, update_func): 46 | return False 47 | 48 | log.info("Adding new %s" % item) 49 | self[item.get_name()] = item 50 | return True 51 | 52 | def replace(self, item): 53 | return self.add(item, self.remove_item) 54 | 55 | def remove_item(self, item): 56 | return self.remove(item.get_name()) 57 | -------------------------------------------------------------------------------- /tron/utils/exitcode.py: -------------------------------------------------------------------------------- 1 | # TRON-1826 2 | EXIT_INVALID_COMMAND = -1 3 | EXIT_NODE_ERROR = -2 4 | EXIT_STOP_KILL = -3 5 | EXIT_TRIGGER_TIMEOUT = -4 6 | EXIT_MESOS_DISABLED = -5 7 | EXIT_KUBERNETES_DISABLED = -6 8 | EXIT_KUBERNETES_NOT_CONFIGURED = -7 9 | EXIT_KUBERNETES_TASK_INVALID = -8 10 | EXIT_KUBERNETES_ABNORMAL = -9 11 | EXIT_KUBERNETES_SPOT_INTERRUPTION = -10 12 | EXIT_KUBERNETES_NODE_SCALEDOWN = -11 13 | EXIT_KUBERNETES_TASK_LOST = -12 14 | 15 | EXIT_REASONS = { 16 | EXIT_INVALID_COMMAND: "Invalid command", 17 | EXIT_NODE_ERROR: "Node error", 18 | EXIT_STOP_KILL: "Stopped or killed", 19 | EXIT_TRIGGER_TIMEOUT: "Timed out waiting for trigger", 20 | EXIT_MESOS_DISABLED: "Mesos disabled", 21 | EXIT_KUBERNETES_DISABLED: "Kubernetes disabled", 22 | EXIT_KUBERNETES_NOT_CONFIGURED: "Kubernetes enabled, but not configured", 23 | EXIT_KUBERNETES_TASK_INVALID: "Kubernetes task was not valid", 24 | EXIT_KUBERNETES_ABNORMAL: "Kubernetes task failed in an unexpected manner", 25 | EXIT_KUBERNETES_SPOT_INTERRUPTION: "Kubernetes task failed due to spot interruption", 26 | EXIT_KUBERNETES_NODE_SCALEDOWN: "Kubernetes task failed due to the autoscaler scaling down a node", 27 | EXIT_KUBERNETES_TASK_LOST: "Tron lost track of a pod it already thought it had started for a job.", 28 | } 29 | -------------------------------------------------------------------------------- /tron/utils/observer.py: -------------------------------------------------------------------------------- 1 | """Implements the Observer/Observable pattern,""" 2 | import logging 3 | 4 | log = logging.getLogger(__name__) 5 | 6 | 7 | class Observable: 8 | """An Observable in the Observer/Observable pattern. It stores 9 | specifications and Observers which can be notified of changes by calling 10 | notify. 11 | """ 12 | 13 | def __init__(self): 14 | self._observers = dict() 15 | 16 | def attach(self, watch_spec, observer): 17 | """Attach another observer to the listen_spec. 18 | 19 | Listener Spec matches on: 20 | True Matches everything 21 | Matches only that event 22 | Matches any of the events in the sequence 23 | """ 24 | if isinstance(watch_spec, (str, bool)): 25 | self._observers.setdefault(watch_spec, []).append(observer) 26 | return 27 | 28 | for spec in watch_spec: 29 | self._observers.setdefault(spec, []).append(observer) 30 | 31 | def clear_observers(self, watch_spec=None): 32 | """Remove all observers for a given watch_spec. Removes all 33 | observers if listen_spec is None 34 | """ 35 | if watch_spec is None or watch_spec is True: 36 | self._observers.clear() 37 | return 38 | 39 | del self._observers[watch_spec] 40 | 41 | def remove_observer(self, observer): 42 | """Remove an observer from all watch_specs.""" 43 | for observers in self._observers.values(): 44 | if observer in observers: 45 | observers.remove(observer) 46 | 47 | def _get_handlers_for_event(self, event): 48 | """Returns the complete list of handlers for the event.""" 49 | return self._observers.get(True, []) + self._observers.get(event, []) 50 | 51 | def notify(self, event, event_data=None): 52 | """Notify all observers of the event.""" 53 | handlers = self._get_handlers_for_event(event) 54 | log.debug( 55 | f"Notifying {len(handlers)} listeners for new event {event!r}", 56 | ) 57 | for handler in handlers: 58 | handler.handler(self, event, event_data) 59 | 60 | 61 | class Observer: 62 | """An observer in the Observer/Observable pattern. Given an observable 63 | object will watch for notify calls. Override handler to act on those 64 | notifications. 65 | """ 66 | 67 | def watch(self, observable, event=True): 68 | """Adds this Observer as a watcher of the observable.""" 69 | observable.attach(event, self) 70 | 71 | def watch_all(self, observables, event=True): 72 | for observable in observables: 73 | self.watch(observable, event) 74 | 75 | def handler(self, observable, event): 76 | """Override this method to call a method to handle events.""" 77 | pass 78 | 79 | def stop_watching(self, observable): 80 | observable.remove_observer(self) 81 | -------------------------------------------------------------------------------- /tron/utils/persistable.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from abc import abstractmethod 3 | from typing import Any 4 | from typing import Dict 5 | from typing import Optional 6 | 7 | 8 | class Persistable(ABC): 9 | @staticmethod 10 | @abstractmethod 11 | def to_json(state_data: Dict[Any, Any]) -> Optional[str]: 12 | pass 13 | 14 | @staticmethod 15 | @abstractmethod 16 | def from_json(state_data: str) -> Dict[str, Any]: 17 | # This method is called on because it is intended to handle the deserialization of JSON data into a 18 | # dictionary representation of the state. This allows the method to be used in a more flexible and generic way, 19 | # enabling different classes to implement their own specific logic for converting the dictionary into an instance of the 20 | # class. By returning a dictionary, the method provides a common interface for deserialization, while allowing subclasses 21 | # to define how the dictionary should be used to restore the state of the object. 22 | pass 23 | -------------------------------------------------------------------------------- /tron/utils/proxy.py: -------------------------------------------------------------------------------- 1 | """Utilities for creating classes that proxy function calls.""" 2 | 3 | 4 | class CollectionProxy: 5 | """Proxy attribute lookups to a sequence of objects.""" 6 | 7 | def __init__(self, obj_list_getter, definition_list=None): 8 | """See add() for a description of proxy definitions.""" 9 | self.obj_list_getter = obj_list_getter 10 | self._defs = {} 11 | for definition in definition_list or []: 12 | self.add(*definition) 13 | 14 | def add(self, attribute_name, aggregate_func, is_callable): 15 | """Add attributes to proxy, the aggregate function to use on the 16 | sequence of returned values, and a boolean identifying if this 17 | attribute is a callable or not. 18 | 19 | attribute_name - the name of the attribute to proxy 20 | aggregate_func - a function that takes a sequence as its only argument 21 | callable - if this attribute is a callable on every object in 22 | the obj_list (boolean) 23 | """ 24 | self._defs[attribute_name] = (aggregate_func, is_callable) 25 | 26 | def perform(self, name): 27 | """Attempt to perform the proxied lookup. Raises AttributeError if 28 | the name is not defined. 29 | """ 30 | if name not in self._defs: 31 | raise AttributeError(name) 32 | 33 | obj_list = self.obj_list_getter 34 | aggregate_func, is_callable = self._defs[name] 35 | 36 | if not is_callable: 37 | return aggregate_func(getattr(i, name) for i in obj_list()) 38 | 39 | def func(*args, **kwargs): 40 | return aggregate_func(getattr(item, name)(*args, **kwargs) for item in obj_list()) 41 | 42 | return func 43 | 44 | 45 | def func_proxy(name, func): 46 | return name, func, True 47 | 48 | 49 | def attr_proxy(name, func): 50 | return name, func, False 51 | 52 | 53 | class AttributeProxy: 54 | """Proxy attribute lookups to another object.""" 55 | 56 | def __init__(self, dest_obj, attribute_list=None): 57 | self._attributes = set(attribute_list or []) 58 | self.dest_obj = dest_obj 59 | 60 | def add(self, attribute_name): 61 | self._attributes.add(attribute_name) 62 | 63 | def perform(self, attribute_name): 64 | if attribute_name not in self._attributes: 65 | raise AttributeError(attribute_name) 66 | 67 | return getattr(self.dest_obj, attribute_name) 68 | -------------------------------------------------------------------------------- /tron/utils/queue.py: -------------------------------------------------------------------------------- 1 | import queue 2 | 3 | from twisted.internet import defer 4 | 5 | 6 | class PyDeferredQueue(defer.DeferredQueue): 7 | """ 8 | Implements the stdlib queue.Queue get/put interface with a DeferredQueue. 9 | """ 10 | 11 | def __init__(self, *args, **kwargs): 12 | super().__init__(*args, **kwargs) 13 | 14 | def put(self, item, block=None, timeout=None): 15 | # Call from reactor thread so callbacks from get() will be executed 16 | # on the reactor thread, even if this is called from another thread. 17 | from twisted.internet import reactor 18 | 19 | try: 20 | reactor.callFromThread(super().put, item) 21 | except defer.QueueOverflow: 22 | raise queue.Full 23 | 24 | def get(self, block=None, timeout=None): 25 | try: 26 | return super().get() 27 | except defer.QueueUnderflow: 28 | raise queue.Empty 29 | -------------------------------------------------------------------------------- /tron/utils/state.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections import defaultdict 3 | 4 | log = logging.getLogger(__name__) 5 | 6 | 7 | class Machine: 8 | @staticmethod 9 | def from_machine(machine, initial=None, state=None): 10 | if initial is None: 11 | initial = machine.initial 12 | if state is None: 13 | state = initial 14 | new_machine = Machine(initial, **machine.transitions) 15 | new_machine.state = state 16 | assert machine.transitions == new_machine.transitions 17 | assert machine.states == new_machine.states 18 | return new_machine 19 | 20 | def __init__(self, initial, **transitions): 21 | super().__init__() 22 | self.transitions = defaultdict(dict, transitions) 23 | self.transition_names = { 24 | transition_name 25 | for (_, transitions) in self.transitions.items() 26 | for (transition_name, _) in (transitions or {}).items() 27 | } 28 | self.states = set(transitions.keys()).union( 29 | state for (_, dst) in transitions.items() for (_, state) in (dst or {}).items() 30 | ) 31 | if initial not in self.states: 32 | raise RuntimeError( 33 | f"invalid machine: {initial} not in {self.states}", 34 | ) 35 | self.state = initial 36 | self.initial = initial 37 | 38 | def set_state(self, state): 39 | if state not in self.states: 40 | raise RuntimeError(f"invalid state: {state} not in {self.states}") 41 | self.state = state 42 | 43 | def reset(self): 44 | self.state = self.initial 45 | 46 | def check(self, transition): 47 | """Check if the state can be transitioned via `transition`. Returns the 48 | destination state. 49 | """ 50 | next_state = self.transitions[self.state].get(transition, None) 51 | return next_state 52 | 53 | def transition(self, transition): 54 | """Checks if machine can be transitioned from current state using 55 | provided transition name. Returns True if transition has taken place. 56 | Listeners for this change will also be notified before returning. 57 | """ 58 | next_state = self.check(transition) 59 | if next_state is None: 60 | return False 61 | 62 | log.debug(f"transitioning from {self.state} to {next_state}") 63 | self.state = next_state 64 | return True 65 | 66 | def __repr__(self): 67 | return f"" 68 | -------------------------------------------------------------------------------- /tron/utils/twistedutils.py: -------------------------------------------------------------------------------- 1 | from twisted.internet import defer 2 | from twisted.internet import reactor 3 | from twisted.python import failure 4 | 5 | 6 | class Error(Exception): 7 | pass 8 | 9 | 10 | def _cancel(deferred): 11 | """Re-implementing what's available in newer twisted in a crappy, but 12 | workable way.""" 13 | 14 | if not deferred.called: 15 | deferred.errback(failure.Failure(Error())) 16 | elif isinstance(deferred.result, defer.Deferred): 17 | _cancel(deferred.result) 18 | 19 | 20 | def defer_timeout(deferred, timeout): 21 | try: 22 | reactor.callLater(timeout, deferred.cancel) 23 | except AttributeError: 24 | reactor.callLater(timeout, lambda: _cancel(deferred)) 25 | -------------------------------------------------------------------------------- /tron/yaml.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | 4 | def dump(*args, **kwargs): 5 | kwargs["Dumper"] = yaml.CSafeDumper 6 | return yaml.dump(*args, **kwargs) 7 | 8 | 9 | def load(*args, **kwargs): 10 | kwargs["Loader"] = yaml.CSafeLoader 11 | return yaml.load(*args, **kwargs) 12 | 13 | 14 | def load_all(*args, **kwargs): 15 | kwargs["Loader"] = yaml.CSafeLoader 16 | return yaml.load_all(*args, **kwargs) 17 | 18 | 19 | safe_dump = dump 20 | safe_load = load 21 | safe_load_all = load_all 22 | -------------------------------------------------------------------------------- /tronweb/coffee/config.coffee: -------------------------------------------------------------------------------- 1 | class window.NamespaceList extends Backbone.Model 2 | url: "/" 3 | 4 | 5 | class window.Config extends Backbone.Model 6 | url: => 7 | "/config?name=" + @get('name') 8 | 9 | 10 | class NamespaceListEntryView extends ClickableListEntry 11 | tagName: "tr" 12 | 13 | template: _.template """ 14 | 15 | 16 | <%= name %> 17 | 18 | 19 | """ 20 | 21 | render: -> 22 | @$el.html @template 23 | name: @model 24 | @ 25 | 26 | 27 | class window.NamespaceListView extends Backbone.View 28 | initialize: (options) => 29 | @listenTo(@model, "sync", @render) 30 | 31 | tagName: "div" 32 | 33 | className: "span12" 34 | 35 | template: _.template """ 36 |

37 | 38 | Configuration Namespaces 39 |

40 |
41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 |
Name
50 |
51 | """ 52 | 53 | 54 | render: => 55 | @$el.html @template() 56 | entry = (name) -> new NamespaceListEntryView(model: name).render().el 57 | @$('tbody').append(entry(name) for name in @model.get('namespaces')) 58 | @ 59 | 60 | 61 | class window.ConfigView extends Backbone.View 62 | initialize: (options) => 63 | @listenTo(@model, "change", @render) 64 | 65 | tagName: "div" 66 | 67 | className: "span12" 68 | 69 | template: _.template """ 70 |

Config <%= name %>

71 |
72 | 73 |
74 | """ 75 | 76 | render: => 77 | @$el.html @template(@model.attributes) 78 | CodeMirror.fromTextArea(@$('textarea').get(0), readOnly: true) 79 | @ 80 | -------------------------------------------------------------------------------- /tronweb/coffee/nodes.coffee: -------------------------------------------------------------------------------- 1 | class NodeModel extends Backbone.Model 2 | 3 | 4 | class NodePoolModel extends Backbone.Model 5 | 6 | 7 | class NodeInlineView extends Backbone.View 8 | tagName: "span" 9 | 10 | template: _.template """ 11 | 12 | <%= name %> 13 | 14 | """ 15 | 16 | render: => 17 | @$el.html @template(@model.attributes) 18 | @ 19 | 20 | 21 | class NodePoolInlineView extends Backbone.View 22 | tagName: "span" 23 | 24 | template: _.template """ 25 | 26 | <%= name %> 27 | 28 | """ 29 | 30 | render: => 31 | @$el.html @template(@model.attributes) 32 | @ 33 | 34 | 35 | window.displayNode = (node) -> 36 | new NodeInlineView(model: new NodeModel(node)).render().$el.html() 37 | 38 | 39 | window.displayNodePool = (pool) -> 40 | new NodePoolInlineView(model: new NodePoolModel(pool)).render().$el.html() 41 | -------------------------------------------------------------------------------- /tronweb/fonts/webhostinghub-glyphs.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tronweb/fonts/webhostinghub-glyphs.ttf -------------------------------------------------------------------------------- /tronweb/img/ui-bg_diagonals-small_10_555_40x40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tronweb/img/ui-bg_diagonals-small_10_555_40x40.png -------------------------------------------------------------------------------- /tronweb/img/ui-bg_dots-medium_100_eee_4x4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tronweb/img/ui-bg_dots-medium_100_eee_4x4.png -------------------------------------------------------------------------------- /tronweb/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | tronweb 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 25 | 26 | 27 | 28 | 29 |
30 |
31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /tronweb/js/plugins.js: -------------------------------------------------------------------------------- 1 | // Avoid `console` errors in browsers that lack a console. 2 | (function() { 3 | var method; 4 | var noop = function () {}; 5 | var methods = [ 6 | 'assert', 'clear', 'count', 'debug', 'dir', 'dirxml', 'error', 7 | 'exception', 'group', 'groupCollapsed', 'groupEnd', 'info', 'log', 8 | 'markTimeline', 'profile', 'profileEnd', 'table', 'time', 'timeEnd', 9 | 'timeStamp', 'trace', 'warn' 10 | ]; 11 | var length = methods.length; 12 | var console = (window.console = window.console || {}); 13 | 14 | while (length--) { 15 | method = methods[length]; 16 | 17 | // Only stub undefined methods. 18 | if (!console[method]) { 19 | console[method] = noop; 20 | } 21 | } 22 | }()); 23 | 24 | // Place any jQuery/helper plugins in here. 25 | -------------------------------------------------------------------------------- /tronweb/js/underscore.extra.js: -------------------------------------------------------------------------------- 1 | _.mixin({ 2 | 3 | /* take elements from list while callback condition is met */ 4 | takeWhile: function(list, callback, context) { 5 | var xs = []; 6 | _.any(list, function(item, index, list) { 7 | var res = callback.call(context, item, index, list); 8 | if (res) { 9 | xs.push(item); 10 | return false; 11 | } else { 12 | return true; 13 | } 14 | }); 15 | return xs; 16 | }, 17 | 18 | /* Build an object with [key, value] from pair list or callback */ 19 | mash: function(list, callback, context) { 20 | var pair_callback = callback || _.identity; 21 | return _.reduce(list, function(obj, value, index, list) { 22 | var pair = pair_callback.call(context, value, index, list); 23 | if (typeof pair == "object" && pair.length == 2) { 24 | obj[pair[0]] = pair[1]; 25 | } 26 | return obj; 27 | }, {}); 28 | }, 29 | 30 | /* Return pairs [key, value] of object */ 31 | pairs: function(object) { 32 | return _.map(object, function(value, key) { 33 | return [key, value]; 34 | }); 35 | }, 36 | 37 | }) 38 | -------------------------------------------------------------------------------- /tronweb/js/yaml.js: -------------------------------------------------------------------------------- 1 | CodeMirror.defineMode("yaml", function() { 2 | 3 | var cons = ['true', 'false', 'on', 'off', 'yes', 'no']; 4 | var keywordRegex = new RegExp("\\b(("+cons.join(")|(")+"))$", 'i'); 5 | 6 | return { 7 | token: function(stream, state) { 8 | var ch = stream.peek(); 9 | var esc = state.escaped; 10 | state.escaped = false; 11 | /* comments */ 12 | if (ch == "#") { stream.skipToEnd(); return "comment"; } 13 | if (state.literal && stream.indentation() > state.keyCol) { 14 | stream.skipToEnd(); return "string"; 15 | } else if (state.literal) { state.literal = false; } 16 | if (stream.sol()) { 17 | state.keyCol = 0; 18 | state.pair = false; 19 | state.pairStart = false; 20 | /* document start */ 21 | if(stream.match(/---/)) { return "def"; } 22 | /* document end */ 23 | if (stream.match(/\.\.\./)) { return "def"; } 24 | /* array list item */ 25 | if (stream.match(/\s*-\s+/)) { return 'meta'; } 26 | } 27 | /* pairs (associative arrays) -> key */ 28 | if (!state.pair && stream.match(/^\s*([a-z0-9\._-])+(?=\s*:)/i)) { 29 | state.pair = true; 30 | state.keyCol = stream.indentation(); 31 | return "atom"; 32 | } 33 | if (state.pair && stream.match(/^:\s*/)) { state.pairStart = true; return 'meta'; } 34 | 35 | /* inline pairs/lists */ 36 | if (stream.match(/^(\{|\}|\[|\])/)) { 37 | if (ch == '{') 38 | state.inlinePairs++; 39 | else if (ch == '}') 40 | state.inlinePairs--; 41 | else if (ch == '[') 42 | state.inlineList++; 43 | else 44 | state.inlineList--; 45 | return 'meta'; 46 | } 47 | 48 | /* list seperator */ 49 | if (state.inlineList > 0 && !esc && ch == ',') { 50 | stream.next(); 51 | return 'meta'; 52 | } 53 | /* pairs seperator */ 54 | if (state.inlinePairs > 0 && !esc && ch == ',') { 55 | state.keyCol = 0; 56 | state.pair = false; 57 | state.pairStart = false; 58 | stream.next(); 59 | return 'meta'; 60 | } 61 | 62 | /* start of value of a pair */ 63 | if (state.pairStart) { 64 | /* block literals */ 65 | if (stream.match(/^\s*(\||\>)\s*/)) { state.literal = true; return 'meta'; }; 66 | /* references */ 67 | if (stream.match(/^\s*(\&|\*)[a-z0-9\._-]+\b/i)) { return 'variable-2'; } 68 | /* numbers */ 69 | if (state.inlinePairs == 0 && stream.match(/^\s*-?[0-9\.\,]+\s?$/)) { return 'number'; } 70 | if (state.inlinePairs > 0 && stream.match(/^\s*-?[0-9\.\,]+\s?(?=(,|}))/)) { return 'number'; } 71 | /* keywords */ 72 | if (stream.match(keywordRegex)) { return 'keyword'; } 73 | } 74 | 75 | /* nothing found, continue */ 76 | state.pairStart = false; 77 | state.escaped = (ch == '\\'); 78 | stream.next(); 79 | return null; 80 | }, 81 | startState: function() { 82 | return { 83 | pair: false, 84 | pairStart: false, 85 | keyCol: 0, 86 | inlinePairs: 0, 87 | inlineList: 0, 88 | literal: false, 89 | escaped: false 90 | }; 91 | } 92 | }; 93 | }); 94 | 95 | CodeMirror.defineMIME("text/x-yaml", "yaml"); 96 | -------------------------------------------------------------------------------- /tronweb/tronweb.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tronweb/tronweb.ico -------------------------------------------------------------------------------- /tronweb2/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | es6: true, 5 | }, 6 | extends: [ 7 | 'plugin:react/recommended', 8 | 'airbnb', 9 | ], 10 | globals: { 11 | Atomics: 'readonly', 12 | SharedArrayBuffer: 'readonly', 13 | }, 14 | parserOptions: { 15 | ecmaFeatures: { 16 | jsx: true, 17 | }, 18 | ecmaVersion: 2018, 19 | sourceType: 'module', 20 | }, 21 | plugins: [ 22 | 'react', 23 | ], 24 | rules: { 25 | 'react/jsx-filename-extension': [1, { 'extensions': ['.js', '.jsx'] }], 26 | }, 27 | }; 28 | -------------------------------------------------------------------------------- /tronweb2/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /tronweb2/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "homepage": "./", 4 | "dependencies": { 5 | "d3": "^5.16.0", 6 | "d3-scale": "^3.2.2", 7 | "fuse.js": "^6.4.1", 8 | "prop-types": "^15.7.2", 9 | "react": "^16.13.1", 10 | "react-bootstrap": "^1.3.0", 11 | "react-dom": "^16.13.1", 12 | "react-router-dom": "^5.2.0", 13 | "react-scripts": "3.4.1", 14 | "react-autosuggest": "^10.0.2" 15 | }, 16 | "scripts": { 17 | "start": "react-scripts start", 18 | "build": "react-scripts build", 19 | "test": "react-scripts test", 20 | "eject": "react-scripts eject" 21 | }, 22 | "eslintConfig": { 23 | "extends": "react-app" 24 | }, 25 | "browserslist": { 26 | "production": [ 27 | ">0.2%", 28 | "not dead", 29 | "not op_mini all" 30 | ], 31 | "development": [ 32 | "last 1 chrome version", 33 | "last 1 firefox version", 34 | "last 1 safari version" 35 | ] 36 | }, 37 | "devDependencies": { 38 | "eslint": "^6.6.0", 39 | "eslint-config-airbnb": "^18.2.0", 40 | "eslint-plugin-import": "^2.22.0", 41 | "eslint-plugin-jsx-a11y": "^6.3.1", 42 | "eslint-plugin-react": "^7.20.6", 43 | "eslint-plugin-react-hooks": "^4.1.0" 44 | }, 45 | "resolutions": { 46 | "axe-core": "4.7.0" 47 | }, 48 | "engines": { 49 | "node-version-shim": "10.x", 50 | "node": ">=10" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /tronweb2/public/fonts/webhostinghub-glyphs.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tronweb2/public/fonts/webhostinghub-glyphs.ttf -------------------------------------------------------------------------------- /tronweb2/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 16 | 17 | 18 | 19 | 28 | tronweb 29 | 30 | 31 | 32 |
33 | 34 | 35 | -------------------------------------------------------------------------------- /tronweb2/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "tronweb", 3 | "name": "tronweb", 4 | "icons": [ 5 | { 6 | "src": "tronweb.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /tronweb2/public/tronweb.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/Tron/d846e23d34a0b1bc184d2fdace5a3c929455fdf1/tronweb2/public/tronweb.ico -------------------------------------------------------------------------------- /tronweb2/src/App.css: -------------------------------------------------------------------------------- 1 | h1 { 2 | margin-bottom: 2rem; 3 | overflow-wrap: break-word; 4 | } 5 | -------------------------------------------------------------------------------- /tronweb2/src/App.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { 3 | HashRouter as Router, 4 | Switch, 5 | Route, 6 | } from 'react-router-dom'; 7 | import { NavBar, JobsDashboard, Job } from './components'; 8 | 9 | import './App.css'; 10 | 11 | function App() { 12 | return ( 13 | 14 |
15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 |
29 |
30 | 31 | ); 32 | } 33 | 34 | function Configs() { 35 | return

Configs

; 36 | } 37 | 38 | export default App; 39 | -------------------------------------------------------------------------------- /tronweb2/src/components/ActionGraph/ActionGraph.css: -------------------------------------------------------------------------------- 1 | .link { 2 | stroke: #000; 3 | stroke-width: 0.8px; 4 | } 5 | 6 | text { 7 | font-family: 'Droid Sans Mono', monospace; 8 | font-size: 12px; 9 | } 10 | 11 | text.external { 12 | fill: #fff; 13 | } 14 | 15 | rect.external{ 16 | fill: #555; 17 | } 18 | 19 | .tooltip.show { 20 | opacity: 1; 21 | } 22 | 23 | .tooltip-inner { 24 | min-width: 20em; 25 | text-align: left; 26 | background-color: white; 27 | border: solid black; 28 | color: black; 29 | font-family: 'Droid Sans Mono', monospace; 30 | padding: 0; 31 | } 32 | 33 | .tooltip-action-name { 34 | background-color: #ddd; 35 | } 36 | -------------------------------------------------------------------------------- /tronweb2/src/components/ActionGraph/index.js: -------------------------------------------------------------------------------- 1 | export { default } from './ActionGraph'; 2 | -------------------------------------------------------------------------------- /tronweb2/src/components/Job/Job.js: -------------------------------------------------------------------------------- 1 | import React, 2 | { 3 | useState, 4 | useEffect, 5 | } from 'react'; 6 | import { 7 | useParams, 8 | } from 'react-router-dom'; 9 | import JobScheduler from '../JobScheduler'; 10 | import JobSettings from '../JobSettings'; 11 | import ActionGraph from '../ActionGraph'; 12 | import { getJobColor, fetchFromApi } from '../../utils/utils'; 13 | 14 | function jobDisplay(job) { 15 | return ( 16 |
17 |
18 |

Details

19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 |
Status{job.status}
Node pool{job.node_pool.name}
Schedule
Settings 37 | 42 |
Last success{job.last_success}
Next run{job.next_run}
54 |
55 |
56 |
57 |

Action Graph

58 | 59 |
60 |
61 | ); 62 | } 63 | 64 | function Job() { 65 | const [job, setJobData] = useState(undefined); 66 | const { jobId } = useParams(); 67 | 68 | useEffect(() => { 69 | document.title = jobId; 70 | return fetchFromApi(`/api/jobs/${jobId}?include_action_graph=1`, setJobData); 71 | }, [jobId]); 72 | 73 | let jobContent = ( 74 |
75 | Loading... 76 |
77 | ); 78 | if (job !== undefined) { 79 | if ('error' in job) { 80 | jobContent = ( 81 |

82 | Error: 83 | {job.error.message} 84 |

85 | ); 86 | } else { 87 | jobContent = jobDisplay(job); 88 | } 89 | } 90 | 91 | return ( 92 |
93 |

{jobId}

94 | {jobContent} 95 |
96 | ); 97 | } 98 | 99 | export default Job; 100 | -------------------------------------------------------------------------------- /tronweb2/src/components/Job/index.js: -------------------------------------------------------------------------------- 1 | export { default } from './Job'; 2 | -------------------------------------------------------------------------------- /tronweb2/src/components/JobScheduler/JobScheduler.js: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types'; 2 | import React from 'react'; 3 | 4 | function JobScheduler(props) { 5 | const { scheduler: { type, value, jitter } } = props; 6 | return ( 7 | 8 | {type} 9 | {' '} 10 | {value} 11 | {jitter} 12 | 13 | ); 14 | } 15 | 16 | JobScheduler.propTypes = { 17 | scheduler: PropTypes.shape({ 18 | type: PropTypes.string.isRequired, 19 | value: PropTypes.string.isRequired, 20 | jitter: PropTypes.string.isRequired, 21 | }).isRequired, 22 | }; 23 | 24 | export default JobScheduler; 25 | -------------------------------------------------------------------------------- /tronweb2/src/components/JobScheduler/index.js: -------------------------------------------------------------------------------- 1 | export { default } from './JobScheduler'; 2 | -------------------------------------------------------------------------------- /tronweb2/src/components/JobSettings/JobSettings.js: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types'; 2 | import React from 'react'; 3 | 4 | function JobSettings(props) { 5 | const { allowOverlap, queueing, allNodes } = props; 6 | let overlapString = 'Cancel overlapping runs'; 7 | if (allowOverlap) { 8 | overlapString = 'Allow overlapping runs'; 9 | } else if (queueing) { 10 | overlapString = 'Queue overlapping runs'; 11 | } 12 | 13 | return ( 14 |
    15 |
  • {overlapString}
    • 16 | {allNodes &&
  • Runs on all nodes
    • } 17 |
18 | ); 19 | } 20 | 21 | JobSettings.propTypes = { 22 | allowOverlap: PropTypes.bool, 23 | queueing: PropTypes.bool, 24 | allNodes: PropTypes.bool, 25 | }; 26 | 27 | JobSettings.defaultProps = { 28 | allowOverlap: true, 29 | queueing: true, 30 | allNodes: false, 31 | }; 32 | 33 | export default JobSettings; 34 | -------------------------------------------------------------------------------- /tronweb2/src/components/JobSettings/index.js: -------------------------------------------------------------------------------- 1 | export { default } from './JobSettings'; 2 | -------------------------------------------------------------------------------- /tronweb2/src/components/JobsDashboard/JobsDashboard.css: -------------------------------------------------------------------------------- 1 | .name-cell { 2 | overflow-wrap: break-word; 3 | max-width: 30em; 4 | } 5 | 6 | tr { 7 | cursor: pointer; 8 | } 9 | -------------------------------------------------------------------------------- /tronweb2/src/components/JobsDashboard/JobsDashboard.js: -------------------------------------------------------------------------------- 1 | import React, 2 | { 3 | useState, 4 | useEffect, 5 | } from 'react'; 6 | import Fuse from 'fuse.js'; 7 | import { getJobColor, fetchFromApi } from '../../utils/utils'; 8 | import JobScheduler from '../JobScheduler'; 9 | import './JobsDashboard.css'; 10 | 11 | function buildJobTable(jobsList) { 12 | const tableRows = jobsList.map((job) => ( 13 | window.location.assign(`#/job/${job.name}`)}> 14 | {job.name} 15 | {job.status} 16 | 17 | {job.node_pool.name} 18 | {job.last_success} 19 | {job.next_run} 20 | 21 | )); 22 | return ( 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {tableRows} 36 | 37 |
NameStatusScheduleNode poolLast successNext run
38 | ); 39 | } 40 | 41 | function JobsDashboard() { 42 | const [jobData, setJobData] = useState(undefined); 43 | const [inputValue, setInputValue] = useState(''); 44 | 45 | useEffect(() => fetchFromApi('/api/jobs', setJobData), []); 46 | 47 | let jobContent = ( 48 |
49 | Loading... 50 |
51 | ); 52 | 53 | function filterJobs(string) { 54 | // Require close to exact match, but allow anywhere in the substring 55 | const searchOptions = { keys: ['name'], threshold: 0.2, ignoreLocation: true }; 56 | const fuseSearch = new Fuse(jobData.jobs, searchOptions); 57 | return fuseSearch.search(string).map((result) => result.item); 58 | } 59 | 60 | if (jobData !== undefined) { 61 | if ('error' in jobData) { 62 | jobContent = ( 63 |

64 | Error: 65 | {jobData.error.message} 66 |

67 | ); 68 | } else { 69 | let jobsListToShow = jobData.jobs; 70 | if (inputValue !== '') { 71 | jobsListToShow = filterJobs(inputValue); 72 | } 73 | jobContent = ( 74 |
75 |
76 | setInputValue(e.target.value)} /> 77 |
78 | {buildJobTable(jobsListToShow)} 79 |
80 | ); 81 | } 82 | } 83 | return ( 84 |
85 |

Scheduled Jobs

86 | {jobContent} 87 |
88 | ); 89 | } 90 | 91 | export default JobsDashboard; 92 | -------------------------------------------------------------------------------- /tronweb2/src/components/JobsDashboard/index.js: -------------------------------------------------------------------------------- 1 | export { default } from './JobsDashboard'; 2 | -------------------------------------------------------------------------------- /tronweb2/src/components/NavBar/NavBar.css: -------------------------------------------------------------------------------- 1 | .navbar { 2 | background-color: #D32323; 3 | } 4 | 5 | .navbar li i { 6 | margin-right: 7px; 7 | } 8 | 9 | .navbar a { 10 | text-decoration: none; 11 | color: #000; 12 | } 13 | 14 | .navbar a:hover { 15 | text-decoration: none; 16 | color: #000; 17 | } 18 | 19 | #autosuggest-input { 20 | border-top-left-radius: 0px; 21 | border-bottom-left-radius: 0px; 22 | } 23 | 24 | .react-autosuggest__suggestions-container--open { 25 | display: block; 26 | position: absolute; 27 | min-width: 100%; 28 | border: 1px solid rgba(0,0,0,.125); 29 | background-color: #fff; 30 | border-radius: 4px; 31 | border-radius: 4px; 32 | z-index: 2; 33 | right: -1px; 34 | } 35 | 36 | .react-autosuggest__suggestions-list { 37 | padding: 0; 38 | margin: 0; 39 | list-style-type: none; 40 | } 41 | 42 | .react-autosuggest__suggestion { 43 | cursor: pointer; 44 | padding: 0.5em 1em; 45 | position: relative; 46 | } 47 | 48 | .react-autosuggest__suggestion--highlighted { 49 | background-color: #ddd; 50 | } 51 | -------------------------------------------------------------------------------- /tronweb2/src/components/NavBar/index.js: -------------------------------------------------------------------------------- 1 | export { default } from './NavBar'; 2 | -------------------------------------------------------------------------------- /tronweb2/src/components/index.js: -------------------------------------------------------------------------------- 1 | export { default as NavBar } from './NavBar'; 2 | export { default as JobsDashboard } from './JobsDashboard'; 3 | export { default as Job } from './Job'; 4 | export { default as JobScheduler } from './JobScheduler'; 5 | export { default as JobSettings } from './JobSettings'; 6 | export { default as ActionGraph } from './ActionGraph'; 7 | -------------------------------------------------------------------------------- /tronweb2/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import App from './App'; 4 | 5 | ReactDOM.render( 6 | 7 | 8 | , 9 | document.getElementById('root'), 10 | ); 11 | -------------------------------------------------------------------------------- /tronweb2/src/utils/utils.js: -------------------------------------------------------------------------------- 1 | export function fetchFromApi(endpoint, dataCallback) { 2 | // Can change for testing 3 | const apiPrefix = ''; 4 | const url = apiPrefix + endpoint; 5 | 6 | // Return function to skip the callback 7 | let cancelled = false; 8 | function cancel() { 9 | cancelled = true; 10 | } 11 | 12 | fetch(url) 13 | .then((response) => { 14 | if (!response.ok) { 15 | return { error: { message: response.statusText, code: response.status } }; 16 | } 17 | return response.json(); 18 | }) 19 | .then((data) => { 20 | if (!cancelled) { 21 | dataCallback(data); 22 | } 23 | }) 24 | .catch((error) => { 25 | console.error(`Error fetching ${url}`, error); 26 | if (!cancelled) { 27 | dataCallback({ error: { message: 'connection error' } }); 28 | } 29 | }); 30 | 31 | return cancel; 32 | } 33 | 34 | export function getJobColor(status) { 35 | switch (status) { 36 | case 'running': 37 | return 'primary'; 38 | case 'disabled': 39 | return 'warning'; 40 | case 'enabled': 41 | return 'success'; 42 | default: 43 | return 'light'; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /tronweb_tests/SpecRunner.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | tronweb test runner 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /tronweb_tests/spec/README: -------------------------------------------------------------------------------- 1 | 2 | Create javascript specs from coffeescript by running: 3 | 4 | coffee -w -o tronweb_tests/spec/ -c tronweb_tests/tests/ 5 | 6 | 7 | These tests require jasmine 1.3.1+ to be installed at: 8 | 9 | tronweb_tests/lib 10 | -------------------------------------------------------------------------------- /tronweb_tests/tests/actionrun_test.coffee: -------------------------------------------------------------------------------- 1 | 2 | 3 | describe "actionrun.coffee", -> 4 | module = modules.actionrun 5 | 6 | describe "ActionRun Model", -> 7 | self = this 8 | 9 | beforeEach -> 10 | self.actionRun = new module.ActionRun 11 | action_name: 'action_name' 12 | job_name: 'job_name' 13 | run_num: 'run_num' 14 | 15 | it "url creates the correct url", -> 16 | url = self.actionRun.url() 17 | expect(url).toEqual('/jobs/job_name/run_num/action_name' + 18 | self.actionRun.urlArgs) 19 | 20 | it "parse builds urls", -> 21 | resp = self.actionRun.parse {} 22 | expect(resp['job_url']).toEqual('#job/job_name') 23 | expect(resp['job_run_url']).toEqual('#job/job_name/run_num') 24 | expect(resp['url']).toEqual('#job/job_name/run_num/action_name') 25 | 26 | describe "ActionRunHistory Model", -> 27 | self = this 28 | 29 | beforeEach -> 30 | self.collection = new module.ActionRunHistory [], 31 | job_name: 'job_name' 32 | action_name: 'action_name' 33 | 34 | it "url creates the correct url", -> 35 | expect(self.collection.url()).toEqual( 36 | '/jobs/job_name/action_name/') 37 | -------------------------------------------------------------------------------- /tronweb_tests/tests/dashboard_test.coffee: -------------------------------------------------------------------------------- 1 | 2 | 3 | describe "Dashboard module", -> 4 | 5 | describe "JobStatusBoxView", -> 6 | test = @ 7 | 8 | beforeEach -> 9 | test.model = new Job() 10 | spyOn(test.model, 'get') 11 | test.view = new modules.dashboard.JobStatusBoxView(model: test.model) 12 | 13 | it "count an empty list", -> 14 | test.model.get.andReturn([]) 15 | expect(test.view.count()).toEqual(0) 16 | 17 | it "count a non-empty list returns first items run number", -> 18 | runs = [{'run_num': 5}, {'run_num': 4}, {'run_num': 3}] 19 | test.model.get.andReturn(runs) 20 | expect(test.view.count()).toEqual(5) 21 | -------------------------------------------------------------------------------- /tronweb_tests/tests/navbar_test.coffee: -------------------------------------------------------------------------------- 1 | 2 | describe "navbar module", -> 3 | 4 | describe "NavView", -> 5 | 6 | describe "sorter", -> 7 | test = @ 8 | 9 | beforeEach -> 10 | test.view = new modules.navbar.NavView() 11 | test.items = [ 12 | "three", 13 | "one", 14 | "one longer", 15 | "TWO", 16 | "a one", 17 | "longone with TwO ones"] 18 | 19 | it "sorts shorter items first", -> 20 | mockThis = query: "one" 21 | sortedItems = test.view.sorter.call(mockThis, test.items) 22 | expected = [ 23 | "one", 24 | "one longer", 25 | "a one", 26 | "longone with TwO ones"] 27 | expect(sortedItems).toEqual(expected) 28 | 29 | it "sorts only matching items", -> 30 | mockThis = query: "TWO" 31 | sortedItems = test.view.sorter.call(mockThis, test.items) 32 | expect(sortedItems).toEqual(["TWO", "longone with TwO ones"]) 33 | 34 | it "matches case insensitive", -> 35 | mockThis = query: "two" 36 | sortedItems = test.view.sorter.call(mockThis, test.items) 37 | expect(sortedItems).toEqual(["TWO", "longone with TwO ones"]) 38 | -------------------------------------------------------------------------------- /tronweb_tests/tests/routes_test.coffee: -------------------------------------------------------------------------------- 1 | 2 | 3 | describe "routes.coffee", -> 4 | module = window.modules.routes 5 | 6 | it "splitKeyValuePairs creates object from list", -> 7 | obj = module.splitKeyValuePairs ['one=two', 'three=four'] 8 | expect(obj).toEqual 9 | one: 'two' 10 | three: 'four' 11 | 12 | it "getParamsMap creates object from string", -> 13 | obj = module.getParamsMap "a=nameThing;b=other" 14 | expect(obj).toEqual 15 | a: 'nameThing' 16 | b: 'other' 17 | 18 | 19 | describe "getLocationParams", -> 20 | 21 | beforeEach -> 22 | spyOn(module, 'getLocationHash') 23 | 24 | it "returns location with params", -> 25 | location = "#base;one=thing;another=what" 26 | module.getLocationHash.andReturn(location) 27 | [base, params] = module.getLocationParams() 28 | expect(base).toEqual("#base") 29 | expect(params).toEqual 30 | one: "thing" 31 | another: "what" 32 | 33 | it "returns location without params", -> 34 | module.getLocationHash.andReturn("#blah") 35 | [base, params] = module.getLocationParams() 36 | expect(base).toEqual("#blah") 37 | expect(params).toEqual {} 38 | 39 | 40 | it "buildLocationString creates a location string", -> 41 | params = 42 | thing: "ok" 43 | bar: "tmp" 44 | location = module.buildLocationString "#base", params 45 | expect(location).toEqual("#base;thing=ok;bar=tmp") 46 | 47 | 48 | describe "updateLocationParam", -> 49 | 50 | beforeEach -> 51 | window.routes = jasmine.createSpyObj('routes', ['navigate']) 52 | spyOn(module, 'getLocationHash') 53 | 54 | it "creates params when params is empty", -> 55 | module.getLocationHash.andReturn("#base") 56 | module.updateLocationParam('name', 'stars') 57 | expected = "#base;name=stars" 58 | expect(window.routes.navigate).toHaveBeenCalledWith(expected) 59 | 60 | it "updates existing param", -> 61 | module.getLocationHash.andReturn("#base;name=foo") 62 | module.updateLocationParam('name', 'stars') 63 | expected = "#base;name=stars" 64 | expect(window.routes.navigate).toHaveBeenCalledWith(expected) 65 | 66 | it "adds new params", -> 67 | module.getLocationHash.andReturn("#base;what=why") 68 | module.updateLocationParam('name', 'stars') 69 | expected = "#base;what=why;name=stars" 70 | expect(window.routes.navigate).toHaveBeenCalledWith(expected) 71 | -------------------------------------------------------------------------------- /tronweb_tests/tests/timeline_test.coffee: -------------------------------------------------------------------------------- 1 | 2 | module = modules.timeline 3 | 4 | describe "Timeline module", -> 5 | 6 | it "padMaxDate adds padding to maxDate", -> 7 | dates = [new Date("2013-04-20 01:00:00"), 8 | new Date("2013-04-20 02:30:30")] 9 | padded = module.padMaxDate(dates, 0.1) 10 | expect(padded).toEqual([dates[0], new Date("2013-04-20 02:39:33")]) 11 | -------------------------------------------------------------------------------- /yelp_package/bionic/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:bionic 2 | 3 | RUN apt-get -q update && \ 4 | DEBIAN_FRONTEND=noninteractive apt-get -q install -y --no-install-recommends \ 5 | coffeescript \ 6 | debhelper \ 7 | devscripts \ 8 | dh-virtualenv \ 9 | dpkg-dev \ 10 | gcc \ 11 | gdebi-core \ 12 | git \ 13 | help2man \ 14 | libffi-dev \ 15 | libgpgme11 \ 16 | libssl-dev \ 17 | libdb5.3-dev \ 18 | libyaml-dev \ 19 | libssl-dev \ 20 | libffi-dev \ 21 | python3.8-dev \ 22 | python3-pip \ 23 | python-tox \ 24 | rust-all \ 25 | wget \ 26 | && apt-get -q clean 27 | 28 | ARG PIP_INDEX_URL 29 | ARG NPM_CONFIG_REGISTRY 30 | ENV PIP_INDEX_URL=${PIP_INDEX_URL:-https://pypi.python.org/simple} 31 | ENV NPM_CONFIG_REGISTRY=${NPM_CONFIG_REGISTRY:-https://npm.yelpcorp.com} 32 | 33 | # Get yarn and node 34 | RUN wget --quiet -O - https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - 35 | RUN wget --quiet -O - https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - 36 | RUN echo "deb http://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list 37 | RUN echo "deb http://deb.nodesource.com/node_10.x bionic main" > /etc/apt/sources.list.d/nodesource.list 38 | RUN apt-get -q update && apt-get -q install -y --no-install-recommends yarn nodejs 39 | 40 | RUN pip3 install --trusted-host 169.254.255.254 --index-url ${PIP_INDEX_URL} virtualenv==16.7.5 41 | WORKDIR /work 42 | -------------------------------------------------------------------------------- /yelp_package/extra_requirements_yelp.txt: -------------------------------------------------------------------------------- 1 | asn1crypto==1.5.1 # vault-tools dependency 2 | atomicfile==1.0.1 # vault-tools dependency 3 | clusterman-metrics==2.2.1 # used by tron for pre-scaling for Spark runs 4 | crypto-lib==4.0.0 # vault-tools dependency 5 | gitdb==4.0.12 # vault-tools dependency 6 | gitpython==3.1.44 # vault-tools dependency 7 | hvac==1.2.1 # vault-tools dependency 8 | ipaddress==1.0.23 # vault-tools dependency 9 | logreader==1.2.0 # used by tron logreader 10 | ndg-httpsclient==0.5.1 # vault-tools dependency 11 | okta-auth==1.1.0 # used for API auth 12 | pygpgme==0.3 # vault-tools dependency 13 | pyhcl==0.4.5 # vault-tools dependency 14 | pyjwt==2.9.0 # required by okta-auth 15 | pyopenssl==23.2.0 # vault-tools dependency 16 | saml-helper==2.5.3 # required by okta-auth 17 | service-identity==24.2.0 # vault-tools dependency 18 | simplejson==3.19.2 # required by tron CLI 19 | smmap==5.0.2 # vault-tools dependency 20 | vault-tools==1.6.0 # used for API auth 21 | yelp-meteorite==2.1.1 # used by task-processing to emit metrics, clusterman-metrics dependency 22 | -------------------------------------------------------------------------------- /yelp_package/itest_dockerfiles/mesos/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015-2017 Yelp Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM ubuntu:xenial 16 | 17 | RUN apt-get update > /dev/null && \ 18 | DEBIAN_FRONTEND=noninteractive apt-get install -y \ 19 | apt-transport-https \ 20 | ca-certificates \ 21 | curl \ 22 | software-properties-common && \ 23 | rm -rf /var/lib/apt/lists/* 24 | 25 | RUN echo "deb http://repos.mesosphere.com/ubuntu xenial main" > /etc/apt/sources.list.d/mesosphere.list && \ 26 | apt-key adv --keyserver keyserver.ubuntu.com --recv 81026D0004C44CF7EF55ADF8DF7D54CBE56151BF && \ 27 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ 28 | echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu xenial stable" > /etc/apt/sources.list.d/docker.list && \ 29 | apt-get update > /dev/null && \ 30 | DEBIAN_FRONTEND=noninteractive apt-get install -y \ 31 | docker-ce \ 32 | docker-ce-cli \ 33 | libsasl2-modules \ 34 | libstdc++6 \ 35 | mesos=1.7.2-2.0.1 > /dev/null && \ 36 | rm -rf /var/lib/apt/lists/* 37 | 38 | COPY mesos-secrets mesos-slave-secret /etc/ 39 | RUN echo '{}' > /root/.dockercfg 40 | RUN chmod 600 /etc/mesos-secrets 41 | -------------------------------------------------------------------------------- /yelp_package/itest_dockerfiles/mesos/mesos-secrets: -------------------------------------------------------------------------------- 1 | { 2 | "credentials": [ 3 | { 4 | "principal": "slave", 5 | "secret": "secret1" 6 | }, 7 | { 8 | "principal": "tron", 9 | "secret": "tron-secret" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /yelp_package/itest_dockerfiles/mesos/mesos-slave-secret: -------------------------------------------------------------------------------- 1 | { 2 | "principal": "slave", 3 | "secret": "secret1" 4 | } 5 | -------------------------------------------------------------------------------- /yelp_package/itest_dockerfiles/tronmaster/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:bionic 2 | 3 | RUN apt-get -q update && \ 4 | DEBIAN_FRONTEND=noninteractive apt-get -q install -y --no-install-recommends \ 5 | coffeescript \ 6 | debhelper \ 7 | devscripts \ 8 | dh-virtualenv \ 9 | dpkg-dev \ 10 | gcc \ 11 | gdebi-core \ 12 | git \ 13 | help2man \ 14 | libffi-dev \ 15 | libgpgme11 \ 16 | libssl-dev \ 17 | libdb5.3-dev \ 18 | libyaml-dev \ 19 | libssl-dev \ 20 | libffi-dev \ 21 | python3.6-dev \ 22 | python3-pip \ 23 | python-tox \ 24 | wget \ 25 | && apt-get -q clean 26 | 27 | ARG PIP_INDEX_URL 28 | ARG NPM_CONFIG_REGISTRY 29 | ENV PIP_INDEX_URL=${PIP_INDEX_URL:-https://pypi.python.org/simple} 30 | ENV NPM_CONFIG_REGISTRY=${NPM_CONFIG_REGISTRY:-https://npm.yelpcorp.com} 31 | 32 | # Get yarn and node 33 | RUN wget --quiet -O - https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - 34 | RUN wget --quiet -O - https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - 35 | RUN echo "deb http://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list 36 | RUN echo "deb http://deb.nodesource.com/node_10.x bionic main" > /etc/apt/sources.list.d/nodesource.list 37 | RUN apt-get -q update && apt-get -q install -y --no-install-recommends yarn nodejs 38 | 39 | RUN pip3 install --trusted-host 169.254.255.254 --index-url ${PIP_INDEX_URL} virtualenv==16.7.5 40 | WORKDIR /work 41 | -------------------------------------------------------------------------------- /yelp_package/itest_dockerfiles/zookeeper/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015-2016 Yelp Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM ubuntu:bionic 16 | 17 | RUN apt-get update > /dev/null && \ 18 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 19 | zookeeper > /dev/null && \ 20 | apt-get clean 21 | 22 | EXPOSE 2181 23 | CMD ["/usr/share/zookeeper/bin/zkServer.sh", "start-foreground"] 24 | -------------------------------------------------------------------------------- /yelp_package/jammy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update -yq && \ 4 | apt-get install -yq \ 5 | # needed to add a ppa 6 | software-properties-common && \ 7 | add-apt-repository ppa:deadsnakes/ppa 8 | 9 | RUN apt-get -q update && \ 10 | DEBIAN_FRONTEND=noninteractive apt-get -q install -y --no-install-recommends \ 11 | coffeescript \ 12 | debhelper \ 13 | devscripts \ 14 | dh-virtualenv \ 15 | dpkg-dev \ 16 | gcc \ 17 | gdebi-core \ 18 | git \ 19 | help2man \ 20 | libffi-dev \ 21 | libgpgme11 \ 22 | libssl-dev \ 23 | libdb5.3-dev \ 24 | libyaml-dev \ 25 | libssl-dev \ 26 | libffi-dev \ 27 | python3.8-dev \ 28 | python3.8-distutils \ 29 | python3-pip \ 30 | rust-all \ 31 | tox \ 32 | wget \ 33 | g++ \ 34 | # 12.22, good enough 35 | nodejs \ 36 | && apt-get -q clean 37 | 38 | ARG PIP_INDEX_URL 39 | ARG NPM_CONFIG_REGISTRY 40 | ENV PIP_INDEX_URL=${PIP_INDEX_URL:-https://pypi.python.org/simple} 41 | ENV NPM_CONFIG_REGISTRY=${NPM_CONFIG_REGISTRY:-https://npm.yelpcorp.com} 42 | 43 | # Get yarn 44 | # I'd use ubuntu yarn (yarnpkg) but https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1019291 45 | RUN wget --quiet -O - https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - 46 | RUN echo "deb http://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list 47 | RUN apt-get -q update && apt-get -q install -y --no-install-recommends yarn 48 | 49 | RUN pip3 install --trusted-host 169.254.255.254 --index-url ${PIP_INDEX_URL} virtualenv==16.7.5 50 | WORKDIR /work 51 | --------------------------------------------------------------------------------