├── .coveragerc ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── custom.md │ ├── feature_request.md │ └── question.md └── workflows │ ├── ci.yml │ └── website.yml ├── .gitignore ├── AUTHORS ├── CHANGES ├── CONTRIBUTING.md ├── LICENSE ├── README.rst ├── TODO.txt ├── bash_completion_doit ├── dev_requirements.txt ├── doc ├── Makefile ├── _static │ ├── doit-logo-small.png │ ├── doit-logo.png │ ├── doit-text-160x60.png │ ├── doit.png │ ├── external.png │ ├── favico.ico │ ├── pygarden.png │ ├── python-powered-w-100x40.png │ ├── requests.models-blue.png │ ├── requests.models.png │ ├── requests.png │ ├── universal │ │ ├── custom.css │ │ ├── style.default.css │ │ └── texture-bw.png │ └── vendor │ │ ├── bootstrap-select │ │ └── css │ │ │ └── bootstrap-select.min.css │ │ ├── bootstrap │ │ ├── css │ │ │ └── bootstrap.min.css │ │ └── js │ │ │ └── bootstrap.min.js │ │ ├── font-awesome │ │ ├── css │ │ │ └── font-awesome.min.css │ │ └── fonts │ │ │ ├── FontAwesome.otf │ │ │ ├── fontawesome-webfont.eot │ │ │ ├── fontawesome-webfont.svg │ │ │ ├── fontawesome-webfont.ttf │ │ │ ├── fontawesome-webfont.woff │ │ │ └── fontawesome-webfont.woff2 │ │ └── owl.carousel │ │ ├── owl.carousel.css │ │ ├── owl.carousel.min.js │ │ └── owl.theme.default.min.css ├── _templates │ ├── donate.html │ └── layout.html ├── changes.rst ├── cmd-other.rst ├── cmd-run.rst ├── conf.py ├── configuration.rst ├── contents.rst ├── dependencies.rst ├── dictionary.txt ├── extending.rst ├── faq.rst ├── globals.rst ├── google726fc03ab55ebbfc.html ├── index.html ├── install.rst ├── make.bat ├── old │ └── index_old.rst ├── open_collective.md ├── presentation.rst ├── related.rst ├── robots.txt ├── samples │ ├── .gitignore │ ├── calc_dep.py │ ├── check_timestamp_unchanged.py │ ├── checker.py │ ├── clean_mix.py │ ├── cmd_actions.py │ ├── cmd_actions_list.py │ ├── cmd_from_callable.py │ ├── command.c │ ├── command.h │ ├── compile.py │ ├── compile_pathlib.py │ ├── config_params.py │ ├── cproject.py │ ├── custom_clean.py │ ├── custom_cmd.py │ ├── custom_loader.py │ ├── custom_reporter.py │ ├── custom_task_def.py │ ├── defs.h │ ├── delayed.py │ ├── delayed_creates.py │ ├── doit_config.py │ ├── download.py │ ├── empty_subtasks.py │ ├── executable.py │ ├── folder.py │ ├── get_var.py │ ├── getargs.py │ ├── getargs_dict.py │ ├── getargs_group.py │ ├── global_dep_manager.py │ ├── group.py │ ├── hello.py │ ├── import_tasks.py │ ├── initial_workdir.py │ ├── kbd.c │ ├── longrunning.py │ ├── main.c │ ├── meta.py │ ├── metadata.py │ ├── module_loader.py │ ├── my_dodo.py │ ├── my_module_with_tasks.py │ ├── my_tasks.py │ ├── parameters.py │ ├── parameters_inverse.py │ ├── pos.py │ ├── report_deps.py │ ├── run_once.py │ ├── sample.py │ ├── save_out.py │ ├── selecttasks.py │ ├── settrace.py │ ├── subtasks.py │ ├── tar.py │ ├── task_doc.py │ ├── task_kwargs.py │ ├── task_name.py │ ├── task_reusable.py │ ├── taskorder.py │ ├── taskresult.py │ ├── timeout.py │ ├── title.py │ ├── titlewithactions.py │ ├── touch.py │ ├── tsetup.py │ ├── tutorial_02.py │ ├── uptodate_callable.py │ └── verbosity.py ├── stories.rst ├── support.rst ├── svg │ ├── doit-sq.svg │ ├── doit-text-full.svg │ ├── doit-text-sq.svg │ ├── doit-text.svg │ └── doit.svg ├── task-args.rst ├── task-creation.rst ├── tasks.rst ├── tools.rst ├── tutorial-1.rst ├── tutorial │ ├── tuto_1_1.py │ ├── tuto_1_2.py │ ├── tuto_1_3.py │ └── tuto_1_4.py ├── uptodate.rst └── usecases.rst ├── doc_requirements.txt ├── dodo.py ├── doit ├── __init__.py ├── __main__.py ├── action.py ├── api.py ├── cmd_base.py ├── cmd_clean.py ├── cmd_completion.py ├── cmd_dumpdb.py ├── cmd_forget.py ├── cmd_help.py ├── cmd_ignore.py ├── cmd_info.py ├── cmd_list.py ├── cmd_resetdep.py ├── cmd_run.py ├── cmd_strace.py ├── cmdparse.py ├── control.py ├── dependency.py ├── doit_cmd.py ├── exceptions.py ├── globals.py ├── loader.py ├── plugin.py ├── reporter.py ├── runner.py ├── task.py ├── tools.py └── version.py ├── pylintrc ├── setup.cfg ├── setup.py ├── tests ├── Dockerfile ├── __init__.py ├── conftest.py ├── data │ └── README ├── loader_sample.py ├── module_with_tasks.py ├── myecho.py ├── pyproject.toml ├── sample.cfg ├── sample.toml ├── sample_md5.txt ├── sample_plugin.py ├── sample_process.py ├── test___init__.py ├── test___main__.py ├── test_action.py ├── test_api.py ├── test_cmd_base.py ├── test_cmd_clean.py ├── test_cmd_completion.py ├── test_cmd_dumpdb.py ├── test_cmd_forget.py ├── test_cmd_help.py ├── test_cmd_ignore.py ├── test_cmd_info.py ├── test_cmd_list.py ├── test_cmd_resetdep.py ├── test_cmd_run.py ├── test_cmd_strace.py ├── test_cmdparse.py ├── test_control.py ├── test_dependency.py ├── test_doit_cmd.py ├── test_exceptions.py ├── test_loader.py ├── test_plugin.py ├── test_reporter.py ├── test_runner.py ├── test_task.py └── test_tools.py └── zsh_completion_doit /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = doit, tests 3 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: schettino72 4 | patreon: # Replace with a single Patreon username 5 | open_collective: doit 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | custom: # Replace with a single custom sponsorship URL 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | PLEASE DO NOT USE ISSUE TRACKER TO ASK QUESTIONS: see [contributing](https://github.com/pydoit/doit/blob/master/CONTRIBUTING.md) 11 | 12 | **Describe the bug** 13 | 14 | Please include a minimal `dodo.py` that reproduces the problem. 15 | If relevant also include the command line used to invoke ``doit``. 16 | 17 | **Environment** 18 | 1. OS: 19 | 2. python version: 20 | 3. doit version: 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | PLEASE DO NOT USE ISSUE TRACKER TO ASK QUESTIONS: see [contributing](https://github.com/pydoit/doit/blob/master/CONTRIBUTING.md) 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | PLEASE DO NOT USE ISSUE TRACKER TO ASK QUESTIONS: see [contributing](https://github.com/pydoit/doit/blob/master/CONTRIBUTING.md) 11 | 12 | If your question has the form "How can I achieve X with ``doit``?" mostly probably you better start by asking questions on email list... 13 | 14 | A feature request **SHOULD** have one of: 15 | - a clear description of what must be changed in ``doit`` with an example ``dodo.py`` file of how the feature would work. 16 | - an example ``dodo.py`` showing a use-case, and ``doit`` limitation 17 | 18 | **Describe alternatives you've considered** 19 | A clear and concise description of any alternative solutions or features you've considered. 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Ask a question 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | PLEASE DO NOT USE ISSUE TRACKER TO ASK QUESTIONS: see [contributing](https://github.com/pydoit/doit/blob/master/CONTRIBUTING.md) 11 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [master, test] 6 | pull_request: 7 | branches: [master, test] 8 | 9 | jobs: 10 | test: 11 | runs-on: ${{ matrix.os }}-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | os: [ubuntu, windows, macos] 16 | python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.8', 'pypy-3.9'] 17 | exclude: 18 | - os: windows 19 | python-version: pypy3 20 | steps: 21 | - if: ${{ matrix.os == 'ubuntu' }} 22 | run: sudo apt-get install strace 23 | - uses: actions/checkout@v3 24 | - uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | # https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ 29 | - id: pip-cache 30 | shell: bash 31 | run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT 32 | 33 | - uses: actions/cache@v3 34 | with: 35 | path: ${{ steps.pip-cache.outputs.dir }} 36 | key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('setup.py', 'dev_requirements.txt') }} 37 | restore-keys: | 38 | ${{ runner.os }}-pip-${{ matrix.python-version }}- 39 | ${{ runner.os }}-pip- 40 | - run: pip install . -r dev_requirements.txt 41 | - run: pip freeze 42 | - run: pip check 43 | - run: doit pyflakes 44 | - run: doit codestyle 45 | - run: py.test -vv ${{ matrix.pytest-args }} 46 | - if: ${{ matrix.os == 'ubuntu' && matrix.python-version == '3.8' }} 47 | run: | 48 | pip install codecov 49 | doit coverage 50 | codecov 51 | -------------------------------------------------------------------------------- /.github/workflows/website.yml: -------------------------------------------------------------------------------- 1 | name: Website 2 | 3 | on: 4 | push: 5 | branches: [master, test] 6 | pull_request: 7 | branches: [master, test] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - run: sudo apt-get install hunspell hunspell-en-us 14 | - uses: actions/checkout@v3 15 | - uses: actions/setup-python@v4 16 | with: 17 | python-version: "3.8" 18 | - run: pip install . -r doc_requirements.txt -r dev_requirements.txt 19 | - run: pip freeze 20 | - run: doit -v2 website 21 | - uses: actions/upload-artifact@v3 22 | with: 23 | name: Website 24 | path: ./doc/_build 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .doit.db 3 | doit.egg-info 4 | .coverage 5 | .cache 6 | dist 7 | MANIFEST.in 8 | revision.txt 9 | 10 | tests/data/* 11 | doc/_build 12 | doc/samples/*.o 13 | doc/samples/*.in 14 | doc/samples/*.out 15 | doc/samples/file* 16 | 17 | dev 18 | .vscode 19 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | 2 | (in chronological order) 3 | 4 | * Eduardo Schettino - schettino72 gmail com 5 | * Javier Collado - https://launchpad.net/~javier.collado 6 | * Philipp Tölke - https://launchpad.net/~toelke+lp 7 | * Daniel Hjelm - doit-d hjelm eu 8 | * Damiro - https://launchpad.net/~damiro 9 | * Charlie Guo - https://launchpad.net/~charlie.guo 10 | * Michael Gliwinski - https://launchpad.net/~tzeentch-gm 11 | * Vadim Fint - mocksoul gmail com 12 | * Thomas Kluyver - https://bitbucket.org/takluyver 13 | * Rob Beagrie - http://rob.beagrie.com 14 | * Miguel Angel Garcia - http://magmax.org 15 | * Roland Puntaier - roland puntaier gmail com 16 | * Vincent Férotin - vincent ferotin gmail com 17 | * Chris Warrick - Kwpolska - http://chriswarrick.com/ 18 | * Ronan Le Gallic - rolegic gmail com 19 | * Simon Conseil - contact saimon org 20 | * Kostis Anagnostopoulos - ankostis gmail com 21 | * Randall Schwager - schwager hsph harvard edu 22 | * Pavel Platto - hinidu gmail com 23 | * Gerlad Storer - https://github.com/gstorer 24 | * Simon Mutch - https://github.com/smutch 25 | * Michael Milton - https://github.com/tmiguelt 26 | * Mike Pagel - https://github.com/moltob 27 | * Marijn van Vliet - https://github.com/wmvanvliet 28 | * Niko Wenselowski - https://github.com/okin 29 | * Jan Felix Langenbach - o hase3 gmail com 30 | * Facundo Ciccioli - facundofc gmail com 31 | * Alexandre Allard - https://github.com/alexandre-allard-scality 32 | * Trevor Bekolay - https://github.com/tbekolay 33 | * Michał Górny - https://github.com/mgorny 34 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | # Contributing to doit 3 | 4 | ## issues/bugs 5 | 6 | If you find issues using `doit` please report at 7 | [github issues](https://github.com/pydoit/doit/issues). 8 | 9 | All issues should contain a sample minimal `dodo.py` and 10 | the used command line to reproduce the problem. 11 | 12 | 13 | ## questions 14 | 15 | Please ask question in the discussion 16 | [forum](http://groups.google.co.in/group/python-doit) 17 | or on StackOverflow using tag `doit`. 18 | 19 | Do not use the github issue tracker to ask questions! 20 | 21 | `doit` has extensive online documentation please read the docs! 22 | 23 | When asking a question it is appreciated if you introduce yourself, 24 | also mention how long are you using doit and using it for what. 25 | 26 | A good question with a code example greatly increases the chance 27 | of it getting a reply. 28 | 29 | Unless you are looking for paid support, do **not** send private emails 30 | to the project maintainer. 31 | 32 | 33 | ## feature request 34 | 35 | Users are expected to implement new features themselves. 36 | You are welcome to add a request on github tracker but if you are not willing 37 | to spend your time on it, probably nobody else will... 38 | 39 | Before you start implementing anything it is a good idea to discuss its 40 | implementation in the discussion forum. 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | 3 | The MIT License 4 | 5 | Copyright (c) 2008-present Eduardo Naufel Schettino 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /TODO.txt: -------------------------------------------------------------------------------- 1 | 2 | see 3 | 4 | https://github.com/pydoit/doit/issues 5 | 6 | 0.X 7 | ---------- 8 | 9 | . setup/task single process/all procces 10 | . better terminal output (#5) 11 | 12 | wishlist 13 | ---------- 14 | 15 | . tools - profile 16 | . tools - code coverage 17 | 18 | . color output on the terminal 19 | . option dont save successful results 20 | . forget a dependency, not a task 21 | . task name alias 22 | . action to be executed on when ctrl-c is hit on auto mode 23 | 24 | 25 | 26 | big refactorings 27 | ------------------ 28 | 29 | . Task into TaskDep + Task 30 | 31 | -------------------------------------------------------------------------------- /bash_completion_doit: -------------------------------------------------------------------------------- 1 | # bash completion for doit 2 | # auto-generate by `doit tabcompletion` 3 | 4 | # to activate it you need to 'source' the generate script 5 | # $ source 6 | 7 | # reference => http://www.debian-administration.org/articles/317 8 | # patch => http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=711879 9 | 10 | _doit() 11 | { 12 | local cur prev words cword basetask sub_cmds tasks i dodof 13 | COMPREPLY=() # contains list of words with suitable completion 14 | # remove colon from word separator list because doit uses colon on task names 15 | _get_comp_words_by_ref -n : cur prev words cword 16 | # list of sub-commands 17 | sub_cmds="auto clean dumpdb forget help ignore info list reset-dep run strace tabcompletion" 18 | 19 | 20 | # options that take file/dir as values should complete file-system 21 | if [[ "$prev" == "-f" || "$prev" == "-d" || "$prev" == "-o" ]]; then 22 | _filedir 23 | return 0 24 | fi 25 | if [[ "$cur" == *=* ]]; then 26 | prev=${cur/=*/} 27 | cur=${cur/*=/} 28 | if [[ "$prev" == "--file=" || "$prev" == "--dir=" || "$prev" == "--output-file=" ]]; then 29 | _filedir -o nospace 30 | return 0 31 | fi 32 | fi 33 | 34 | 35 | # get name of the dodo file 36 | for (( i=0; i < ${#words[@]}; i++)); do 37 | case "${words[i]}" in 38 | -f) 39 | dodof=${words[i+1]} 40 | break 41 | ;; 42 | --file=*) 43 | dodof=${words[i]/*=/} 44 | break 45 | ;; 46 | esac 47 | done 48 | # dodo file not specified, use default 49 | if [ ! $dodof ] 50 | then 51 | dodof="dodo.py" 52 | fi 53 | 54 | 55 | # get task list 56 | # if it there is colon it is getting a subtask, complete only subtask names 57 | if [[ "$cur" == *:* ]]; then 58 | # extract base task name (remove everything after colon) 59 | basetask=${cur%:*} 60 | # sub-tasks 61 | tasks=$(doit list --file="$dodof" --quiet --all ${basetask} 2>/dev/null) 62 | COMPREPLY=( $(compgen -W "${tasks}" -- ${cur}) ) 63 | __ltrim_colon_completions "$cur" 64 | return 0 65 | # without colons get only top tasks 66 | else 67 | tasks=$(doit list --file="$dodof" --quiet 2>/dev/null) 68 | fi 69 | 70 | 71 | # match for first parameter must be sub-command or task 72 | # FIXME doit accepts options "-" in the first parameter but we ignore this case 73 | if [[ ${cword} == 1 ]] ; then 74 | COMPREPLY=( $(compgen -W "${sub_cmds} ${tasks}" -- ${cur}) ) 75 | return 0 76 | fi 77 | 78 | case ${words[1]} in 79 | 80 | auto) 81 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 82 | return 0 83 | ;; 84 | clean) 85 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 86 | return 0 87 | ;; 88 | dumpdb) 89 | COMPREPLY=( $(compgen -f -- $cur) ) 90 | return 0 91 | ;; 92 | forget) 93 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 94 | return 0 95 | ;; 96 | help) 97 | COMPREPLY=( $(compgen -W "${tasks} ${sub_cmds}" -- $cur) ) 98 | return 0 99 | ;; 100 | ignore) 101 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 102 | return 0 103 | ;; 104 | info) 105 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 106 | return 0 107 | ;; 108 | list) 109 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 110 | return 0 111 | ;; 112 | reset-dep) 113 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 114 | return 0 115 | ;; 116 | run) 117 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 118 | return 0 119 | ;; 120 | strace) 121 | COMPREPLY=( $(compgen -W "${tasks}" -- $cur) ) 122 | return 0 123 | ;; 124 | tabcompletion) 125 | COMPREPLY=( $(compgen -f -- $cur) ) 126 | return 0 127 | ;; 128 | esac 129 | 130 | # if there is already one parameter match only tasks (no commands) 131 | COMPREPLY=( $(compgen -W "${tasks}" -- ${cur}) ) 132 | 133 | } 134 | complete -o filenames -F _doit doit 135 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | # modules required for development only 2 | # $ pip install --requirement dev_requirements.txt 3 | 4 | setuptools # for plugins 5 | pyflakes 6 | pycodestyle 7 | pytest>=7.1.0 8 | coverage>=6.0 9 | doit-py>=0.4.0 10 | tomli; python_version<"3.11" 11 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = -v # debug 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = doit 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /doc/_static/doit-logo-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/doit-logo-small.png -------------------------------------------------------------------------------- /doc/_static/doit-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/doit-logo.png -------------------------------------------------------------------------------- /doc/_static/doit-text-160x60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/doit-text-160x60.png -------------------------------------------------------------------------------- /doc/_static/doit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/doit.png -------------------------------------------------------------------------------- /doc/_static/external.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/external.png -------------------------------------------------------------------------------- /doc/_static/favico.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/favico.ico -------------------------------------------------------------------------------- /doc/_static/pygarden.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/pygarden.png -------------------------------------------------------------------------------- /doc/_static/python-powered-w-100x40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/python-powered-w-100x40.png -------------------------------------------------------------------------------- /doc/_static/requests.models-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/requests.models-blue.png -------------------------------------------------------------------------------- /doc/_static/requests.models.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/requests.models.png -------------------------------------------------------------------------------- /doc/_static/requests.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/requests.png -------------------------------------------------------------------------------- /doc/_static/universal/custom.css: -------------------------------------------------------------------------------- 1 | /* your styles go here */ 2 | .pycode .hll { background-color: #ffffcc } 3 | .pycode { background: #f0f0f0; } 4 | .pycode .c { color: #60a0b0; font-style: italic } /* Comment */ 5 | .pycode .err { border: 1px solid #FF0000 } /* Error */ 6 | .pycode .k { color: #007020; font-weight: bold } /* Keyword */ 7 | .pycode .o { color: #666666 } /* Operator */ 8 | .pycode .ch { color: #60a0b0; font-style: italic } /* Comment.Hashbang */ 9 | .pycode .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */ 10 | .pycode .cp { color: #007020 } /* Comment.Preproc */ 11 | .pycode .cpf { color: #60a0b0; font-style: italic } /* Comment.PreprocFile */ 12 | .pycode .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */ 13 | .pycode .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */ 14 | .pycode .gd { color: #A00000 } /* Generic.Deleted */ 15 | .pycode .ge { font-style: italic } /* Generic.Emph */ 16 | .pycode .gr { color: #FF0000 } /* Generic.Error */ 17 | .pycode .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 18 | .pycode .gi { color: #00A000 } /* Generic.Inserted */ 19 | .pycode .go { color: #888888 } /* Generic.Output */ 20 | .pycode .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 21 | .pycode .gs { font-weight: bold } /* Generic.Strong */ 22 | .pycode .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 23 | .pycode .gt { color: #0044DD } /* Generic.Traceback */ 24 | .pycode .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 25 | .pycode .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 26 | .pycode .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 27 | .pycode .kp { color: #007020 } /* Keyword.Pseudo */ 28 | .pycode .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 29 | .pycode .kt { color: #902000 } /* Keyword.Type */ 30 | .pycode .m { color: #40a070 } /* Literal.Number */ 31 | .pycode .s { color: #4070a0 } /* Literal.String */ 32 | .pycode .na { color: #4070a0 } /* Name.Attribute */ 33 | .pycode .nb { color: #007020 } /* Name.Builtin */ 34 | .pycode .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 35 | .pycode .no { color: #60add5 } /* Name.Constant */ 36 | .pycode .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 37 | .pycode .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 38 | .pycode .ne { color: #007020 } /* Name.Exception */ 39 | .pycode .nf { color: #06287e } /* Name.Function */ 40 | .pycode .nl { color: #002070; font-weight: bold } /* Name.Label */ 41 | .pycode .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 42 | .pycode .nt { color: #062873; font-weight: bold } /* Name.Tag */ 43 | .pycode .nv { color: #bb60d5 } /* Name.Variable */ 44 | .pycode .ow { color: #007020; font-weight: bold } /* Operator.Word */ 45 | .pycode .w { color: #bbbbbb } /* Text.Whitespace */ 46 | .pycode .mb { color: #40a070 } /* Literal.Number.Bin */ 47 | .pycode .mf { color: #40a070 } /* Literal.Number.Float */ 48 | .pycode .mh { color: #40a070 } /* Literal.Number.Hex */ 49 | .pycode .mi { color: #40a070 } /* Literal.Number.Integer */ 50 | .pycode .mo { color: #40a070 } /* Literal.Number.Oct */ 51 | .pycode .sa { color: #4070a0 } /* Literal.String.Affix */ 52 | .pycode .sb { color: #4070a0 } /* Literal.String.Backtick */ 53 | .pycode .sc { color: #4070a0 } /* Literal.String.Char */ 54 | .pycode .dl { color: #4070a0 } /* Literal.String.Delimiter */ 55 | .pycode .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 56 | .pycode .s2 { color: #4070a0 } /* Literal.String.Double */ 57 | .pycode .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 58 | .pycode .sh { color: #4070a0 } /* Literal.String.Heredoc */ 59 | .pycode .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 60 | .pycode .sx { color: #c65d09 } /* Literal.String.Other */ 61 | .pycode .sr { color: #235388 } /* Literal.String.Regex */ 62 | .pycode .s1 { color: #4070a0 } /* Literal.String.Single */ 63 | .pycode .ss { color: #517918 } /* Literal.String.Symbol */ 64 | .pycode .bp { color: #007020 } /* Name.Builtin.Pseudo */ 65 | .pycode .fm { color: #06287e } /* Name.Function.Magic */ 66 | .pycode .vc { color: #bb60d5 } /* Name.Variable.Class */ 67 | .pycode .vg { color: #bb60d5 } /* Name.Variable.Global */ 68 | .pycode .vi { color: #bb60d5 } /* Name.Variable.Instance */ 69 | .pycode .vm { color: #bb60d5 } /* Name.Variable.Magic */ 70 | .pycode .il { color: #40a070 } /* Literal.Number.Integer.Long */ 71 | -------------------------------------------------------------------------------- /doc/_static/universal/texture-bw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/universal/texture-bw.png -------------------------------------------------------------------------------- /doc/_static/vendor/font-awesome/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/vendor/font-awesome/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /doc/_static/vendor/font-awesome/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/vendor/font-awesome/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /doc/_static/vendor/font-awesome/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/vendor/font-awesome/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /doc/_static/vendor/font-awesome/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/vendor/font-awesome/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /doc/_static/vendor/font-awesome/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/doc/_static/vendor/font-awesome/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /doc/_static/vendor/owl.carousel/owl.carousel.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Owl Carousel v2.2.0 3 | * Copyright 2013-2016 David Deutsch 4 | * Licensed under MIT (https://github.com/OwlCarousel2/OwlCarousel2/blob/master/LICENSE) 5 | */ 6 | /* 7 | * Owl Carousel - Core 8 | */ 9 | .owl-carousel { 10 | display: none; 11 | width: 100%; 12 | -webkit-tap-highlight-color: transparent; 13 | /* position relative and z-index fix webkit rendering fonts issue */ 14 | position: relative; 15 | z-index: 1; } 16 | .owl-carousel .owl-stage { 17 | position: relative; 18 | -ms-touch-action: pan-Y; } 19 | .owl-carousel .owl-stage:after { 20 | content: "."; 21 | display: block; 22 | clear: both; 23 | visibility: hidden; 24 | line-height: 0; 25 | height: 0; } 26 | .owl-carousel .owl-stage-outer { 27 | position: relative; 28 | overflow: hidden; 29 | /* fix for flashing background */ 30 | -webkit-transform: translate3d(0px, 0px, 0px); } 31 | .owl-carousel .owl-item { 32 | position: relative; 33 | min-height: 1px; 34 | float: left; 35 | -webkit-backface-visibility: hidden; 36 | -webkit-tap-highlight-color: transparent; 37 | -webkit-touch-callout: none; } 38 | .owl-carousel .owl-item img { 39 | display: block; 40 | width: 100%; 41 | -webkit-transform-style: preserve-3d; } 42 | .owl-carousel .owl-nav.disabled, 43 | .owl-carousel .owl-dots.disabled { 44 | display: none; } 45 | .owl-carousel .owl-nav .owl-prev, 46 | .owl-carousel .owl-nav .owl-next, 47 | .owl-carousel .owl-dot { 48 | cursor: pointer; 49 | cursor: hand; 50 | -webkit-user-select: none; 51 | -khtml-user-select: none; 52 | -moz-user-select: none; 53 | -ms-user-select: none; 54 | user-select: none; } 55 | .owl-carousel.owl-loaded { 56 | display: block; } 57 | .owl-carousel.owl-loading { 58 | opacity: 0; 59 | display: block; } 60 | .owl-carousel.owl-hidden { 61 | opacity: 0; } 62 | .owl-carousel.owl-refresh .owl-item { 63 | visibility: hidden; } 64 | .owl-carousel.owl-drag .owl-item { 65 | -webkit-user-select: none; 66 | -moz-user-select: none; 67 | -ms-user-select: none; 68 | user-select: none; } 69 | .owl-carousel.owl-grab { 70 | cursor: move; 71 | cursor: grab; } 72 | .owl-carousel.owl-rtl { 73 | direction: rtl; } 74 | .owl-carousel.owl-rtl .owl-item { 75 | float: right; } 76 | 77 | /* No Js */ 78 | .no-js .owl-carousel { 79 | display: block; } 80 | 81 | /* 82 | * Owl Carousel - Animate Plugin 83 | */ 84 | .owl-carousel .animated { 85 | -webkit-animation-duration: 1000ms; 86 | animation-duration: 1000ms; 87 | -webkit-animation-fill-mode: both; 88 | animation-fill-mode: both; } 89 | 90 | .owl-carousel .owl-animated-in { 91 | z-index: 0; } 92 | 93 | .owl-carousel .owl-animated-out { 94 | z-index: 1; } 95 | 96 | .owl-carousel .fadeOut { 97 | -webkit-animation-name: fadeOut; 98 | animation-name: fadeOut; } 99 | 100 | @-webkit-keyframes fadeOut { 101 | 0% { 102 | opacity: 1; } 103 | 100% { 104 | opacity: 0; } } 105 | 106 | @keyframes fadeOut { 107 | 0% { 108 | opacity: 1; } 109 | 100% { 110 | opacity: 0; } } 111 | 112 | /* 113 | * Owl Carousel - Auto Height Plugin 114 | */ 115 | .owl-height { 116 | transition: height 500ms ease-in-out; } 117 | 118 | /* 119 | * Owl Carousel - Lazy Load Plugin 120 | */ 121 | .owl-carousel .owl-item .owl-lazy { 122 | opacity: 0; 123 | transition: opacity 400ms ease; } 124 | 125 | .owl-carousel .owl-item img.owl-lazy { 126 | -webkit-transform-style: preserve-3d; 127 | transform-style: preserve-3d; } 128 | 129 | /* 130 | * Owl Carousel - Video Plugin 131 | */ 132 | .owl-carousel .owl-video-wrapper { 133 | position: relative; 134 | height: 100%; 135 | background: #000; } 136 | 137 | .owl-carousel .owl-video-play-icon { 138 | position: absolute; 139 | height: 80px; 140 | width: 80px; 141 | left: 50%; 142 | top: 50%; 143 | margin-left: -40px; 144 | margin-top: -40px; 145 | background: url("owl.video.play.png") no-repeat; 146 | cursor: pointer; 147 | z-index: 1; 148 | -webkit-backface-visibility: hidden; 149 | transition: -webkit-transform 100ms ease; 150 | transition: transform 100ms ease; } 151 | 152 | .owl-carousel .owl-video-play-icon:hover { 153 | -webkit-transform: scale(1.3, 1.3); 154 | -ms-transform: scale(1.3, 1.3); 155 | transform: scale(1.3, 1.3); } 156 | 157 | .owl-carousel .owl-video-playing .owl-video-tn, 158 | .owl-carousel .owl-video-playing .owl-video-play-icon { 159 | display: none; } 160 | 161 | .owl-carousel .owl-video-tn { 162 | opacity: 0; 163 | height: 100%; 164 | background-position: center center; 165 | background-repeat: no-repeat; 166 | background-size: contain; 167 | transition: opacity 400ms ease; } 168 | 169 | .owl-carousel .owl-video-frame { 170 | position: relative; 171 | z-index: 1; 172 | height: 100%; 173 | width: 100%; } 174 | -------------------------------------------------------------------------------- /doc/_static/vendor/owl.carousel/owl.theme.default.min.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Owl Carousel v2.2.0 3 | * Copyright 2013-2016 David Deutsch 4 | * Licensed under MIT (https://github.com/OwlCarousel2/OwlCarousel2/blob/master/LICENSE) 5 | */ 6 | .owl-theme .owl-dots,.owl-theme .owl-nav{text-align:center;-webkit-tap-highlight-color:transparent}.owl-theme .owl-nav{margin-top:10px}.owl-theme .owl-nav [class*=owl-]{color:#FFF;font-size:14px;margin:5px;padding:4px 7px;background:#D6D6D6;display:inline-block;cursor:pointer;border-radius:3px}.owl-theme .owl-nav [class*=owl-]:hover{background:#869791;color:#FFF;text-decoration:none}.owl-theme .owl-nav .disabled{opacity:.5;cursor:default}.owl-theme .owl-nav.disabled+.owl-dots{margin-top:10px}.owl-theme .owl-dots .owl-dot{display:inline-block;zoom:1}.owl-theme .owl-dots .owl-dot span{width:10px;height:10px;margin:5px 7px;background:#D6D6D6;display:block;-webkit-backface-visibility:visible;transition:opacity .2s ease;border-radius:30px}.owl-theme .owl-dots .owl-dot.active span,.owl-theme .owl-dots .owl-dot:hover span{background:#869791} -------------------------------------------------------------------------------- /doc/_templates/donate.html: -------------------------------------------------------------------------------- 1 | {% block extranav %} 2 | 3 | {% if include_donate %} 4 |
5 |

Donate

6 |

If you find doit useful, please consider supporting the maintainer with a donation.

7 |

doit is 100% open-source and maintained by individuals without any company backing it... Your donation keeps the project healthy and maintained.

8 |
9 | 10 |
11 |

OpenCollective

12 |
13 | 14 |
15 | 16 |
17 |

Paypal

18 |
19 | 20 | 21 | 22 | 23 |
24 |
25 |
26 | {% endif %} 27 | {% endblock %} 28 | -------------------------------------------------------------------------------- /doc/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | 3 | {% block footer_scripts %} 4 | {{ super() }} 5 | {% block analytics %} 6 | {% if include_analytics %} 7 | 16 | {% endif %} 17 | {% endblock %} 18 | {% endblock %} 19 | 20 | 21 | {# FIXME show version #} 22 | {#
#} 23 | {# {% if theme_display_version %} #} 24 | {# {%- set nav_version = version %} #} 25 | {# {% if nav_version %} #} 26 | {# {{ nav_version }} #} 27 | {# {% endif %} #} 28 | {# {% endif %} #} 29 | {#
#} 30 | -------------------------------------------------------------------------------- /doc/changes.rst: -------------------------------------------------------------------------------- 1 | ../CHANGES -------------------------------------------------------------------------------- /doc/configuration.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: Configuring pydoit with TOML and INI files 3 | :keywords: python, doit, documentation, guide, configuration, ini, toml 4 | 5 | .. title:: Configuration with TOML and INI files - pydoit guide 6 | 7 | 8 | Configuration 9 | ============= 10 | 11 | 12 | config param names 13 | ------------------ 14 | 15 | Note that the configuration option's name is not always the same as the 16 | *long* argument name used in the command line. 17 | 18 | I.e. To specify dodo file other than `dodo.py` from the command line 19 | you specify the option as ``-f`` or ``--file``, but from a config file 20 | it is called ``dodoFile``. 21 | 22 | The name can be seem from ``doit help`` output:: 23 | 24 | -f ARG, --file=ARG load task from dodo FILE [default: dodo.py] (config: dodoFile) 25 | 26 | 27 | pyproject.toml 28 | -------------- 29 | 30 | `doit` configuration can be read from `pyproject.toml `_ 31 | under the `tool.doit` namespace. 32 | This is the preferred configuration source, and may gain features not available in the legacy `doit.cfg`. 33 | 34 | .. note:: 35 | 36 | A TOML parser (`tomllib `_) 37 | is part of the standard library since Python 3.11. 38 | For earlier Python versions, a third-party package is required, one of: 39 | 40 | - `tomli `_ 41 | - `tomlkit `_ 42 | 43 | 44 | TOML vs INI 45 | ^^^^^^^^^^^ 46 | 47 | While mostly similar, `TOML `_ differs from the INI format 48 | in a few ways: 49 | 50 | - all strings must be quoted with `'` or `"` 51 | - triple-quoted strings may contain new line characters (`\n`) and quotes 52 | - must be saved as UTF-8 53 | - integers and floating point numbers can be written without quotes 54 | - boolean values can be written unquoted and lower-cased, as `true` and `false` 55 | 56 | Unlike "plain" TOML, `doit` will parse pythonic strings into their correct types, 57 | e.g. `"True"`, `"False"`, `"3"`, but using "native" TOML types may be preferable. 58 | 59 | 60 | tool.doit 61 | ^^^^^^^^^ 62 | 63 | The `tool.doit` section may contain command line options that will be used 64 | (if applicable) by any commands. 65 | 66 | Example setting the DB backend type: 67 | 68 | .. code-block:: toml 69 | 70 | [tool.doit] 71 | backend = "json" 72 | 73 | All commands that have a `backend` option (*run*, *clean*, *forget*, etc), 74 | will use this option without the need for this option in the command line. 75 | 76 | 77 | tool.doit.commands 78 | ^^^^^^^^^^^^^^^^^^ 79 | 80 | To configure options for a specific command, use a section with 81 | the command name under `tool.doit.commands`: 82 | 83 | .. code-block:: toml 84 | 85 | [tools.doit.commands.list] 86 | status = true 87 | subtasks = true 88 | 89 | 90 | tool.doit.plugins 91 | ^^^^^^^^^^^^^^^^^ 92 | 93 | Check the :ref:`plugins ` section for an introduction 94 | on available plugin categories. 95 | 96 | 97 | tool.doit.tasks 98 | ^^^^^^^^^^^^^^^ 99 | 100 | To configure options for a specific task, use a section with 101 | the task name under `tool.doit.tasks`: 102 | 103 | .. code-block:: toml 104 | 105 | [tool.doit.tasks.make_cookies] 106 | cookie_type = "chocolate" 107 | temp = "375F" 108 | duration = 12 109 | 110 | 111 | doit.cfg 112 | -------- 113 | 114 | `doit` also supports an INI style configuration file 115 | (see `configparser `_). 116 | Note: key/value entries can be separated only by the equal sign `=`. 117 | 118 | If a file name `doit.cfg` is present in the current working directory, 119 | it is processed. It supports 4 kind of sections: 120 | 121 | - a `GLOBAL` section 122 | - a section for each plugin category 123 | - a section for each command 124 | - a section for each task 125 | 126 | 127 | GLOBAL section 128 | ^^^^^^^^^^^^^^ 129 | 130 | The `GLOBAL` section may contain command line options that will 131 | be used (if applicable) by any commands. 132 | 133 | Example setting the DB backend type: 134 | 135 | .. code-block:: ini 136 | 137 | [GLOBAL] 138 | backend = json 139 | 140 | All commands that have a `backend` option (*run*, *clean*, *forget*, etc), 141 | will use this option without the need for this option in the command line. 142 | 143 | 144 | commands section 145 | ^^^^^^^^^^^^^^^^ 146 | 147 | To configure options for a specific command, use a section with 148 | the command name: 149 | 150 | .. code-block:: ini 151 | 152 | [list] 153 | status = True 154 | subtasks = True 155 | 156 | 157 | plugins sections 158 | ^^^^^^^^^^^^^^^^ 159 | 160 | Check the :ref:`plugins ` section for an introduction 161 | on available plugin categories. 162 | 163 | 164 | per-task sections 165 | ^^^^^^^^^^^^^^^^^ 166 | 167 | To configure options for a specific task, use a section with 168 | the task name prefixed with "task:": 169 | 170 | .. code-block:: ini 171 | 172 | [task:make_cookies] 173 | cookie_type = chocolate 174 | temp = 375F 175 | duration = 12 176 | 177 | 178 | configuration at *dodo.py* 179 | -------------------------- 180 | 181 | As a convenience you can also set `GLOBAL` options directly into a `dodo.py`. 182 | Just put the option in the `DOIT_CONFIG` dict. 183 | This example below sets the default tasks to be run, the ``continue`` option, 184 | and a different reporter. 185 | 186 | .. literalinclude:: samples/doit_config.py 187 | 188 | So if you just execute 189 | 190 | .. code-block:: console 191 | 192 | $ doit 193 | 194 | it will have the same effect as executing 195 | 196 | .. code-block:: console 197 | 198 | $ doit --continue --reporter json my_task_1 my_task_2 199 | 200 | 201 | .. note:: 202 | 203 | Not all options can be set on `dodo.py` file. 204 | The parameters ``--file`` and ``--dir`` can not be used on config because 205 | they control how the *dodo* file itself is loaded. 206 | 207 | Also if the command does not read the `dodo.py` file it obviously will 208 | not be used. 209 | -------------------------------------------------------------------------------- /doc/contents.rst: -------------------------------------------------------------------------------- 1 | .. doit documentation master file, created by 2 | sphinx-quickstart on Wed Apr 2 22:40:41 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. meta:: 7 | :description: Index of user documentation for pydoit - python task-runner & build-tool 8 | :keywords: python, doit, tutorial, documentation, task-runner, guide, how-to, getting started 9 | 10 | .. title:: documentation index | pydoit - python task-runner 11 | 12 | 13 | `doit` documentation 14 | ==================== 15 | 16 | Getting Started 17 | --------------- 18 | 19 | Introduction of `doit` basic features with a real example. 20 | 21 | .. toctree:: 22 | :maxdepth: 1 23 | 24 | usecases 25 | tutorial-1 26 | 27 | 28 | 29 | .. _main-ref-toc: 30 | 31 | Guide 32 | ----- 33 | 34 | Introduces the concepts of *every* feature one by one with examples. 35 | It was written to be both read in order and also serves as a complete reference. 36 | 37 | The total reading time for the whole documentation is about one hour. 38 | 39 | .. toctree:: 40 | :maxdepth: 2 41 | 42 | install 43 | tasks 44 | dependencies 45 | task-creation 46 | cmd-run 47 | cmd-other 48 | configuration 49 | task-args 50 | globals 51 | uptodate 52 | tools 53 | extending 54 | 55 | 56 | Project 57 | ------- 58 | 59 | .. toctree:: 60 | :maxdepth: 2 61 | 62 | support 63 | changes 64 | stories 65 | faq 66 | related 67 | 68 | 69 | 70 | .. Indices and tables 71 | ================== 72 | * :ref:`genindex` 73 | * :ref:`modindex` 74 | * :ref:`search` 75 | -------------------------------------------------------------------------------- /doc/faq.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: Frequently Asked Questions on doit design and usage 3 | :keywords: python, doit, faq 4 | 5 | .. title:: FAQ - Frequently Asked Questions on doit design and usage 6 | 7 | 8 | ======= 9 | FAQ 10 | ======= 11 | 12 | 13 | Why is `doit` written in all lowercase instead of CamelCase? 14 | ------------------------------------------------------------- 15 | 16 | At first it would be written in CamelCase `DoIt` but depending on the font 17 | some people would read it as `dolt `_ 18 | with an `L` instead of `I`. So I just set it as lowercase to avoid confusion. 19 | 20 | 21 | *doit* is too verbose, why don't you use decorators? 22 | ----------------------------------------------------- 23 | 24 | `doit` is designed to be extensible. 25 | A simple dictionary is actually the most flexible representation. 26 | It is possible to create different interfaces on top of it. 27 | Check this `blog post `_ 28 | for some examples. 29 | 30 | 31 | `dodo.py` file itself should be a `file_dep` for all tasks 32 | ----------------------------------------------------------- 33 | 34 | If I edit my `dodo.py` file and re-run *doit*, 35 | and my tasks are otherwise up-to-date, the modified tasks are not re-run. 36 | 37 | While developing your tasks it is recommended 38 | to use ``doit forget`` after you change your tasks 39 | or use ``doit --always-run``. 40 | 41 | In case you really want, you will need to explicitly 42 | add the `dodo.py` in `file_dep` of your tasks manually. 43 | 44 | If `dodo.py` was an implicit `file_dep`: 45 | 46 | * how would you disable it? 47 | * should imported files from your `dodo.py` also be a `file_dep`? 48 | 49 | 50 | Why `file_dep` can not depend on a directory/folder? 51 | ------------------------------------------------------ 52 | 53 | A `file_dep` is considered to not be up-to-date when the content of 54 | the file changes. But what is a folder change? 55 | Some people expect it to be a change in any of its containing files 56 | (for this case see question below). 57 | Others expect it to be whether the folder exist or not, 58 | or if a new file was added or removed from the folder (for these 59 | cases you should implement a custom ``uptodate`` 60 | (:ref:`check the API`). 61 | 62 | 63 | How to make a dependency on all files in a folder? 64 | ---------------------------------------------------- 65 | 66 | ``file_dep`` does NOT support folders. 67 | If you want to specify all files from a folder you can use a third 68 | party library like `pathlib `_ ( 69 | `pathlib` was add on python's 3.4 stdlib). 70 | -------------------------------------------------------------------------------- /doc/globals.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: Accessing doit internal database 3 | :keywords: python, doit, documentation, guide, database, status 4 | 5 | .. title:: Accessing doit internal database - pydoit guide 6 | 7 | 8 | ================================== 9 | Globals - accessing doit internals 10 | ================================== 11 | 12 | During the life-cycle of a command invocation, 13 | some properties of `doit` are stored in global singletons, 14 | provided by the ``doit.Globals`` class. 15 | 16 | .. autoclass:: doit.globals.Globals 17 | :members: 18 | 19 | dep_manager 20 | ----------- 21 | 22 | The doit dependency manager holds the persistent state of `doit`. 23 | This includes relations of tasks among each other or results, 24 | returned from their latest runs. It basically consists of all the 25 | information stored in `doit`'s database file. 26 | 27 | The ``dep_manager`` attribute is initialized right before tasks are loaded, 28 | which means it allows to be accessed during *all* task evaluation phases, 29 | in particular during: 30 | 31 | * Task creation, i.e. from the body of any ``task_*`` function. 32 | * Task execution, i.e. from the code executed by one of the tasks' actions. 33 | * Task cleanup, i.e. from the the code executed by one of the tasks' clean activities. 34 | 35 | The `Dependency` class has members to access persistent doit data via its API: 36 | 37 | .. autoclass:: doit.dependency.Dependency 38 | :members: get_values, get_value, get_result 39 | 40 | The class internally interacts with a data base backend 41 | which may be accessed via the `backend` attribute. 42 | An experienced user may also *modify* persistently stored *doit* data through that attribute. 43 | As an example of a backend API, 44 | look at the common methods exposed by the default `DbmDB` backend implementation: 45 | 46 | .. class:: doit.dependency.DbmDB 47 | 48 | A simple Key-Value database interface 49 | 50 | .. automethod:: doit.dependency.DbmDB.get 51 | .. automethod:: doit.dependency.DbmDB.set 52 | .. automethod:: doit.dependency.DbmDB.remove 53 | 54 | There are other backends available in *doit*, 55 | see the documentation on :ref:`db_backends` on how to select between them. 56 | 57 | 58 | ``dep_manager`` example 59 | +++++++++++++++++++++++ 60 | 61 | An example of using the exposed dependency manager is a task, 62 | where at creation time the *target* of the task action is not yet known, 63 | because it is determined during execution. 64 | Then it would be possible to store that target in the dependency manager 65 | by returning it from the action. 66 | A `clean` action is subsequently able to query `dep_manager` for that result 67 | and perform the cleanup action: 68 | 69 | 70 | .. literalinclude:: samples/global_dep_manager.py 71 | -------------------------------------------------------------------------------- /doc/google726fc03ab55ebbfc.html: -------------------------------------------------------------------------------- 1 | google-site-verification: google726fc03ab55ebbfc.html -------------------------------------------------------------------------------- /doc/install.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: How-to install pydoit using pip, from source or OS package 3 | :keywords: python, doit, install, git, github, ubuntu, linux, windows 4 | 5 | .. title:: pydoit package installation options 6 | 7 | ========== 8 | Installing 9 | ========== 10 | 11 | 12 | pip 13 | ^^^ 14 | 15 | `package `_:: 16 | 17 | $ pip install doit 18 | 19 | Latest version of `doit` supports only python 3. 20 | If you are using python 2:: 21 | 22 | $ pip install "doit<0.30" 23 | 24 | 25 | 26 | Source 27 | ^^^^^^ 28 | 29 | Download `source `_:: 30 | 31 | $ pip install -e . 32 | 33 | 34 | git repository 35 | ^^^^^^^^^^^^^^ 36 | 37 | Get latest development version:: 38 | 39 | $ git clone https://github.com/pydoit/doit.git 40 | 41 | 42 | OS package 43 | ^^^^^^^^^^ 44 | 45 | Several distributions include native `doit` packages. 46 | `Repology.org `_ 47 | provides up-to-date information about available packages and 48 | `doit` versions on each distribution. 49 | 50 | Anaconda 51 | ^^^^^^^^ 52 | 53 | `doit` is also packaged on `Anaconda `_. 54 | Note this is not an official package and might be outdated. 55 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=doit 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /doc/open_collective.md: -------------------------------------------------------------------------------- 1 | the tool for **stateful processing of your interdependent tasks** powered by: 2 | 3 | - nuclear power-plant within your **shell** 4 | - **python** batteries 5 | - **doit** processing engine 6 | 7 | doit is mature project started in 2008 by @schettino72 and maintained by him up to date. It already serves years in numerous projects to: 8 | 9 | - simplify cumbersome command line calls, 10 | - automate complex data processing or typical project related actions, 11 | - share unified way of doing things, 12 | - optimize processing time by skipping things already done. 13 | 14 | People often compare `doit` to tools like `make`, `grunt` or `gulp` but they always appreciate 15 | 16 | - strong features and flexibility 17 | - simplicity of authoring and ease of use 18 | - python 19 | 20 | # Current focus 21 | 22 | ## Maintain existing code-base 23 | The aim is to **keep the product in shape and usable**. 24 | 25 | Abandoned open-source project does not work for long. Thriving project requires a maintainer to keep list of issues and list of pull requests short. 26 | 27 | Financial goal is 500 USD per month to allow the maintainer working few hours a week on the project. 28 | 29 | Main capacity shall be provided by @schettino72 30 | 31 | 32 | # Long term vision 33 | 34 | ## B: Rewrite documentation 35 | The aim is to **lower initial learning barrier for newcomers to get them on board** and to **help pro-users to unlock more features and earn fame**. 36 | 37 | Existing documentation is good as it served well existing users. But we can serve better. 38 | Inspired by great talk [What nobody tells you about documentation](https://www.youtube.com/watch?v=t4vKPhjcMZg&t=4s) by Daniele Procida we plan to rewrite the documentation into following parts: 39 | 40 | - Introduction (basic features overview) 41 | - Quick start and tutorials 42 | - Reference documentation 43 | - Concepts 44 | - How-to's 45 | 46 | 47 | ## C: Promote 48 | We believe, many more users deserve `doit` and we shall help them to know about it. 49 | 50 | Promotion may have form of: 51 | 52 | - helping selected python projects to adopt doit as internal tool 53 | - video presentations 54 | - presentation(s) at pycon(s) 55 | 56 | 57 | ## D: Launch doit task libraries 58 | 59 | Grunt library has over 6000 tasks, gulp has over 3000 plug-ins. 60 | 61 | `doit` has similar potential. E.g. [doit-py](https://github.com/pydoit/doit-py) covers python code specific tasks. Few lines of code in [dodo.py](https://github.com/pydoit/doit-py/blob/master/dodo.py) to lint the code; run the tests and measure coverage; build and upload package; spell, build and publish sphinx based documentation. 62 | 63 | 64 | # Why to contribute? 65 | 66 | Because: 67 | 68 | - You can already use existing power of the tool. 69 | - The [github doit repository](https://github.com/pydoit/doit) shows stable, long term activity with small but existing community of secondary contributors. 70 | - You can see, we have clear plan and priorities. 71 | - You can improve your productivity with maintained tool and streamlined documentation. 72 | - You can doit. 73 | -------------------------------------------------------------------------------- /doc/related.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: Related Projects - task runner, build tool & pipelines 3 | :keywords: python, doit, task-runner, build-tool, pipeline, workflow 4 | 5 | .. title:: Related Projects - task runner, build tool & pipelines 6 | 7 | ================ 8 | Related Projects 9 | ================ 10 | 11 | These are the main build tools in use today. 12 | 13 | - `make `_ 14 | - `ant `_ 15 | - `SCons `_ 16 | - `Rake `_ 17 | 18 | There are `many `_ more... 19 | 20 | In this `post `_ I briefly explained my motivation to start another build tool like project. 21 | 22 | -------------------------------------------------------------------------------- /doc/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | 3 | Disallow: /_modules/ 4 | Disallow: /_sources/ 5 | 6 | Sitemap: https://pydoit.org/sitemap.xml 7 | -------------------------------------------------------------------------------- /doc/samples/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | edit 3 | foo 4 | foo.txt 5 | hello.txt 6 | my_input.txt 7 | poo 8 | report.txt 9 | task1 10 | -------------------------------------------------------------------------------- /doc/samples/calc_dep.py: -------------------------------------------------------------------------------- 1 | DOIT_CONFIG = {'verbosity': 2} 2 | 3 | MOD_IMPORTS = {'a': ['b','c'], 4 | 'b': ['f','g'], 5 | 'c': [], 6 | 'f': ['a'], 7 | 'g': []} 8 | 9 | 10 | def print_deps(mod, dependencies): 11 | print("%s -> %s" % (mod, dependencies)) 12 | def task_mod_deps(): 13 | """task that depends on all direct imports""" 14 | for mod in MOD_IMPORTS.keys(): 15 | yield {'name': mod, 16 | 'actions': [(print_deps,(mod,))], 17 | 'file_dep': [mod], 18 | 'calc_dep': ["get_dep:%s" % mod], 19 | } 20 | 21 | def get_dep(mod): 22 | # fake implementation 23 | return {'file_dep': MOD_IMPORTS[mod]} 24 | def task_get_dep(): 25 | """get direct dependencies for each module""" 26 | for mod in MOD_IMPORTS.keys(): 27 | yield {'name': mod, 28 | 'actions':[(get_dep,[mod])], 29 | 'file_dep': [mod], 30 | } 31 | -------------------------------------------------------------------------------- /doc/samples/check_timestamp_unchanged.py: -------------------------------------------------------------------------------- 1 | from doit.tools import check_timestamp_unchanged 2 | 3 | def task_create_foo(): 4 | return { 5 | 'actions': ['touch foo', 'chmod 750 foo'], 6 | 'targets': ['foo'], 7 | 'uptodate': [True], 8 | } 9 | 10 | def task_on_foo_changed(): 11 | # will execute if foo or its metadata is modified 12 | return { 13 | 'actions': ['echo foo modified'], 14 | 'task_dep': ['create_foo'], 15 | 'uptodate': [check_timestamp_unchanged('foo', 'ctime')], 16 | } 17 | -------------------------------------------------------------------------------- /doc/samples/checker.py: -------------------------------------------------------------------------------- 1 | def task_checker(): 2 | return {'actions': ["pyflakes sample.py"], 3 | 'file_dep': ["sample.py"]} 4 | -------------------------------------------------------------------------------- /doc/samples/clean_mix.py: -------------------------------------------------------------------------------- 1 | from doit.task import clean_targets 2 | 3 | def simple(): 4 | print("ok") 5 | 6 | def task_poo(): 7 | return { 8 | 'actions': ['touch poo'], 9 | 'targets': ['poo'], 10 | 'clean': [clean_targets, simple], 11 | } 12 | -------------------------------------------------------------------------------- /doc/samples/cmd_actions.py: -------------------------------------------------------------------------------- 1 | def task_hello(): 2 | """hello cmd """ 3 | msg = 3 * "hi! " 4 | return { 5 | 'actions': ['echo %s ' % msg], 6 | } 7 | 8 | -------------------------------------------------------------------------------- /doc/samples/cmd_actions_list.py: -------------------------------------------------------------------------------- 1 | def task_python_version(): 2 | return { 3 | 'actions': [['python', '--version']] 4 | } 5 | 6 | -------------------------------------------------------------------------------- /doc/samples/cmd_from_callable.py: -------------------------------------------------------------------------------- 1 | from doit.action import CmdAction 2 | 3 | def task_hello(): 4 | """hello cmd """ 5 | 6 | def create_cmd_string(): 7 | return "echo hi" 8 | 9 | return { 10 | 'actions': [CmdAction(create_cmd_string)], 11 | 'verbosity': 2, 12 | } 13 | -------------------------------------------------------------------------------- /doc/samples/command.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void command(int a){ 4 | printf("geez"); 5 | }; 6 | -------------------------------------------------------------------------------- /doc/samples/command.h: -------------------------------------------------------------------------------- 1 | void command(int a); 2 | -------------------------------------------------------------------------------- /doc/samples/compile.py: -------------------------------------------------------------------------------- 1 | def task_compile(): 2 | return {'actions': ["cc -c main.c"], 3 | 'file_dep': ["main.c", "defs.h"], 4 | 'targets': ["main.o"] 5 | } 6 | -------------------------------------------------------------------------------- /doc/samples/compile_pathlib.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | def task_compile(): 4 | working_directory = Path('.') 5 | # Path.glob returns an iterator so turn it into a list 6 | headers = list(working_directory.glob('*.h')) 7 | for source_file in working_directory.glob('*.c'): 8 | object_file = source_file.with_suffix('.o') 9 | yield { 10 | 'name': object_file.name, 11 | 'actions': [['cc', '-c', source_file]], 12 | 'file_dep': [source_file] + headers, 13 | 'targets': [object_file], 14 | } 15 | -------------------------------------------------------------------------------- /doc/samples/config_params.py: -------------------------------------------------------------------------------- 1 | from doit.tools import config_changed 2 | 3 | option = "AB" 4 | def task_with_params(): 5 | return {'actions': ['echo %s' % option], 6 | 'uptodate': [config_changed(option)], 7 | 'verbosity': 2, 8 | } 9 | -------------------------------------------------------------------------------- /doc/samples/cproject.py: -------------------------------------------------------------------------------- 1 | DOIT_CONFIG = {'default_tasks': ['link']} 2 | 3 | # map source file to dependencies 4 | SOURCE = { 5 | 'main': ["defs.h"], 6 | 'kbd': ["defs.h", "command.h"], 7 | 'command': ["defs.h", "command.h"], 8 | } 9 | 10 | def task_link(): 11 | "create binary program" 12 | OBJECTS = ["%s.o" % module for module in SOURCE.keys()] 13 | return {'actions': ['cc -o %(targets)s %(dependencies)s'], 14 | 'file_dep': OBJECTS, 15 | 'targets': ['edit'], 16 | 'clean': True 17 | } 18 | 19 | def task_compile(): 20 | "compile C files" 21 | for module, dep in SOURCE.items(): 22 | dependencies = dep + ['%s.c' % module] 23 | yield {'name': module, 24 | 'actions': ["cc -c %s.c" % module], 25 | 'targets': ["%s.o" % module], 26 | 'file_dep': dependencies, 27 | 'clean': True 28 | } 29 | 30 | def task_install(): 31 | "install" 32 | return {'actions': ['echo install comes here...'], 33 | 'task_dep': ['link'], 34 | 'doc': 'install executable (TODO)' 35 | } 36 | -------------------------------------------------------------------------------- /doc/samples/custom_clean.py: -------------------------------------------------------------------------------- 1 | def my_cleaner(dryrun): 2 | if dryrun: 3 | print('dryrun, dont really execute') 4 | return 5 | print('execute cleaner...') 6 | 7 | def task_sample(): 8 | return { 9 | "actions" : None, 10 | "clean" : [my_cleaner], 11 | } 12 | 13 | -------------------------------------------------------------------------------- /doc/samples/custom_cmd.py: -------------------------------------------------------------------------------- 1 | from doit.cmd_base import Command 2 | 3 | 4 | class Init(Command): 5 | doc_purpose = 'create a project scaffolding' 6 | doc_usage = '' 7 | doc_description = """This is a multiline command description. 8 | It will be displayed on `doit help init`""" 9 | 10 | def execute(self, opt_values, pos_args): 11 | print("TODO: create some files for my project") 12 | -------------------------------------------------------------------------------- /doc/samples/custom_loader.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import sys 4 | 5 | from doit.task import dict_to_task 6 | from doit.cmd_base import TaskLoader2 7 | from doit.doit_cmd import DoitMain 8 | 9 | my_builtin_task = { 10 | 'name': 'sample_task', 11 | 'actions': ['echo hello from built in'], 12 | 'doc': 'sample doc', 13 | } 14 | 15 | 16 | class MyLoader(TaskLoader2): 17 | def setup(self, opt_values): 18 | pass 19 | 20 | def load_doit_config(self): 21 | return {'verbosity': 2} 22 | 23 | def load_tasks(self, cmd, pos_args): 24 | task_list = [dict_to_task(my_builtin_task)] 25 | return task_list 26 | 27 | 28 | if __name__ == "__main__": 29 | sys.exit(DoitMain(MyLoader()).run(sys.argv[1:])) 30 | -------------------------------------------------------------------------------- /doc/samples/custom_reporter.py: -------------------------------------------------------------------------------- 1 | from doit.reporter import ConsoleReporter 2 | 3 | class MyReporter(ConsoleReporter): 4 | def execute_task(self, task): 5 | self.outstream.write('MyReporter --> %s\n' % task.title()) 6 | 7 | DOIT_CONFIG = {'reporter': MyReporter, 8 | 'verbosity': 2} 9 | 10 | def task_sample(): 11 | for x in range(3): 12 | yield {'name': str(x), 13 | 'actions': ['echo out %d' % x]} 14 | -------------------------------------------------------------------------------- /doc/samples/custom_task_def.py: -------------------------------------------------------------------------------- 1 | def make_task(func): 2 | """make decorated function a task-creator""" 3 | func.create_doit_tasks = func 4 | return func 5 | 6 | @make_task 7 | def sample(): 8 | return { 9 | 'verbosity': 2, 10 | 'actions': ['echo hi'], 11 | } 12 | -------------------------------------------------------------------------------- /doc/samples/defs.h: -------------------------------------------------------------------------------- 1 | 2 | static int SIZE = 20; 3 | 4 | -------------------------------------------------------------------------------- /doc/samples/delayed.py: -------------------------------------------------------------------------------- 1 | import glob 2 | 3 | from doit import create_after 4 | 5 | 6 | @create_after(executed='early', target_regex='.*\.out') 7 | def task_build(): 8 | for inf in glob.glob('*.in'): 9 | yield { 10 | 'name': inf, 11 | 'actions': ['cp %(dependencies)s %(targets)s'], 12 | 'file_dep': [inf], 13 | 'targets': [inf[:-3] + '.out'], 14 | 'clean': True, 15 | } 16 | 17 | def task_early(): 18 | """a task that create some files...""" 19 | inter_files = ('a.in', 'b.in', 'c.in') 20 | return { 21 | 'actions': ['touch %(targets)s'], 22 | 'targets': inter_files, 23 | 'clean': True, 24 | } 25 | -------------------------------------------------------------------------------- /doc/samples/delayed_creates.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from doit import create_after 4 | 5 | def say_hello(your_name): 6 | sys.stderr.write("Hello from {}!\n".format(your_name)) 7 | 8 | def task_a(): 9 | return { 10 | "actions": [ (say_hello, ["a"]) ] 11 | } 12 | 13 | @create_after("a", creates=['b']) 14 | def task_another_task(): 15 | return { 16 | "basename": "b", 17 | "actions": [ (say_hello, ["b"]) ], 18 | } 19 | -------------------------------------------------------------------------------- /doc/samples/doit_config.py: -------------------------------------------------------------------------------- 1 | DOIT_CONFIG = {'default_tasks': ['my_task_1', 'my_task_2'], 2 | 'continue': True, 3 | 'reporter': 'json'} 4 | -------------------------------------------------------------------------------- /doc/samples/download.py: -------------------------------------------------------------------------------- 1 | from doit.tools import run_once 2 | 3 | def task_get_pylogo(): 4 | url = "http://python.org/images/python-logo.gif" 5 | return {'actions': ["wget %s" % url], 6 | 'targets': ["python-logo.gif"], 7 | 'uptodate': [run_once], 8 | } 9 | -------------------------------------------------------------------------------- /doc/samples/empty_subtasks.py: -------------------------------------------------------------------------------- 1 | import glob 2 | 3 | def task_xxx(): 4 | """my doc""" 5 | LIST = glob.glob('*.xyz') # might be empty 6 | yield { 7 | 'basename': 'do_x', 8 | 'name': None, 9 | 'doc': 'docs for X', 10 | 'watch': ['.'], 11 | } 12 | for item in LIST: 13 | yield { 14 | 'basename': 'do_x', 15 | 'name': item, 16 | 'actions': ['echo %s' % item], 17 | 'verbosity': 2, 18 | } 19 | -------------------------------------------------------------------------------- /doc/samples/executable.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | def task_echo(): 4 | return { 5 | 'actions': ['echo hi'], 6 | 'verbosity': 2, 7 | } 8 | 9 | if __name__ == '__main__': 10 | import doit 11 | doit.run(globals()) 12 | -------------------------------------------------------------------------------- /doc/samples/folder.py: -------------------------------------------------------------------------------- 1 | from doit.tools import create_folder 2 | 3 | BUILD_PATH = "_build" 4 | 5 | def task_build(): 6 | return {'actions': [(create_folder, [BUILD_PATH]), 7 | 'touch %(targets)s'], 8 | 'targets': ["%s/file.o" % BUILD_PATH] 9 | } 10 | -------------------------------------------------------------------------------- /doc/samples/get_var.py: -------------------------------------------------------------------------------- 1 | from doit import get_var 2 | 3 | config = {"abc": get_var('abc', 'NO')} 4 | 5 | def task_echo(): 6 | return {'actions': ['echo hi %s' % config], 7 | 'verbosity': 2, 8 | } 9 | -------------------------------------------------------------------------------- /doc/samples/getargs.py: -------------------------------------------------------------------------------- 1 | DOIT_CONFIG = { 2 | 'default_tasks': ['use_cmd', 'use_python'], 3 | 'action_string_formatting': 'both', 4 | } 5 | 6 | def task_compute(): 7 | def comp(): 8 | return {'x':5,'y':10, 'z': 20} 9 | return {'actions': [(comp,)]} 10 | 11 | 12 | def task_use_cmd(): 13 | return {'actions': ['echo x={x}', # new-style formatting 14 | 'echo z=%(z)s'], # old-style formatting 15 | 'getargs': {'x': ('compute', 'x'), 16 | 'z': ('compute', 'z')}, 17 | 'verbosity': 2, 18 | } 19 | 20 | 21 | def task_use_python(): 22 | return {'actions': [show_getargs], 23 | 'getargs': {'x': ('compute', 'x'), 24 | 'y': ('compute', 'z')}, 25 | 'verbosity': 2, 26 | } 27 | def show_getargs(x, y): 28 | print("this is x: {}".format(x)) 29 | print("this is y: {}".format(y)) 30 | -------------------------------------------------------------------------------- /doc/samples/getargs_dict.py: -------------------------------------------------------------------------------- 1 | def task_compute(): 2 | def comp(): 3 | return {'x':5,'y':10, 'z': 20} 4 | return {'actions': [(comp,)]} 5 | 6 | 7 | def show_getargs(values): 8 | print(values) 9 | 10 | def task_args_dict(): 11 | return {'actions': [show_getargs], 12 | 'getargs': {'values': ('compute', None)}, 13 | 'verbosity': 2, 14 | } 15 | -------------------------------------------------------------------------------- /doc/samples/getargs_group.py: -------------------------------------------------------------------------------- 1 | def task_compute(): 2 | def comp(x): 3 | return {'x':x} 4 | yield {'name': '5', 5 | 'actions': [ (comp, [5]) ] 6 | } 7 | yield {'name': '7', 8 | 'actions': [ (comp, [7]) ] 9 | } 10 | 11 | 12 | def show_getargs(values): 13 | print(values) 14 | assert sum(v['x'] for v in values.values()) == 12 15 | 16 | def task_args_dict(): 17 | return {'actions': [show_getargs], 18 | 'getargs': {'values': ('compute', None)}, 19 | 'verbosity': 2, 20 | } 21 | -------------------------------------------------------------------------------- /doc/samples/global_dep_manager.py: -------------------------------------------------------------------------------- 1 | import doit 2 | 3 | DOIT_CONFIG = dict( 4 | verbosity=2, 5 | ) 6 | 7 | 8 | def task_create(): 9 | # dependency manager is defined for all code inside the generator: 10 | dep_manager = doit.Globals.dep_manager 11 | 12 | def action(): 13 | # assume some involved logic to define ident: 14 | ident = 42 15 | print('Created', ident) 16 | 17 | # store for clean: 18 | return dict(created=ident) 19 | 20 | def clean(task): 21 | result = dep_manager.get_result(task.name) 22 | if result: 23 | ident = result['created'] 24 | print('Deleted', ident) 25 | 26 | # possibly forget the task, after it was cleaned: 27 | dep_manager.remove(task.name) 28 | 29 | return dict( 30 | actions=[action], 31 | clean=[clean], 32 | ) 33 | -------------------------------------------------------------------------------- /doc/samples/group.py: -------------------------------------------------------------------------------- 1 | def task_foo(): 2 | return {'actions': ["echo foo"]} 3 | 4 | def task_bar(): 5 | return {'actions': ["echo bar"]} 6 | 7 | def task_mygroup(): 8 | return {'actions': None, 9 | 'task_dep': ['foo', 'bar']} 10 | -------------------------------------------------------------------------------- /doc/samples/hello.py: -------------------------------------------------------------------------------- 1 | def task_hello(): 2 | """hello""" 3 | 4 | def python_hello(targets): 5 | with open(targets[0], "a") as output: 6 | output.write("Python says Hello World!!!\n") 7 | 8 | return { 9 | 'actions': [python_hello], 10 | 'targets': ["hello.txt"], 11 | } 12 | 13 | -------------------------------------------------------------------------------- /doc/samples/import_tasks.py: -------------------------------------------------------------------------------- 1 | # import task_ functions 2 | from get_var import task_echo 3 | 4 | # import tasks with create_doit_tasks callable 5 | from custom_task_def import sample 6 | 7 | 8 | def task_hello(): 9 | return {'actions': ['echo hello']} 10 | 11 | -------------------------------------------------------------------------------- /doc/samples/initial_workdir.py: -------------------------------------------------------------------------------- 1 | ### README 2 | # Sample to test doit.get_initial_workdir 3 | # First create a folder named 'sub1'. 4 | # Invoking doit from the root folder will execute both tasks 'base' and 'sub1'. 5 | # Invoking 'doit -k' from path 'sub1' will execute only task 'sub1' 6 | ################## 7 | 8 | import os 9 | 10 | import doit 11 | 12 | DOIT_CONFIG = { 13 | 'verbosity': 2, 14 | 'default_tasks': None, # all by default 15 | } 16 | 17 | 18 | # change default tasks based on dir from where doit was run 19 | sub1_dir = os.path.join(os.path.dirname(__file__), 'sub1') 20 | if doit.get_initial_workdir() == sub1_dir: 21 | DOIT_CONFIG['default_tasks'] = ['sub1'] 22 | 23 | 24 | def task_base(): 25 | return {'actions': ['echo root']} 26 | 27 | def task_sub1(): 28 | return {'actions': ['echo sub1']} 29 | -------------------------------------------------------------------------------- /doc/samples/kbd.c: -------------------------------------------------------------------------------- 1 | #include "defs.h" 2 | #include "command.h" 3 | 4 | -------------------------------------------------------------------------------- /doc/samples/longrunning.py: -------------------------------------------------------------------------------- 1 | from doit.tools import LongRunning 2 | 3 | def task_top(): 4 | cmd = "top" 5 | return {'actions': [LongRunning(cmd)],} 6 | -------------------------------------------------------------------------------- /doc/samples/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() 4 | { 5 | printf("\nHello World\n"); 6 | return 0; 7 | } 8 | -------------------------------------------------------------------------------- /doc/samples/meta.py: -------------------------------------------------------------------------------- 1 | def who(task): 2 | print('my name is', task.name) 3 | print(task.targets) 4 | 5 | def task_x(): 6 | return { 7 | 'actions': [who], 8 | 'targets': ['asdf'], 9 | 'verbosity': 2, 10 | } 11 | -------------------------------------------------------------------------------- /doc/samples/metadata.py: -------------------------------------------------------------------------------- 1 | def task_unittest(): 2 | return { 3 | 'actions': ['echo unit-test'], 4 | 'meta': {'tags': ['test']}, 5 | } 6 | -------------------------------------------------------------------------------- /doc/samples/module_loader.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import sys 4 | 5 | from doit.cmd_base import ModuleTaskLoader 6 | from doit.doit_cmd import DoitMain 7 | 8 | if __name__ == "__main__": 9 | import my_module_with_tasks 10 | sys.exit(DoitMain(ModuleTaskLoader(my_module_with_tasks)).run(sys.argv[1:])) 11 | -------------------------------------------------------------------------------- /doc/samples/my_dodo.py: -------------------------------------------------------------------------------- 1 | 2 | DOIT_CONFIG = {'verbosity': 2} 3 | 4 | TASKS_MODULE = __import__('my_tasks') 5 | 6 | def task_do(): 7 | # get functions that are tasks from module 8 | for name in dir(TASKS_MODULE): 9 | item = getattr(TASKS_MODULE, name) 10 | if not hasattr(item, 'task_metadata'): 11 | continue 12 | 13 | # get task metadata attached to the function 14 | metadata = item.task_metadata 15 | 16 | # get name of task from function name 17 | metadata['name'] = item.__name__ 18 | 19 | # *I* dont like the names file_dep, targets. So I use 'input', 'output' 20 | class Sentinel(object): pass 21 | input_ = metadata.pop('input', Sentinel) 22 | output_ = metadata.pop('output', Sentinel) 23 | args = [] 24 | if input_ != Sentinel: 25 | metadata['file_dep'] = input_ 26 | args.append(input_) 27 | if output_ != Sentinel: 28 | metadata['targets'] = output_ 29 | args.append(output_) 30 | 31 | # the action is the function iteself 32 | metadata['actions'] = [(item, args)] 33 | 34 | yield metadata 35 | 36 | -------------------------------------------------------------------------------- /doc/samples/my_module_with_tasks.py: -------------------------------------------------------------------------------- 1 | 2 | def task_sample(): 3 | return {'actions': ['echo hello from module loader'], 4 | 'verbosity': 2,} 5 | 6 | -------------------------------------------------------------------------------- /doc/samples/my_tasks.py: -------------------------------------------------------------------------------- 1 | def task(*fn, **kwargs): 2 | # decorator without parameters 3 | if fn: 4 | function = fn[0] 5 | function.task_metadata = {} 6 | return function 7 | 8 | # decorator with parameters 9 | def wrap(function): 10 | function.task_metadata = kwargs 11 | return function 12 | return wrap 13 | 14 | 15 | 16 | @task 17 | def simple(): 18 | print("thats all folks") 19 | 20 | @task(output=['my_input.txt']) 21 | def pre(to_create): 22 | with open(to_create[0], 'w') as fp: 23 | fp.write('foo') 24 | 25 | @task(output=['out1.txt', 'out2.txt']) 26 | def create(to_be_created): 27 | print("I should create these files: %s" % " ".join(to_be_created)) 28 | 29 | @task(input=['my_input.txt'], output=['my_output_result.txt']) 30 | def process(in_, out_): 31 | print("processing %s" % in_[0]) 32 | print("creating %s" % out_[0]) 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /doc/samples/parameters.py: -------------------------------------------------------------------------------- 1 | def task_py_params(): 2 | def show_params(param1, param2): 3 | print(param1) 4 | print(5 + param2) 5 | return {'actions':[(show_params,)], 6 | 'params':[{'name':'param1', 7 | 'short':'p', 8 | 'default':'default value'}, 9 | 10 | {'name':'param2', 11 | 'long':'param2', 12 | 'type': int, 13 | 'default':0}], 14 | 'verbosity':2, 15 | } 16 | 17 | def task_py_params_list(): 18 | def print_a_list(list): 19 | for item in list: 20 | print(item) 21 | return {'actions':[(print_a_list,)], 22 | 'params':[{'name':'list', 23 | 'short':'l', 24 | 'long': 'list', 25 | 'type': list, 26 | 'default': [], 27 | 'help': 'Collect a list with multiple -l flags'}], 28 | 'verbosity':2, 29 | } 30 | 31 | def task_py_params_choice(): 32 | def print_choice(choice): 33 | print(choice) 34 | 35 | return {'actions':[(print_choice,)], 36 | 'params':[{'name':'choice', 37 | 'short':'c', 38 | 'long': 'choice', 39 | 'type': str, 40 | 'choices': (('this', ''), ('that', '')), 41 | 'default': 'this', 42 | 'help': 'Choose between this and that'}], 43 | 'verbosity':2,} 44 | 45 | def task_cmd_params(): 46 | return {'actions':["echo mycmd %(flag)s xxx"], 47 | 'params':[{'name':'flag', 48 | 'short':'f', 49 | 'long': 'flag', 50 | 'default': '', 51 | 'help': 'helpful message about this flag'}], 52 | 'verbosity': 2 53 | } 54 | 55 | -------------------------------------------------------------------------------- /doc/samples/parameters_inverse.py: -------------------------------------------------------------------------------- 1 | def task_with_flag(): 2 | def _task(flag): 3 | print("Flag {0}".format("On" if flag else "Off")) 4 | 5 | return { 6 | 'params': [{ 7 | 'name': 'flag', 8 | 'long': 'flagon', 9 | 'short': 'f', 10 | 'type': bool, 11 | 'default': True, 12 | 'inverse': 'flagoff'}], 13 | 'actions': [(_task, )], 14 | 'verbosity': 2 15 | } 16 | -------------------------------------------------------------------------------- /doc/samples/pos.py: -------------------------------------------------------------------------------- 1 | def task_pos_args(): 2 | def show_params(param1, pos): 3 | print('param1 is: {0}'.format(param1)) 4 | for index, pos_arg in enumerate(pos): 5 | print('positional-{0}: {1}'.format(index, pos_arg)) 6 | return {'actions':[(show_params,)], 7 | 'params':[{'name':'param1', 8 | 'short':'p', 9 | 'default':'default value'}, 10 | ], 11 | 'pos_arg': 'pos', 12 | 'verbosity': 2, 13 | } 14 | -------------------------------------------------------------------------------- /doc/samples/report_deps.py: -------------------------------------------------------------------------------- 1 | DOIT_CONFIG = {'action_string_formatting': 'both'} 2 | 3 | def task_report_deps(): 4 | """ 5 | Report dependencies and changed dependencies to a file. 6 | """ 7 | return { 8 | 'file_dep': ['req.in', 'req-dev.in'], 9 | 'actions': [ 10 | # New style formatting 11 | 'echo D: {dependencies}, CH: {changed} > {targets}', 12 | # Old style formatting 13 | 'cat %(targets)s', 14 | ], 15 | 'targets': ['report.txt'], 16 | } 17 | 18 | -------------------------------------------------------------------------------- /doc/samples/run_once.py: -------------------------------------------------------------------------------- 1 | 2 | def run_once(task, values): 3 | def save_executed(): 4 | return {'run-once': True} 5 | task.value_savers.append(save_executed) 6 | return values.get('run-once', False) 7 | -------------------------------------------------------------------------------- /doc/samples/sample.py: -------------------------------------------------------------------------------- 1 | print("hello") 2 | -------------------------------------------------------------------------------- /doc/samples/save_out.py: -------------------------------------------------------------------------------- 1 | from doit.action import CmdAction 2 | 3 | def task_save_output(): 4 | return { 5 | 'actions': [CmdAction("echo x1", save_out='out')], 6 | } 7 | # The task values will contain: {'out': u'x1'} 8 | -------------------------------------------------------------------------------- /doc/samples/selecttasks.py: -------------------------------------------------------------------------------- 1 | 2 | DOIT_CONFIG = {'default_tasks': ['t3']} 3 | 4 | def task_t1(): 5 | return {'actions': ["touch task1"], 6 | 'targets': ['task1']} 7 | 8 | def task_t2(): 9 | return {'actions': ["echo task2"]} 10 | 11 | def task_t3(): 12 | return {'actions': ["echo task3"], 13 | 'file_dep': ['task1']} 14 | -------------------------------------------------------------------------------- /doc/samples/settrace.py: -------------------------------------------------------------------------------- 1 | 2 | def need_to_debug(): 3 | # some code here 4 | from doit import tools 5 | tools.set_trace() 6 | # more code 7 | 8 | def task_X(): 9 | return {'actions':[(need_to_debug,)]} 10 | -------------------------------------------------------------------------------- /doc/samples/subtasks.py: -------------------------------------------------------------------------------- 1 | def task_create_file(): 2 | for i in range(3): 3 | filename = "file%d.txt" % i 4 | yield {'name': filename, 5 | 'actions': ["touch %s" % filename]} 6 | -------------------------------------------------------------------------------- /doc/samples/tar.py: -------------------------------------------------------------------------------- 1 | def task_tar(): 2 | return {'actions': ["tar -cf foo.tar *"], 3 | 'task_dep':['version'], 4 | 'targets':['foo.tar']} 5 | 6 | def task_version(): 7 | return {'actions': ["hg tip --template '{rev}' > revision.txt"]} 8 | -------------------------------------------------------------------------------- /doc/samples/task_doc.py: -------------------------------------------------------------------------------- 1 | def task_hello(): 2 | return { 3 | 'actions': ['echo hello'], 4 | 'doc': 'say hello', 5 | } 6 | -------------------------------------------------------------------------------- /doc/samples/task_kwargs.py: -------------------------------------------------------------------------------- 1 | def func_with_args(arg_first, arg_second): 2 | print(arg_first) 3 | print(arg_second) 4 | return True 5 | 6 | def task_call_func(): 7 | return { 8 | 'actions': [(func_with_args, [], { 9 | 'arg_second': 'This is a second argument.', 10 | 'arg_first': 'This is a first argument.'}) 11 | ], 12 | 'verbosity': 2, 13 | } 14 | -------------------------------------------------------------------------------- /doc/samples/task_name.py: -------------------------------------------------------------------------------- 1 | def task_hello(): 2 | """say hello""" 3 | return { 4 | 'actions': ['echo hello'] 5 | } 6 | 7 | def task_xxx(): 8 | """say hello again""" 9 | return { 10 | 'basename': 'hello2', 11 | 'actions': ['echo hello2'] 12 | } 13 | -------------------------------------------------------------------------------- /doc/samples/task_reusable.py: -------------------------------------------------------------------------------- 1 | 2 | def gen_many_tasks(): 3 | yield {'basename': 't1', 4 | 'actions': ['echo t1']} 5 | yield {'basename': 't2', 6 | 'actions': ['echo t2']} 7 | 8 | def task_all(): 9 | yield gen_many_tasks() 10 | -------------------------------------------------------------------------------- /doc/samples/taskorder.py: -------------------------------------------------------------------------------- 1 | def task_modify(): 2 | return {'actions': ["echo bar > foo.txt"], 3 | 'file_dep': ["foo.txt"], 4 | } 5 | 6 | def task_create(): 7 | return {'actions': ["touch foo.txt"], 8 | 'targets': ["foo.txt"] 9 | } 10 | -------------------------------------------------------------------------------- /doc/samples/taskresult.py: -------------------------------------------------------------------------------- 1 | from doit.tools import result_dep 2 | 3 | def task_version(): 4 | return {'actions': ["hg tip --template '{rev}:{node}'"]} 5 | 6 | def task_send_email(): 7 | return {'actions': ['echo "TODO: send an email"'], 8 | 'uptodate': [result_dep('version')]} 9 | -------------------------------------------------------------------------------- /doc/samples/timeout.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from doit.tools import timeout 3 | 4 | def task_expire(): 5 | return { 6 | 'actions': ['echo test expire; date'], 7 | 'uptodate': [timeout(datetime.timedelta(minutes=5))], 8 | 'verbosity': 2, 9 | } 10 | -------------------------------------------------------------------------------- /doc/samples/title.py: -------------------------------------------------------------------------------- 1 | 2 | def show_cmd(task): 3 | return "executing... %s" % task.actions[0] 4 | 5 | def task_custom_display(): 6 | return {'actions':['echo abc efg'], 7 | 'title': show_cmd} 8 | -------------------------------------------------------------------------------- /doc/samples/titlewithactions.py: -------------------------------------------------------------------------------- 1 | from doit.tools import title_with_actions 2 | 3 | def task_with_details(): 4 | return {'actions': ['echo abc 123'], 5 | 'title': title_with_actions} 6 | -------------------------------------------------------------------------------- /doc/samples/touch.py: -------------------------------------------------------------------------------- 1 | def task_touch(): 2 | return { 3 | 'actions': ['touch foo.txt'], 4 | 'targets': ['foo.txt'], 5 | # force doit to always mark the task 6 | # as up-to-date (unless target removed) 7 | 'uptodate': [True], 8 | } 9 | -------------------------------------------------------------------------------- /doc/samples/tsetup.py: -------------------------------------------------------------------------------- 1 | ### task setup env. good for functional tests! 2 | DOIT_CONFIG = {'verbosity': 2, 3 | 'default_tasks': ['withenvX', 'withenvY']} 4 | 5 | def start(name): 6 | print("start %s" % name) 7 | def stop(name): 8 | print("stop %s" % name) 9 | 10 | def task_setup_sample(): 11 | for name in ('setupX', 'setupY'): 12 | yield {'name': name, 13 | 'actions': [(start, (name,))], 14 | 'teardown': [(stop, (name,))], 15 | } 16 | 17 | def task_withenvX(): 18 | for fin in ('a','b','c'): 19 | yield {'name': fin, 20 | 'actions':['echo x %s' % fin], 21 | 'setup': ['setup_sample:setupX'], 22 | } 23 | 24 | def task_withenvY(): 25 | return {'actions':['echo y'], 26 | 'setup': ['setup_sample:setupY'], 27 | } 28 | -------------------------------------------------------------------------------- /doc/samples/tutorial_02.py: -------------------------------------------------------------------------------- 1 | def task_hello(): 2 | """hello py """ 3 | 4 | def python_hello(times, text, targets): 5 | with open(targets[0], "a") as output: 6 | output.write(times * text) 7 | 8 | return {'actions': [(python_hello, [3, "py!\n"])], 9 | 'targets': ["hello.txt"], 10 | } 11 | -------------------------------------------------------------------------------- /doc/samples/uptodate_callable.py: -------------------------------------------------------------------------------- 1 | 2 | def fake_get_value_from_db(): 3 | return 5 4 | 5 | def check_outdated(): 6 | total = fake_get_value_from_db() 7 | return total > 10 8 | 9 | 10 | def task_put_more_stuff_in_db(): 11 | def put_stuff(): pass 12 | return {'actions': [put_stuff], 13 | 'uptodate': [check_outdated], 14 | } 15 | -------------------------------------------------------------------------------- /doc/samples/verbosity.py: -------------------------------------------------------------------------------- 1 | def task_print(): 2 | return {'actions': ['echo hello'], 3 | 'verbosity': 2} 4 | -------------------------------------------------------------------------------- /doc/support.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: doit community & getting help/support 3 | :keywords: python, doit, question, support, feature request, donations 4 | 5 | .. title:: doit community & getting help/support 6 | 7 | 8 | ======= 9 | Support 10 | ======= 11 | 12 | *doit* has extensive online documentation please read the docs! 13 | 14 | 15 | Feedback & Ideas 16 | ---------------- 17 | 18 | `Discussion forum `_ 19 | 20 | It is always nice to receive feedback. 21 | If you use *doit* please drop us line sharing your experience. 22 | It is appreciated if you introduce yourself, 23 | also mention how long are you using *doit* and using it for what. 24 | 25 | A lot of *doit* features were driven by external ideas and use-cases, 26 | so new ideas are welcome too. 27 | 28 | 29 | Questions 30 | --------- 31 | 32 | Question can be asked in the 33 | `discussion forum `_ 34 | or on `StackOverflow using tag doit `_. 35 | 36 | Please, do not use the github issue tracker to ask questions! 37 | Nor send private emails to project maintainer. 38 | 39 | 40 | Issues/bugs 41 | ----------- 42 | 43 | If you find issues using *doit* please report at 44 | `github issues `_. 45 | 46 | All issues should contain a sample minimal ``dodo.py`` and 47 | the used command line to reproduce the problem. 48 | 49 | 50 | 51 | Feature requests 52 | ---------------- 53 | 54 | Feature requests should also be raised on `github `_, sometimes it is a good idea to have an initial discussion of a new feature in the discussion forum. 55 | 56 | You will be expected provide a real world / example of why and how this feature would be used (if not obvious). 57 | 58 | Users are expected to implement new features themselves. 59 | If you are not willing to spend your time on it, probably nobody else will... 60 | 61 | You might sponsor a feature by placing `bounties `_ or by contacting the maintainer directly. 62 | 63 | 64 | Donations 65 | --------- 66 | 67 | If you consider **doit** to be a useful project, and would like to see it continue to be maintained and developed, please consider helping cover the development cost. Thriving project requires a maintainer to keep list of issues and list of pull requests short. 68 | 69 | **doit** is 100% open-source and maintained by individuals without any company backing it... financial help is appreciated. 70 | 71 | The project receives donation through `OpenCollective `_ or Paypal. 72 | 73 | 74 | Commercial Support 75 | ------------------ 76 | 77 | For commercial support as well as consulting on computational pipelines 78 | by `doit` creator & maintainer, please contact: *schettino72* at gmail.com. 79 | -------------------------------------------------------------------------------- /doc/svg/doit.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 24 | 26 | 33 | 35 | 39 | 43 | 44 | 54 | 64 | 65 | 93 | 97 | 101 | 105 | 109 | 121 | 122 | 124 | 125 | 127 | image/svg+xml 128 | 130 | 131 | 132 | 133 | 137 | 141 | 142 | 143 | -------------------------------------------------------------------------------- /doc/task-creation.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: pydoit guide - re-using task definitions & alternative ways to define tasks. 3 | :keywords: python, doit, documentation, guide, task, dynamic workflow 4 | 5 | .. title:: Alternative way of defining tasks - pydoit guide 6 | 7 | 8 | More on Task creation 9 | ===================== 10 | 11 | 12 | importing tasks 13 | --------------- 14 | 15 | The *doit* loader will look at **all** objects in the namespace of the *dodo*. 16 | It will look for functions staring with ``task_`` and objects with 17 | ``create_doit_tasks``. 18 | So it is also possible to load task definitions from other 19 | modules just by importing them into your *dodo* file. 20 | 21 | .. literalinclude:: samples/import_tasks.py 22 | 23 | .. code-block:: console 24 | 25 | $ doit list 26 | echo 27 | hello 28 | sample 29 | 30 | 31 | .. note:: 32 | 33 | Importing tasks from different modules is useful if you want to split 34 | your task definitions in different modules. 35 | 36 | The best way to create re-usable tasks that can be used in several projects 37 | is to call functions that return task dict's. 38 | For example take a look at a reusable *pyflakes* 39 | `task generator `_. 40 | Check the project `doit-py `_ 41 | for more examples. 42 | 43 | 44 | .. _delayed-task-creation: 45 | 46 | delayed task creation 47 | --------------------- 48 | 49 | 50 | `doit` execution model is divided in two phases: 51 | 52 | - *task-loading* : search for task-creator functions (that starts with string `task_`) and create task metadata 53 | 54 | - *task-execution* : check which tasks are out-of-date and execute them 55 | 56 | Normally *task-loading* is completed before the *task-execution* starts. 57 | `doit` allows some task metadata to be modified during *task-execution* with 58 | `calc_deps` and on `uptodate`, but those are restricted to modifying 59 | already created tasks... 60 | 61 | Sometimes it is not possible to know all tasks that should be created before 62 | some tasks are executed. For these cases `doit` supports 63 | *delayed task creation*, that means *task-execution* starts before 64 | *task-loading* is completed. 65 | 66 | When *task-creator* function is decorated with `doit.create_after`, 67 | its evaluation to create the tasks will be delayed to happen after the 68 | execution of the specified task in the `executed` param. 69 | 70 | .. literalinclude:: samples/delayed.py 71 | 72 | 73 | .. _specify-target-regex: 74 | 75 | .. note:: 76 | 77 | To be able to specify targets created by delayed task loaders to `doit run`, 78 | it is possible to also specify a regular expression (regex) for every 79 | delayed task loader. If specified, this regex should match any target name 80 | possibly generated by this delayed task generator. It can be specified via 81 | the additional *task-generator* argument `target_regex`. In the above example, 82 | the regex `.*\\.out` matches every target name ending with `.out`. 83 | 84 | It is possible to match every possible target name by specifying `.*`. 85 | Alternatively, one can use the command line option `--auto-delayed-regex` 86 | to `doit run`; see :ref:`here ` for more information. 87 | 88 | 89 | Parameter: `creates` 90 | ++++++++++++++++++++ 91 | 92 | In case the task created by a `DelayedTask` has a different *basename* than 93 | then creator function, or creates several tasks with different *basenames*, 94 | you should pass the parameter `creates`. 95 | 96 | Since `doit` will only execute the body of the task-creator function on demand, 97 | the tasks names must be explicitly specified... Example: 98 | 99 | .. literalinclude:: samples/delayed_creates.py 100 | 101 | 102 | .. warning:: 103 | 104 | `doit` normally automatically sets `task_dep` between tasks by checking 105 | the relation of `file_dep` and `targets`. 106 | Due to performance reasons, these `task_dep` relations are NOT computed 107 | for delayed-task's `targets`. 108 | This problem can avoided by ordering the creation of delayed-tasks with 109 | the expected order of execution. 110 | 111 | 112 | .. _create-doit-tasks: 113 | 114 | custom task definition 115 | ------------------------ 116 | 117 | Apart from collect functions that start with the name `task_`. 118 | The *doit* loader will also execute the ``create_doit_tasks`` 119 | callable from any object that contains this attribute. 120 | 121 | 122 | .. literalinclude:: samples/custom_task_def.py 123 | 124 | The `project letsdoit `_ 125 | has some real-world implementations. 126 | 127 | For simple examples to help you create your own check this 128 | `blog post `_. 129 | 130 | 131 | -------------------------------------------------------------------------------- /doc/tools.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: Help snippets for tasks and integrations 3 | :keywords: python, doit, documentation, guide, ipython, interactive 4 | 5 | .. title:: help snippets for tasks and integrations - pydoit guide 6 | 7 | 8 | ==================== 9 | Tools & Integrations 10 | ==================== 11 | 12 | `doit.tools` includes some commonly used code. These are not used by the `doit` 13 | core, you can see it as a "standard library". 14 | The functions/class used with `uptodate` were already introduced in the previous 15 | section. 16 | 17 | 18 | 19 | create_folder (action) 20 | ------------------------- 21 | 22 | Creates a folder if it does not exist yet. Uses `os.makedirs() _`. 23 | 24 | .. literalinclude:: samples/folder.py 25 | 26 | 27 | title_with_actions (title) 28 | ---------------------------- 29 | 30 | Return task name task actions from a task. This function can be used as 'title' attribute of a task dictionary to provide more detailed information of the action being executed. 31 | 32 | .. literalinclude:: samples/titlewithactions.py 33 | 34 | 35 | .. _tools.LongRunning: 36 | 37 | LongRunning (action) 38 | ----------------------------- 39 | 40 | .. autoclass:: doit.tools.LongRunning 41 | 42 | 43 | This is useful for executing long running process like a web-server. 44 | 45 | .. literalinclude:: samples/longrunning.py 46 | 47 | 48 | Interactive (action) 49 | ---------------------------------- 50 | 51 | .. autoclass:: doit.tools.Interactive 52 | 53 | 54 | 55 | PythonInteractiveAction (action) 56 | ---------------------------------- 57 | 58 | .. autoclass:: doit.tools.PythonInteractiveAction 59 | 60 | 61 | 62 | set_trace 63 | ----------- 64 | 65 | `doit` by default redirects stdout and stderr. Because of this when you try to 66 | use the python debugger with ``pdb.set_trace``, it does not work properly. 67 | To make sure you get a proper PDB shell you should use doit.tools.set_trace 68 | instead of ``pdb.set_trace``. 69 | 70 | .. literalinclude:: samples/settrace.py 71 | 72 | 73 | .. _tools.IPython: 74 | 75 | IPython integration 76 | ---------------------- 77 | 78 | A handy possibility for interactive experimentation is to define tasks from 79 | within *ipython* sessions and use the ``%doit`` `magic function 80 | `_ 81 | to discover and execute them. 82 | 83 | 84 | First you need to register the new magic function into ipython shell. 85 | 86 | .. code-block:: pycon 87 | 88 | >>> %load_ext doit.tools 89 | 90 | 91 | .. Tip:: 92 | To permanently add this magic-function to your IPython include it on your 93 | `profile `_, 94 | create a new script inside your startup-profile 95 | (i.e. :file:`~/.ipython/profile_default/startup/doit_magic.ipy`) 96 | with the following content:: 97 | 98 | from doit import load_ipython_extension 99 | load_ipython_extension() 100 | 101 | Then you can define your `task_creator` functions and invoke them with `%doit` 102 | magic-function, instead of invoking the cmd-line script with a :file:`dodo.py` 103 | file. 104 | 105 | 106 | Examples: 107 | 108 | .. code-block:: pycon 109 | 110 | >>> %doit --help ## Show help for options and arguments. 111 | 112 | >>> def task_foo(): 113 | return {'actions': ["echo hi IPython"], 114 | 'verbosity': 2} 115 | 116 | >>> %doit list ## List any tasks discovered. 117 | foo 118 | 119 | >>> %doit ## Run any tasks. 120 | . foo 121 | hi IPython 122 | -------------------------------------------------------------------------------- /doc/tutorial/tuto_1_1.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import pygraphviz 3 | 4 | 5 | def task_imports(): 6 | """find imports from a python module""" 7 | return { 8 | 'file_dep': ['projects/requests/requests/models.py'], 9 | 'targets': ['requests.models.deps'], 10 | 'actions': ['python -m import_deps %(dependencies)s > %(targets)s'], 11 | 'clean': True, 12 | } 13 | 14 | def module_to_dot(dependencies, targets): 15 | graph = pygraphviz.AGraph(strict=False, directed=True) 16 | graph.node_attr['color'] = 'lightblue2' 17 | graph.node_attr['style'] = 'filled' 18 | for dep in dependencies: 19 | filepath = pathlib.Path(dep) 20 | source = filepath.stem 21 | with filepath.open() as fh: 22 | for line in fh: 23 | sink = line.strip() 24 | if sink: 25 | graph.add_edge(source, sink) 26 | graph.write(targets[0]) 27 | 28 | def task_dot(): 29 | """generate a graphviz's dot graph from module imports""" 30 | return { 31 | 'file_dep': ['requests.models.deps'], 32 | 'targets': ['requests.models.dot'], 33 | 'actions': [module_to_dot], 34 | 'clean': True, 35 | } 36 | 37 | 38 | def task_draw(): 39 | """generate image from a dot file""" 40 | return { 41 | 'file_dep': ['requests.models.dot'], 42 | 'targets': ['requests.models.png'], 43 | 'actions': ['dot -Tpng %(dependencies)s -o %(targets)s'], 44 | 'clean': True, 45 | } 46 | -------------------------------------------------------------------------------- /doc/tutorial/tuto_1_2.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import pygraphviz 3 | 4 | from import_deps import PyModule, ModuleSet 5 | 6 | def get_imports(module_path): 7 | module = PyModule(module_path) 8 | base_path = module.pkg_path().resolve() 9 | mset = ModuleSet(base_path.glob('**/*.py')) 10 | imports = mset.get_imports(module, return_fqn=True) 11 | return {'modules': list(sorted(imports))} 12 | 13 | def task_imports(): 14 | """find imports from a python module""" 15 | module_path = 'projects/requests/requests/models.py' 16 | return { 17 | 'file_dep': [module_path], 18 | 'actions': [(get_imports, [module_path])], 19 | } 20 | 21 | 22 | def module_to_dot(source, sinks, targets): 23 | graph = pygraphviz.AGraph(strict=False, directed=True) 24 | graph.node_attr['color'] = 'lightblue2' 25 | graph.node_attr['style'] = 'filled' 26 | for sink in sinks: 27 | graph.add_edge(source, sink) 28 | graph.write(targets[0]) 29 | 30 | 31 | def task_dot(): 32 | """generate a graphviz's dot graph from module imports""" 33 | return { 34 | 'targets': ['requests.models.dot'], 35 | 'actions': [(module_to_dot, (), {'source': 'requests.models'})], 36 | 'getargs': {'sinks': ('imports', 'modules')}, 37 | 'clean': True, 38 | } 39 | 40 | 41 | def task_draw(): 42 | """generate image from a dot file""" 43 | return { 44 | 'file_dep': ['requests.models.dot'], 45 | 'targets': ['requests.models.png'], 46 | 'actions': ['dot -Tpng %(dependencies)s -o %(targets)s'], 47 | 'clean': True, 48 | } 49 | -------------------------------------------------------------------------------- /doc/tutorial/tuto_1_3.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import pygraphviz 3 | 4 | from import_deps import PyModule, ModuleSet 5 | 6 | 7 | 8 | def get_imports(pkg_modules, module_path): 9 | module = pkg_modules.by_path[module_path] 10 | imports = pkg_modules.get_imports(module, return_fqn=True) 11 | return {'modules': list(sorted(imports))} 12 | 13 | def task_imports(): 14 | """find imports from a python module""" 15 | base_path = pathlib.Path('projects/requests/requests') 16 | pkg_modules = ModuleSet(base_path.glob('**/*.py')) 17 | for name, module in pkg_modules.by_name.items(): 18 | yield { 19 | 'name': name, 20 | 'file_dep': [module.path], 21 | 'actions': [(get_imports, (pkg_modules, module.path))], 22 | } 23 | 24 | 25 | def module_to_dot(imports, targets): 26 | graph = pygraphviz.AGraph(strict=False, directed=True) 27 | graph.node_attr['color'] = 'lightblue2' 28 | graph.node_attr['style'] = 'filled' 29 | for source, sinks in imports.items(): 30 | for sink in sinks: 31 | graph.add_edge(source, sink) 32 | graph.write(targets[0]) 33 | 34 | def task_dot(): 35 | """generate a graphviz's dot graph from module imports""" 36 | return { 37 | 'targets': ['requests.dot'], 38 | 'actions': [module_to_dot], 39 | 'getargs': {'imports': ('imports', 'modules')}, 40 | 'clean': True, 41 | } 42 | 43 | 44 | def task_draw(): 45 | """generate image from a dot file""" 46 | return { 47 | 'file_dep': ['requests.dot'], 48 | 'targets': ['requests.png'], 49 | 'actions': ['dot -Tpng %(dependencies)s -o %(targets)s'], 50 | 'clean': True, 51 | } 52 | -------------------------------------------------------------------------------- /doc/tutorial/tuto_1_4.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import pygraphviz 3 | 4 | from import_deps import PyModule, ModuleSet 5 | 6 | 7 | DOIT_CONFIG = { 8 | 'default_tasks': ['imports', 'dot', 'draw'], 9 | } 10 | 11 | 12 | base_path = pathlib.Path('projects/requests/requests') 13 | PKG_MODULES = ModuleSet(base_path.glob('**/*.py')) 14 | 15 | 16 | def get_imports(pkg_modules, module_path): 17 | module = pkg_modules.by_path[module_path] 18 | imports = pkg_modules.get_imports(module, return_fqn=True) 19 | return {'modules': list(sorted(imports))} 20 | 21 | 22 | def task_imports(): 23 | """find imports from a python module""" 24 | for name, module in PKG_MODULES.by_name.items(): 25 | yield { 26 | 'name': name, 27 | 'file_dep': [module.path], 28 | 'actions': [(get_imports, (PKG_MODULES, module.path))], 29 | } 30 | 31 | 32 | def print_imports(modules): 33 | print('\n'.join(modules)) 34 | 35 | def task_print(): 36 | """print on stdout list of direct module imports""" 37 | for name, module in PKG_MODULES.by_name.items(): 38 | yield { 39 | 'name': name, 40 | 'actions': [print_imports], 41 | 'getargs': {'modules': ('imports:{}'.format(name), 'modules')}, 42 | 'uptodate': [False], 43 | 'verbosity': 2, 44 | } 45 | 46 | 47 | def module_to_dot(imports, targets): 48 | graph = pygraphviz.AGraph(strict=False, directed=True) 49 | graph.node_attr['color'] = 'lightblue2' 50 | graph.node_attr['style'] = 'filled' 51 | for source, sinks in imports.items(): 52 | for sink in sinks: 53 | graph.add_edge(source, sink) 54 | graph.write(targets[0]) 55 | 56 | def task_dot(): 57 | """generate a graphviz's dot graph from module imports""" 58 | return { 59 | 'targets': ['requests.dot'], 60 | 'actions': [module_to_dot], 61 | 'getargs': {'imports': ('imports', 'modules')}, 62 | 'clean': True, 63 | } 64 | 65 | 66 | def task_draw(): 67 | """generate image from a dot file""" 68 | return { 69 | 'file_dep': ['requests.dot'], 70 | 'targets': ['requests.png'], 71 | 'actions': ['dot -Tpng %(dependencies)s -o %(targets)s'], 72 | 'clean': True, 73 | } 74 | -------------------------------------------------------------------------------- /doc/uptodate.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: Customize the up-to-date check for incremental task execution 3 | :keywords: python, doit, documentation, guide, incremental-build, result-dependency, run_one, config_changed, timeout 4 | 5 | .. title:: Customize the up-to-date check for incremental execution 6 | 7 | 8 | ================= 9 | custom up-to-date 10 | ================= 11 | 12 | The basics of `uptodate` was already :ref:`introduced `. 13 | Here we look in more 14 | detail into some implementations shipped with `doit`. 15 | 16 | 17 | .. _result_dep: 18 | 19 | result-dependency 20 | ---------------------- 21 | 22 | In some cases you can not determine if a task is "up-to-date" only based on 23 | input files, the input could come from a database or an external process. 24 | *doit* defines a "result-dependency" to deal with these cases without need to 25 | create an intermediate file with the results of the process. 26 | 27 | i.e. Suppose you want to send an email every time you run *doit* on a mercurial 28 | repository that contains a new revision number. 29 | 30 | .. literalinclude:: samples/taskresult.py 31 | 32 | 33 | Note the `result_dep` with the name of the task ('version'). `doit` will keep 34 | track of the output of the task *version* and will execute *send_email* only 35 | when the mercurial repository has a new version since last 36 | time *doit* was executed. 37 | 38 | The "result" from the dependent task compared between different runs is given 39 | by its last action. 40 | The content for python-action is the value of the returned string or dict. 41 | For cmd-actions it is the output send to stdout plus stderr. 42 | 43 | `result_dep` also supports group-tasks. In this case it will check that the 44 | result of all subtasks did not change. And also the existing sub-tasks are 45 | the same. 46 | 47 | .. _run_once: 48 | 49 | 50 | run_once() 51 | --------------- 52 | 53 | Sometimes there is no dependency for a task but you do not want to execute it 54 | all the time. With "run_once" the task will not be executed again after the first 55 | successful run. This is mostly used together with targets. 56 | 57 | Suppose you need to download something from internet. 58 | There is no dependency, but you do not want to download it many times. 59 | 60 | 61 | .. literalinclude:: samples/download.py 62 | 63 | Note that even with *run_once* the file will be downloaded again in case the target is removed. 64 | 65 | 66 | .. code-block:: console 67 | 68 | $ doit 69 | . get_pylogo 70 | $ doit 71 | -- get_pylogo 72 | $ rm python-logo.gif 73 | $ doit 74 | . get_pylogo 75 | 76 | 77 | .. _timeout: 78 | 79 | timeout() 80 | ----------- 81 | 82 | ``timeout`` is used to expire a task after a certain time interval. 83 | 84 | i.e. You want to re-execute a task only if the time elapsed since the last 85 | time it was executed is bigger than 5 minutes. 86 | 87 | .. literalinclude:: samples/timeout.py 88 | 89 | 90 | ``timeout`` is function that takes an ``int`` (seconds) or ``timedelta`` as a 91 | parameter. It returns a callable suitable to be used as an ``uptodate`` callable. 92 | 93 | 94 | .. _config_changed: 95 | 96 | config_changed() 97 | ----------------- 98 | 99 | ``config_changed`` is used to check if any "configuration" value for the task has 100 | changed. Config values can be a string or dict. 101 | 102 | For dict's the values are converted to string (using `json.dumps()` with `sort_key=True`) 103 | and only a digest/checksum of the dictionaries keys and values are saved. 104 | 105 | If converting the values of the dict requires a special encoder this can be 106 | passed with the argument ``encoder=...``. This will be passed on to `json.dumps()`. 107 | 108 | .. literalinclude:: samples/config_params.py 109 | 110 | 111 | .. _check_timestamp_unchanged: 112 | 113 | check_timestamp_unchanged() 114 | ----------------------------- 115 | 116 | ``check_timestamp_unchanged`` is used to check if specified timestamp of a given 117 | file/dir is unchanged since last run. 118 | 119 | The timestamp field to check defaults to ``mtime``, but can be selected by 120 | passing ``time`` parameter which can be one of: ``atime``, ``ctime``, ``mtime`` 121 | (or their aliases ``access``, ``status``, ``modify``). 122 | 123 | Note that ``ctime`` or ``status`` is platform dependent. 124 | On Unix it is the time of most recent metadata change, 125 | on Windows it is the time of creation. 126 | See `Python library documentation for os.stat`__ and Linux man page for 127 | stat(2) for details. 128 | 129 | __ http://docs.python.org/library/os.html#os.stat 130 | 131 | It also accepts an ``cmp_op`` parameter which defaults to ``operator.eq`` (==). 132 | To use it pass a callable which takes two parameters (prev_time, current_time) 133 | and returns True if task should be considered up-to-date, False otherwise. 134 | Here ``prev_time`` is the time from the last successful run and ``current_time`` 135 | is the time obtained in current run. 136 | 137 | If the specified file does not exist, an exception will be raised. 138 | If a file is a target of another task you should probably add 139 | ``task_dep`` on that task to ensure the file is created before it is checked. 140 | 141 | .. literalinclude:: samples/check_timestamp_unchanged.py 142 | -------------------------------------------------------------------------------- /doc/usecases.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :description: pydoit is a generic tool based on build-tool concepts. Flexible and scalable. 3 | :keywords: python, doit, CLI, linux, windows, task-runner, build-tool, pipeline, workflow, incremental build, data pipeline 4 | 5 | .. title:: pydoit use cases - from CLI task-runner to complex pipelines 6 | 7 | ========= 8 | Use Cases 9 | ========= 10 | 11 | Here are some use cases, where `doit` can help you with automation of your tasks. 12 | 13 | 14 | 15 | Simplify cumbersome command line calls 16 | ====================================== 17 | 18 | Do you have to repeatedly call complex command like this? 19 | 20 | .. code-block:: console 21 | 22 | $ aws s3 sync _built/html s3://buck/et --exclude "*" --include "*.html" 23 | 24 | 25 | Wrap it into `dodo.py` file: 26 | 27 | .. code-block:: python 28 | 29 | def task_publish(): 30 | """Publish to AWS S3""" 31 | return { 32 | "actions": [ 33 | 'aws s3 sync _built/html s3://buck/et --exclude "*" --include "*.html"' 34 | ] 35 | } 36 | 37 | and next time just: 38 | 39 | .. code-block:: console 40 | 41 | $ doit publish 42 | 43 | It is easy to include multiple actions into one task or use multiple tasks. 44 | 45 | 46 | Automate typical project related actions 47 | ======================================== 48 | 49 | Do you have to lint your code, run test suite, evaluate coverage, 50 | generate documentation incl. spelling? 51 | 52 | Create the `dodo.py`, which defines tasks you have to do and next time: 53 | 54 | .. code-block:: console 55 | 56 | $ doit list 57 | coverage show coverage for all modules including tests 58 | coverage_module show coverage for individual modules 59 | coverage_src show coverage for all modules (exclude tests) 60 | package create/upload package to pypi 61 | pyflakes 62 | pypi create/upload package to pypi 63 | spell spell checker for doc files 64 | sphinx build sphinx docs 65 | tutorial_check check tutorial sample are at least runnable without error 66 | ut run unit-tests 67 | website dodo file create website html files 68 | website_update update website on SITE_PATH 69 | 70 | and then decide which task to run: 71 | 72 | .. code-block:: console 73 | 74 | $ doit spell 75 | 76 | Share unified way of doing things 77 | ================================= 78 | 79 | Do you expect your colleagues perform the same steps before committing 80 | changes to repository? What to do with the complains the steps are too complex? 81 | 82 | Provide them with the `dodo.py` file doing the things. What goes easy, 83 | is more likely to be used. 84 | 85 | `dodo.py` will become easy to use prescription of best practices. 86 | 87 | Optimize processing time by skipping tasks already done 88 | ======================================================= 89 | 90 | You dump your database and convert the data to CSV. It takes minutes, 91 | but often the input is the same as before. Why to do things already 92 | done and wait? 93 | 94 | Wrap the conversion into `doit` task and `doit` will automatically 95 | detect, the input and output are already in sync and complete in 96 | fraction of a second, when possible. 97 | 98 | Manage complex set of depending tasks 99 | ===================================== 100 | 101 | The system you code shall do many small actions, which are interdependent. 102 | 103 | Split it into small tasks, define (file) dependencies and let `doit` 104 | do the planning of what shall be processed first and what next. 105 | 106 | Your solution will be clean and modular. 107 | 108 | Speed up by parallel task execution 109 | =================================== 110 | You already have bunch of tasks defined, results are correct, it only takes so 111 | much time. But wait, you have multi-core machine! 112 | 113 | Just ask for parallel processing: 114 | 115 | .. code-block:: console 116 | 117 | $ doit -n 4 118 | 119 | and `doit` will take care of planning and make all your CPU cores hot. 120 | 121 | No need to rewrite your processing from scratch, properly declared tasks is all 122 | what you have to provide. 123 | 124 | Extend your project by doit features 125 | ==================================== 126 | Your own python project would need features of `doit`, but you cannot ask your users to call `doit` on command line? 127 | 128 | Simply integrate `doit` functionality into your own command line tool and nobody will notice where it comes from. 129 | 130 | Create cross-platform tool for processing your stuff 131 | ===================================================== 132 | Do you have team members working on MS Windows and others on Linux? 133 | 134 | Scripts are great, but all those small shell differences prevent 135 | single reusable solution. 136 | 137 | With `dodo.py` and python you are more likely to write the processing 138 | in cross-platform way. Use `pathlib.Path` and `shutils` magic to 139 | create directories, move files around, copy them, etc. 140 | -------------------------------------------------------------------------------- /doc_requirements.txt: -------------------------------------------------------------------------------- 1 | # modules required to generate documentation 2 | # $ pip install --requirement doc_requirements.txt 3 | 4 | sphinx 5 | sphinx_press_theme 6 | sphinx-sitemap 7 | sphinx_reredirects 8 | -------------------------------------------------------------------------------- /dodo.py: -------------------------------------------------------------------------------- 1 | """dodo file. test + management stuff""" 2 | 3 | import glob 4 | import os 5 | 6 | import pytest 7 | from doitpy.pyflakes import Pyflakes 8 | from doitpy.coverage import Config, Coverage, PythonPackage 9 | from doitpy import docs 10 | from doitpy.package import Package 11 | 12 | 13 | DOIT_CONFIG = { 14 | 'minversion': '0.24.0', 15 | 'default_tasks': ['pyflakes', 'ut'], 16 | # 'backend': 'sqlite3', 17 | 'forget_disable_default': True, 18 | } 19 | 20 | CODE_FILES = glob.glob("doit/*.py") 21 | TEST_FILES = glob.glob("tests/test_*.py") 22 | TESTING_FILES = glob.glob("tests/*.py") 23 | PY_FILES = CODE_FILES + TESTING_FILES 24 | 25 | 26 | def task_pyflakes(): 27 | flaker = Pyflakes() 28 | yield flaker('dodo.py') 29 | yield flaker.tasks('doit/*.py') 30 | yield flaker.tasks('tests/*.py') 31 | 32 | def run_test(test): 33 | return not bool(pytest.main([test])) 34 | #return not bool(pytest.main("-v " + test)) 35 | def task_ut(): 36 | """run unit-tests""" 37 | for test in TEST_FILES: 38 | yield {'name': test, 39 | 'actions': [(run_test, (test,))], 40 | 'file_dep': PY_FILES, 41 | 'verbosity': 0} 42 | 43 | 44 | def task_coverage(): 45 | """show coverage for all modules including tests""" 46 | config = Config(branch=False, parallel=True, concurrency='multiprocessing', 47 | omit=['tests/myecho.py', 'tests/sample_process.py']) 48 | cov = Coverage([PythonPackage('doit', 'tests')], config=config) 49 | yield cov.all() 50 | yield cov.src() 51 | yield cov.by_module() 52 | 53 | 54 | 55 | ############################ website 56 | 57 | 58 | DOC_ROOT = 'doc/' 59 | DOC_BUILD_PATH = DOC_ROOT + '_build/html/' 60 | 61 | def task_rm_index(): 62 | """remove/clean copied index.html if source changed""" 63 | # work around https://github.com/sphinx-doc/sphinx/issues/1649 64 | return { 65 | 'actions': ['cd doc && make clean'], 66 | 'file_dep': ['doc/index.html'], 67 | } 68 | 69 | def task_docs(): 70 | doc_files = glob.glob('doc/*.rst') 71 | doc_files += ['README.rst', 'CONTRIBUTING.md', 72 | 'doc/open_collective.md'] 73 | yield docs.spell(doc_files, 'doc/dictionary.txt') 74 | sphinx_opts = "-A include_analytics=1 -A include_donate=1" 75 | yield docs.sphinx(DOC_ROOT, DOC_BUILD_PATH, sphinx_opts=sphinx_opts, 76 | task_dep=['spell', 'rm_index']) 77 | 78 | def task_samples_check(): 79 | """check samples are at least runnuable without error""" 80 | black_list = [ 81 | 'longrunning.py', # long running doesn't terminate on its own 82 | 'settrace.py', 83 | 'download.py', # uses network 84 | 'taskresult.py', # uses mercurial 85 | 'tar.py', # uses mercurial 86 | 'calc_dep.py', # uses files not created by the script 87 | 'report_deps.py', # uses files not created by the script 88 | 'doit_config.py', # no tasks defined 89 | ] 90 | exclude = set('doc/samples/{}'.format(m) for m in black_list) 91 | arguments = {'doc/samples/pos.py': 'pos_args -p 4 foo bar'} 92 | 93 | for sample in glob.glob("doc/samples/*.py"): 94 | if sample in exclude: 95 | continue 96 | args = arguments.get(sample, '') 97 | yield { 98 | 'name': sample, 99 | 'actions': ['doit -f {} {}'.format(sample, args)], 100 | } 101 | 102 | 103 | def task_website(): 104 | """dodo file create website html files""" 105 | return {'actions': None, 106 | 'task_dep': ['sphinx', 'samples_check'], 107 | } 108 | 109 | def task_website_update(): 110 | """update website on SITE_PATH 111 | website is hosted on github-pages 112 | this task just copy the generated content to SITE_PATH, 113 | need to commit/push to deploy site. 114 | """ 115 | SITE_PATH = '../doit-website' 116 | SITE_URL = 'pydoit.org' 117 | return { 118 | 'actions': [ 119 | "rsync -avP %s %s" % (DOC_BUILD_PATH, SITE_PATH), 120 | "echo %s > %s" % (SITE_URL, os.path.join(SITE_PATH, 'CNAME')), 121 | "touch %s" % os.path.join(SITE_PATH, '.nojekyll'), 122 | ], 123 | 'task_dep': ['website'], 124 | } 125 | 126 | 127 | 128 | def task_package(): 129 | """create/upload package to pypi""" 130 | pkg = Package() 131 | yield pkg.revision_git() 132 | yield pkg.manifest_git() 133 | yield pkg.sdist() 134 | # yield pkg.sdist_upload() 135 | 136 | 137 | def task_codestyle(): 138 | return { 139 | 'actions': ['pycodestyle doit'], 140 | } 141 | -------------------------------------------------------------------------------- /doit/__init__.py: -------------------------------------------------------------------------------- 1 | """doit - Automation Tool 2 | 3 | The MIT License 4 | 5 | Copyright (c) 2008-present Eduardo Naufel Schettino 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | THE SOFTWARE. 24 | """ 25 | 26 | from doit.version import VERSION 27 | 28 | __version__ = VERSION 29 | 30 | 31 | from doit import loader 32 | from doit.loader import create_after, task_params 33 | from doit.doit_cmd import get_var 34 | from doit.api import run 35 | from doit.tools import load_ipython_extension 36 | from doit.globals import Globals 37 | 38 | 39 | __all__ = ['get_var', 'run', 'create_after', 'task_params', 'Globals'] 40 | 41 | def get_initial_workdir(): 42 | """working-directory from where the doit command was invoked on shell""" 43 | return loader.initial_workdir 44 | 45 | assert load_ipython_extension # silence pyflakes 46 | -------------------------------------------------------------------------------- /doit/__main__.py: -------------------------------------------------------------------------------- 1 | # lazy way to ignore coverage in this file 2 | if True: # pragma: no cover 3 | def main(): 4 | import sys 5 | 6 | from doit.doit_cmd import DoitMain 7 | 8 | sys.exit(DoitMain().run(sys.argv[1:])) 9 | 10 | if __name__ == '__main__': 11 | main() 12 | -------------------------------------------------------------------------------- /doit/api.py: -------------------------------------------------------------------------------- 1 | """APIs to execute doit in non standard way. 2 | 3 | - run(): shortcut to get tasks from current module/dict (instead of dodo.py). 4 | - run_tasks(): to be used by custom CLIs, run tasks without CLI parsing. 5 | 6 | """ 7 | 8 | import sys 9 | 10 | from doit.cmdparse import CmdParseError 11 | from doit.exceptions import InvalidDodoFile, InvalidCommand, InvalidTask 12 | from doit.cmd_base import ModuleTaskLoader, get_loader 13 | from doit.doit_cmd import DoitMain 14 | 15 | 16 | def run(task_creators): 17 | """run doit using task_creators 18 | 19 | @param task_creators: module or dict containing task creators 20 | """ 21 | sys.exit(DoitMain(ModuleTaskLoader(task_creators)).run(sys.argv[1:])) 22 | 23 | 24 | 25 | 26 | def run_tasks(loader, tasks, extra_config=None): 27 | """run DoitMain instance with speficied tasks and parameters 28 | 29 | :params tasks: list of task names (str) 30 | """ 31 | loader.task_opts = tasks # task_opts will be used as @task_param 32 | main = DoitMain(loader, extra_config=extra_config) 33 | task_names = list(tasks.keys()) 34 | 35 | # get list of available commands 36 | sub_cmds = main.get_cmds() 37 | task_loader = get_loader(main.config, main.task_loader, sub_cmds) 38 | 39 | # execute command 40 | cmd_name = 'run' 41 | command = sub_cmds.get_plugin(cmd_name)( 42 | task_loader=task_loader, 43 | config=main.config, 44 | bin_name=main.BIN_NAME, 45 | cmds=sub_cmds, 46 | opt_vals={}, 47 | ) 48 | 49 | try: 50 | return command.parse_execute(task_names) 51 | except (CmdParseError, InvalidDodoFile, 52 | InvalidCommand, InvalidTask) as err: 53 | if isinstance(err, InvalidCommand): 54 | err.cmd_used = cmd_name 55 | err.bin_name = main.BIN_NAME 56 | raise err 57 | -------------------------------------------------------------------------------- /doit/cmd_dumpdb.py: -------------------------------------------------------------------------------- 1 | import pprint 2 | import json 3 | import dbm 4 | from dbm import whichdb 5 | 6 | 7 | from .exceptions import InvalidCommand 8 | from .cmd_base import Command, opt_depfile 9 | 10 | 11 | def dbm_iter(db): 12 | # try dictionary interface - used in dumbdb 13 | try: 14 | return db.items() 15 | except AttributeError: # pragma: no cover 16 | pass 17 | 18 | # try firstkey/nextkey - ok for py3 dbm.gnu 19 | try: # pragma: no cover 20 | db.firstkey 21 | def iter_gdbm(db): 22 | k = db.firstkey() 23 | while k is not None: 24 | yield k, db[k] 25 | k = db.nextkey(k) 26 | return iter_gdbm(db) 27 | except Exception: # pragma: no cover 28 | raise InvalidCommand("It seems your DB backend doesn't support " 29 | "iterating through all elements") 30 | 31 | 32 | class DumpDB(Command): 33 | """dump dependency DB""" 34 | doc_purpose = 'dump dependency DB' 35 | doc_usage = '' 36 | doc_description = None 37 | 38 | cmd_options = (opt_depfile,) 39 | 40 | def execute(self, opt_values, pos_args): 41 | dep_file = opt_values['dep_file'] 42 | db_type = whichdb(dep_file) 43 | print("DBM type is '%s'" % db_type) 44 | if db_type in ('dbm', 'dbm.ndbm'): # pragma: no cover 45 | raise InvalidCommand('ndbm does not support iteration of elements') 46 | data = dbm.open(dep_file) 47 | for key, value_str in dbm_iter(data): 48 | value_dict = json.loads(value_str.decode('utf-8')) 49 | value_fmt = pprint.pformat(value_dict, indent=4, width=100) 50 | print("{key} -> {value}".format(key=key, value=value_fmt)) 51 | -------------------------------------------------------------------------------- /doit/cmd_forget.py: -------------------------------------------------------------------------------- 1 | from .cmd_base import DoitCmdBase, check_tasks_exist 2 | from .cmd_base import tasks_and_deps_iter, subtasks_iter 3 | 4 | 5 | opt_forget_taskdep = { 6 | 'name': 'forget_sub', 7 | 'short': 's', 8 | 'long': 'follow-sub', 9 | 'type': bool, 10 | 'default': False, 11 | 'help': 'forget task dependencies too', 12 | } 13 | 14 | opt_disable_default = { 15 | 'name': 'forget_disable_default', 16 | 'long': 'disable-default', 17 | 'inverse': 'enable-default', 18 | 'type': bool, 19 | 'default': False, 20 | 'help': 'disable forgetting default tasks (when no arguments are passed)', 21 | } 22 | 23 | opt_forget_all = { 24 | 'name': 'forget_all', 25 | 'short': 'a', 26 | 'long': 'all', 27 | 'type': bool, 28 | 'default': False, 29 | 'help': 'forget all tasks', 30 | } 31 | 32 | 33 | 34 | class Forget(DoitCmdBase): 35 | doc_purpose = "clear successful run status from internal DB" 36 | doc_usage = "[TASK ...]" 37 | doc_description = None 38 | 39 | cmd_options = (opt_forget_taskdep, opt_disable_default, opt_forget_all) 40 | 41 | def _execute(self, forget_sub, forget_disable_default, forget_all): 42 | """remove saved data successful runs from DB 43 | """ 44 | if forget_all: 45 | self.dep_manager.remove_all() 46 | self.outstream.write("forgetting all tasks\n") 47 | 48 | elif self.sel_default_tasks and forget_disable_default: 49 | self.outstream.write( 50 | "no tasks specified, pass task name, --enable-default or --all\n") 51 | 52 | # forget tasks from list 53 | else: 54 | tasks = dict([(t.name, t) for t in self.task_list]) 55 | check_tasks_exist(tasks, self.sel_tasks) 56 | forget_list = self.sel_tasks 57 | 58 | if forget_sub: 59 | to_forget = list(tasks_and_deps_iter(tasks, forget_list, True)) 60 | else: 61 | to_forget = [] 62 | for name in forget_list: 63 | task = tasks[name] 64 | to_forget.append(task) 65 | to_forget.extend(subtasks_iter(tasks, task)) 66 | 67 | for task in to_forget: 68 | # forget it - remove from dependency file 69 | self.dep_manager.remove(task.name) 70 | self.outstream.write("forgetting %s\n" % task.name) 71 | self.dep_manager.close() 72 | -------------------------------------------------------------------------------- /doit/cmd_ignore.py: -------------------------------------------------------------------------------- 1 | from .cmd_base import DoitCmdBase, check_tasks_exist, subtasks_iter 2 | 3 | 4 | class Ignore(DoitCmdBase): 5 | doc_purpose = "ignore task (skip) on subsequent runs" 6 | doc_usage = "TASK [TASK ...]" 7 | doc_description = None 8 | 9 | cmd_options = () 10 | 11 | def _execute(self, pos_args): 12 | """mark tasks to be ignored 13 | @param ignore_tasks: (list - str) tasks to be ignored. 14 | """ 15 | ignore_tasks = pos_args 16 | # no task specified. 17 | if not ignore_tasks: 18 | msg = "You cant ignore all tasks! Please select a task.\n" 19 | self.outstream.write(msg) 20 | return 21 | 22 | tasks = dict([(t.name, t) for t in self.task_list]) 23 | check_tasks_exist(tasks, ignore_tasks) 24 | 25 | for task_name in ignore_tasks: 26 | # for group tasks also remove all tasks from group 27 | sub_list = [t.name for t in subtasks_iter(tasks, tasks[task_name])] 28 | for to_ignore in [task_name] + sub_list: 29 | # ignore it - remove from dependency file 30 | self.dep_manager.ignore(tasks[to_ignore]) 31 | self.outstream.write("ignoring %s\n" % to_ignore) 32 | 33 | self.dep_manager.close() 34 | -------------------------------------------------------------------------------- /doit/cmd_info.py: -------------------------------------------------------------------------------- 1 | """command doit info - display info on task metadata""" 2 | 3 | import pprint 4 | 5 | from .cmd_base import DoitCmdBase 6 | from .exceptions import InvalidCommand 7 | 8 | 9 | opt_hide_status = { 10 | 'name': 'hide_status', 11 | 'long': 'no-status', 12 | 'type': bool, 13 | 'default': False, 14 | 'help': """Hides reasons why this task would be executed. 15 | [default: %(default)s]""" 16 | } 17 | 18 | 19 | class Info(DoitCmdBase): 20 | """command doit info""" 21 | 22 | doc_purpose = "show info about a task" 23 | doc_usage = "TASK" 24 | doc_description = None 25 | 26 | cmd_options = (opt_hide_status, ) 27 | 28 | def _execute(self, pos_args, hide_status=False): 29 | if len(pos_args) != 1: 30 | msg = ('`info` failed, must select *one* task.' 31 | '\nCheck `{} help info`.'.format(self.bin_name)) 32 | raise InvalidCommand(msg) 33 | 34 | task_name = pos_args[0] 35 | # dict of all tasks 36 | tasks = dict([(t.name, t) for t in self.task_list]) 37 | 38 | printer = pprint.PrettyPrinter(indent=4, stream=self.outstream) 39 | 40 | task = tasks[task_name] 41 | task_attrs = ( 42 | ('file_dep', 'list'), 43 | ('task_dep', 'list'), 44 | ('setup_tasks', 'list'), 45 | ('calc_dep', 'list'), 46 | ('targets', 'list'), 47 | # these fields usually contains reference to python functions 48 | # 'actions', 'clean', 'uptodate', 'teardown', 'title' 49 | ('getargs', 'dict'), 50 | ('params', 'list'), 51 | ('verbosity', 'scalar'), 52 | ('watch', 'list'), 53 | ('meta', 'dict') 54 | ) 55 | 56 | self.outstream.write('\n{}\n'.format(task.name)) 57 | if task.doc: 58 | self.outstream.write('\n{}\n'.format(task.doc)) 59 | 60 | # print reason task is not up-to-date 61 | retcode = 0 62 | if not hide_status: 63 | status = self.dep_manager.get_status(task, tasks, get_log=True) 64 | self.outstream.write('\n{:11s}: {}\n' 65 | .format('status', status.status)) 66 | if status.status != 'up-to-date': 67 | # status.status == 'run' or status.status == 'error' 68 | self.outstream.write(self.get_reasons(status.reasons)) 69 | self.outstream.write('\n') 70 | retcode = 1 71 | 72 | for (attr, attr_type) in task_attrs: 73 | value = getattr(task, attr) 74 | # only print fields that have non-empty value 75 | if value: 76 | self.outstream.write('\n{:11s}: '.format(attr)) 77 | if attr_type == 'list': 78 | self.outstream.write('\n') 79 | for val in value: 80 | self.outstream.write(' - {}\n'.format(val)) 81 | else: 82 | printer.pprint(getattr(task, attr)) 83 | 84 | return retcode 85 | 86 | @staticmethod 87 | def get_reasons(reasons): 88 | '''return string with description of reason task is not up-to-date''' 89 | lines = [] 90 | if reasons['has_no_dependencies']: 91 | lines.append(' * The task has no dependencies.') 92 | 93 | if reasons['uptodate_false']: 94 | lines.append(' * The following uptodate objects evaluate to false:') 95 | for utd, utd_args, utd_kwargs in reasons['uptodate_false']: 96 | msg = ' - {} (args={}, kwargs={})' 97 | lines.append(msg.format(utd, utd_args, utd_kwargs)) 98 | 99 | if reasons['checker_changed']: 100 | msg = ' * The file_dep checker changed from {0} to {1}.' 101 | lines.append(msg.format(*reasons['checker_changed'])) 102 | 103 | sentences = { 104 | 'missing_target': 'The following targets do not exist:', 105 | 'changed_file_dep': 'The following file dependencies have changed:', 106 | 'missing_file_dep': 'The following file dependencies are missing:', 107 | 'removed_file_dep': 'The following file dependencies were removed:', 108 | 'added_file_dep': 'The following file dependencies were added:', 109 | } 110 | for reason, sentence in sentences.items(): 111 | entries = reasons.get(reason) 112 | if entries: 113 | lines.append(' * {}'.format(sentence)) 114 | for item in entries: 115 | lines.append(' - {}'.format(item)) 116 | return '\n'.join(lines) 117 | -------------------------------------------------------------------------------- /doit/cmd_list.py: -------------------------------------------------------------------------------- 1 | from .cmd_base import DoitCmdBase, check_tasks_exist, subtasks_iter 2 | 3 | opt_listall = { 4 | 'name': 'subtasks', 5 | 'short': '', 6 | 'long': 'all', 7 | 'type': bool, 8 | 'default': False, 9 | 'help': "list include all sub-tasks from dodo file" 10 | } 11 | 12 | opt_list_quiet = { 13 | 'name': 'quiet', 14 | 'short': 'q', 15 | 'long': 'quiet', 16 | 'type': bool, 17 | 'default': False, 18 | 'help': 'print just task name (less verbose than default)' 19 | } 20 | 21 | opt_list_status = { 22 | 'name': 'status', 23 | 'short': 's', 24 | 'long': 'status', 25 | 'type': bool, 26 | 'default': False, 27 | 'help': 'print task status (R)un, (U)p-to-date, (I)gnored' 28 | } 29 | 30 | opt_list_private = { 31 | 'name': 'private', 32 | 'short': 'p', 33 | 'long': 'private', 34 | 'type': bool, 35 | 'default': False, 36 | 'help': "print private tasks (start with '_')" 37 | } 38 | 39 | opt_list_dependencies = { 40 | 'name': 'list_deps', 41 | 'short': '', 42 | 'long': 'deps', 43 | 'type': bool, 44 | 'default': False, 45 | 'help': ("print list of dependencies " 46 | "(file dependencies only)") 47 | } 48 | 49 | opt_template = { 50 | 'name': 'template', 51 | 'short': '', 52 | 'long': 'template', 53 | 'type': str, 54 | 'default': None, 55 | 'help': "display entries with template" 56 | } 57 | 58 | opt_sort = { 59 | 'name': 'sort', 60 | 'short': '', 61 | 'long': 'sort', 62 | 'type': str, 63 | 'choices': [('name', 'sort by task name'), 64 | ('definition', 'list tasks in the order they were defined')], 65 | 'default': 'name', 66 | 'help': ("choose the manner in which the task list is sorted. " 67 | "[default: %(default)s]") 68 | } 69 | 70 | 71 | class List(DoitCmdBase): 72 | doc_purpose = "list tasks from dodo file" 73 | doc_usage = "[TASK ...]" 74 | doc_description = None 75 | 76 | cmd_options = (opt_listall, opt_list_quiet, opt_list_status, 77 | opt_list_private, opt_list_dependencies, opt_template, 78 | opt_sort) 79 | 80 | 81 | STATUS_MAP = {'ignore': 'I', 'up-to-date': 'U', 'run': 'R', 'error': 'E'} 82 | 83 | 84 | def _print_task(self, template, task, status, list_deps, tasks): 85 | """print a single task""" 86 | line_data = {'name': task.name, 'doc': task.doc} 87 | # FIXME group task status is never up-to-date 88 | if status: 89 | # FIXME: 'ignore' handling is ugly 90 | if self.dep_manager.status_is_ignore(task): 91 | task_status = 'ignore' 92 | else: 93 | task_status = self.dep_manager.get_status(task, tasks).status 94 | line_data['status'] = self.STATUS_MAP[task_status] 95 | 96 | self.outstream.write(template.format(**line_data)) 97 | 98 | # print dependencies 99 | if list_deps: 100 | for dep in task.file_dep: 101 | self.outstream.write(" - %s\n" % dep) 102 | self.outstream.write("\n") 103 | 104 | @staticmethod 105 | def _list_filtered(tasks, filter_tasks, include_subtasks): 106 | """return list of task based on selected 'filter_tasks' """ 107 | check_tasks_exist(tasks, filter_tasks) 108 | 109 | # get task by name 110 | print_list = [] 111 | for name in filter_tasks: 112 | task = tasks[name] 113 | print_list.append(task) 114 | if include_subtasks: 115 | print_list.extend(subtasks_iter(tasks, task)) 116 | return print_list 117 | 118 | 119 | def _list_all(self, include_subtasks): 120 | """list of tasks""" 121 | print_list = [] 122 | for task in self.task_list: 123 | if (not include_subtasks) and task.subtask_of: 124 | continue 125 | print_list.append(task) 126 | return print_list 127 | 128 | 129 | def _execute(self, subtasks=False, quiet=True, status=False, private=False, 130 | list_deps=False, template=None, sort='name', pos_args=None): 131 | """List task generators""" 132 | filter_tasks = pos_args 133 | # dict of all tasks 134 | tasks = dict([(t.name, t) for t in self.task_list]) 135 | 136 | if filter_tasks: 137 | # list only tasks passed on command line 138 | print_list = self._list_filtered(tasks, filter_tasks, subtasks) 139 | else: 140 | print_list = self._list_all(subtasks) 141 | 142 | # exclude private tasks 143 | if not private: 144 | print_list = [t for t in print_list if not t.name.startswith('_')] 145 | 146 | # set template 147 | if template is None: 148 | max_name_len = 0 149 | if print_list: 150 | max_name_len = max(len(t.name) for t in print_list) 151 | 152 | template = '{name:<' + str(max_name_len + 3) + '}' 153 | if not quiet: 154 | template += '{doc}' 155 | if status: 156 | template = '{status} ' + template 157 | template += '\n' 158 | 159 | # sort list of tasks 160 | if sort == 'name': 161 | print_list = sorted(print_list) 162 | elif sort == 'definition': 163 | pass # task list is already sorted in order of definition 164 | 165 | # print list of tasks 166 | for task in print_list: 167 | self._print_task(template, task, status, list_deps, tasks) 168 | return 0 169 | -------------------------------------------------------------------------------- /doit/cmd_resetdep.py: -------------------------------------------------------------------------------- 1 | from .cmd_base import DoitCmdBase, check_tasks_exist 2 | from .cmd_base import subtasks_iter 3 | import os 4 | 5 | 6 | class ResetDep(DoitCmdBase): 7 | name = "reset-dep" 8 | doc_purpose = ("recompute and save the state of file dependencies without " 9 | "executing actions") 10 | doc_usage = "[TASK ...]" 11 | cmd_options = () 12 | doc_description = """ 13 | This command allows to recompute the information on file dependencies 14 | (timestamp, md5sum, ... depending on the ``check_file_uptodate`` setting), and 15 | save this in the database, without executing the actions. 16 | 17 | The command run on all tasks by default, but it is possible to specify a list 18 | of tasks to work on. 19 | 20 | This is useful when the targets of your tasks already exist, and you want doit 21 | to consider your tasks as up-to-date. One use-case for this command is when you 22 | change the ``check_file_uptodate`` setting, which cause doit to consider all 23 | your tasks as not up-to-date. It is also useful if you start using doit while 24 | some of your data as already been computed, or when you add a file dependency 25 | to a task that has already run. 26 | """ 27 | 28 | def _execute(self, pos_args=None): 29 | filter_tasks = pos_args 30 | 31 | # dict of all tasks 32 | tasks = dict([(t.name, t) for t in self.task_list]) 33 | 34 | # select tasks that command will be applied to 35 | if filter_tasks: 36 | # list only tasks passed on command line 37 | check_tasks_exist(tasks, filter_tasks) 38 | # get task by name 39 | task_list = [] 40 | for name in filter_tasks: 41 | task = tasks[name] 42 | task_list.append(task) 43 | task_list.extend(subtasks_iter(tasks, task)) 44 | else: 45 | task_list = self.task_list 46 | 47 | write = self.outstream.write 48 | for task in task_list: 49 | # Get these now because dep_manager.get_status will remove the task 50 | # from the db if the checker changed. 51 | values = self.dep_manager.get_values(task.name) 52 | result = self.dep_manager.get_result(task.name) 53 | 54 | missing_deps = [dep for dep in task.file_dep 55 | if not os.path.exists(dep)] 56 | 57 | if len(missing_deps) > 0: 58 | deps = "', '".join(missing_deps) 59 | write(f"failed {task.name} (Dependent file '{deps}' does not exist.)\n") 60 | continue 61 | 62 | res = self.dep_manager.get_status(task, tasks) 63 | 64 | # An 'up-to-date' status means that it is useless to recompute the 65 | # state: file deps and targets exists, the state has not changed, 66 | # there is nothing more to do. 67 | if res.status == 'up-to-date': 68 | write("skip {}\n".format(task.name)) 69 | continue 70 | 71 | task.values = values 72 | self.dep_manager.save_success(task, result_hash=result) 73 | write("processed {}\n".format(task.name)) 74 | 75 | self.dep_manager.close() 76 | -------------------------------------------------------------------------------- /doit/cmd_strace.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import re 4 | 5 | from .exceptions import InvalidCommand 6 | from .action import CmdAction 7 | from .task import Task 8 | from .cmd_run import Run 9 | 10 | 11 | # filter to display only files from cwd 12 | opt_show_all = { 13 | 'name': 'show_all', 14 | 'short': 'a', 15 | 'long': 'all', 16 | 'type': bool, 17 | 'default': False, 18 | 'help': "display all files (not only from within CWD path)", 19 | } 20 | 21 | opt_keep_trace = { 22 | 'name': 'keep_trace', 23 | 'long': 'keep', 24 | 'type': bool, 25 | 'default': False, 26 | 'help': "save strace command output into strace.txt", 27 | } 28 | 29 | 30 | class Strace(Run): 31 | doc_purpose = "use strace to list file_deps and targets" 32 | doc_usage = "TASK" 33 | doc_description = """ 34 | The output is a list of files prefixed with 'R' for open in read mode 35 | or 'W' for open in write mode. 36 | The files are listed in chronological order. 37 | 38 | This is a debugging feature with many limitations. 39 | * can strace only one task at a time 40 | * can only strace CmdAction 41 | * the process being traced itself might have some kind of cache, 42 | that means it might not write a target file if it exist 43 | * does not handle chdir 44 | 45 | So this is NOT 100% reliable, use with care! 46 | """ 47 | 48 | cmd_options = (opt_show_all, opt_keep_trace) 49 | 50 | TRACE_CMD = "strace -f -e trace=file %s 2>>%s " 51 | TRACE_OUT = 'strace.txt' 52 | 53 | def execute(self, params, args): 54 | """remove existing output file if any and do sanity checking""" 55 | if os.path.exists(self.TRACE_OUT): # pragma: no cover 56 | os.unlink(self.TRACE_OUT) 57 | if len(args) != 1: 58 | msg = ('strace failed, must select *one* task to strace.' 59 | '\nCheck `{} help strace`.'.format(self.bin_name)) 60 | raise InvalidCommand(msg) 61 | result = Run.execute(self, params, args) 62 | if (not params['keep_trace']) and os.path.exists(self.TRACE_OUT): 63 | os.unlink(self.TRACE_OUT) 64 | return result 65 | 66 | def _execute(self, show_all): 67 | """1) wrap the original action with strace and save output in file 68 | 2) add a second task that will generate the report from temp file 69 | """ 70 | # find task to trace and wrap it 71 | selected = self.sel_tasks[0] 72 | for task in self.task_list: 73 | if task.name == selected: 74 | self.wrap_strace(task) 75 | break 76 | 77 | # add task to print report 78 | report_strace = Task( 79 | 'strace_report', 80 | actions=[(find_deps, [self.outstream, self.TRACE_OUT, show_all])], 81 | verbosity=2, 82 | task_dep=[selected], 83 | uptodate=[False], 84 | ) 85 | self.task_list.append(report_strace) 86 | self.sel_tasks.append(report_strace.name) 87 | 88 | # clear strace file 89 | return Run._execute(self, sys.stdout) 90 | 91 | 92 | @classmethod 93 | def wrap_strace(cls, task): 94 | """wrap task actions into strace command""" 95 | wrapped_actions = [] 96 | for action in task.actions: 97 | if isinstance(action, CmdAction): 98 | cmd = cls.TRACE_CMD % (action._action, cls.TRACE_OUT) 99 | wrapped = CmdAction(cmd, task, save_out=action.save_out) 100 | wrapped_actions.append(wrapped) 101 | else: 102 | wrapped_actions.append(action) 103 | task._action_instances = wrapped_actions 104 | # task should be always executed 105 | task._extend_uptodate([False]) 106 | 107 | 108 | def find_deps(outstream, strace_out, show_all): 109 | """read file witn strace output, return dict with deps, targets""" 110 | # 7978 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 111 | # get "mode" file was open, until ')' is closed 112 | # ignore rest of line 113 | # .*\( # ignore text until '(' 114 | # [^"]*" # ignore text until '"' 115 | # (?P[^"]*)" # get "file" name inside " 116 | # , (\[.*\])* # ignore elements if inside [] - used by execve 117 | # (?P[^)]*)\) # get mode opening file 118 | # = ].* # check syscall was successful""", 119 | regex = re.compile( 120 | r'.*\([^"]*"(?P[^"]*)",' 121 | + r' (\[.*\])*(?P[^)]*)\) = [^-].*') 122 | 123 | read = set() 124 | write = set() 125 | cwd = os.getcwd() 126 | if not os.path.exists(strace_out): 127 | return 128 | with open(strace_out) as text: 129 | for line in text: 130 | # ignore non file operation 131 | match = regex.match(line) 132 | if not match: 133 | continue 134 | rel_name = match.group('file') 135 | name = os.path.abspath(rel_name) 136 | 137 | # ignore files out of cwd 138 | if not show_all: 139 | if not name.startswith(cwd): 140 | continue 141 | 142 | if 'WR' in match.group('mode'): 143 | if name not in write: 144 | write.add(name) 145 | outstream.write("W %s\n" % name) 146 | else: 147 | if name not in read: 148 | read.add(name) 149 | outstream.write("R %s\n" % name) 150 | -------------------------------------------------------------------------------- /doit/exceptions.py: -------------------------------------------------------------------------------- 1 | """Handle exceptions generated from 'user' code""" 2 | 3 | import sys 4 | import traceback 5 | 6 | 7 | class InvalidCommand(Exception): 8 | """Invalid command line argument.""" 9 | def __init__(self, *args, **kwargs): 10 | self.not_found = kwargs.pop('not_found', None) 11 | super(InvalidCommand, self).__init__(*args, **kwargs) 12 | self.cmd_used = None 13 | self.bin_name = 'doit' # default but might be overwriten 14 | 15 | def __str__(self): 16 | if self.not_found is None: 17 | return super(InvalidCommand, self).__str__() 18 | 19 | if self.cmd_used: 20 | msg_task_not_found = ( 21 | 'command `{cmd_used}` invalid parameter: "{not_found}".' 22 | ' Must be a task, or a target.\n' 23 | 'Type "{bin_name} list" to see available tasks') 24 | return msg_task_not_found.format(**self.__dict__) 25 | else: 26 | msg_cmd_task_not_found = ( 27 | 'Invalid parameter: "{not_found}".' 28 | ' Must be a command, task, or a target.\n' 29 | 'Type "{bin_name} help" to see available commands.\n' 30 | 'Type "{bin_name} list" to see available tasks.\n') 31 | return msg_cmd_task_not_found.format(**self.__dict__) 32 | 33 | 34 | 35 | 36 | class InvalidDodoFile(Exception): 37 | """Invalid dodo file""" 38 | pass 39 | 40 | class InvalidTask(Exception): 41 | """Invalid task instance. User error on specifying the task.""" 42 | pass 43 | 44 | 45 | 46 | class CatchedException(): 47 | """DEPRECATED, use BaseFail instead. 2022-04-22 0.36.0 release. 48 | 49 | Wrong grammar and not all BaseFail contains an Exception 50 | 51 | :ivar report: used by (some) reporters to decide if Failure/Error should be in printed 52 | """ 53 | def __init__(self, msg, exception=None, report=True): 54 | self.message = msg 55 | self.traceback = '' 56 | self.report = report 57 | # It would be nice to include original exception, but they are not always pickable 58 | # https://stackoverflow.com/questions/49715881/how-to-pickle-inherited-exceptions 59 | 60 | if isinstance(exception, BaseFail): 61 | self.traceback = exception.traceback 62 | elif exception is not None: 63 | self.traceback = traceback.format_exception( 64 | exception.__class__, exception, sys.exc_info()[2]) 65 | 66 | def get_msg(self): 67 | """return full exception description (includes traceback)""" 68 | return "%s\n%s" % (self.message, "".join(self.traceback)) 69 | 70 | def get_name(self): 71 | """get fail kind name""" 72 | return self.__class__.__name__ 73 | 74 | def __repr__(self): 75 | return "(<%s> %s)" % (self.get_name(), self.message) 76 | 77 | def __str__(self): 78 | return "%s\n%s" % (self.get_name(), self.get_msg()) 79 | 80 | 81 | class BaseFail(CatchedException): 82 | """This used to save info Task failures/errors 83 | 84 | Might contain a caught Exception. 85 | """ 86 | pass 87 | 88 | class TaskFailed(BaseFail): 89 | """Task execution was not successful.""" 90 | pass 91 | 92 | 93 | class TaskError(BaseFail): 94 | """Error while trying to execute task.""" 95 | pass 96 | 97 | 98 | class UnmetDependency(TaskError): 99 | """Task was not executed because a dependent task failed or is ignored""" 100 | pass 101 | 102 | 103 | class SetupError(TaskError): 104 | """Error while trying to execute setup object""" 105 | pass 106 | 107 | 108 | class DependencyError(TaskError): 109 | """Error while trying to check if task is up-to-date or saving task status""" 110 | pass 111 | -------------------------------------------------------------------------------- /doit/globals.py: -------------------------------------------------------------------------------- 1 | """Simple registry of singletons.""" 2 | 3 | 4 | class Globals: 5 | """Accessors to doit singletons. 6 | 7 | :cvar dep_manager: (doit.dependency.Dependency) The doit dependency manager, 8 | holding all persistent task data. 9 | """ 10 | dep_manager = None 11 | -------------------------------------------------------------------------------- /doit/plugin.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import importlib 3 | 4 | 5 | def entry_points_impl(): 6 | # entry_points is available since 3.8 but "horrible inefficient" 7 | if sys.version_info < (3, 10): 8 | from importlib_metadata import entry_points 9 | else: 10 | from importlib.metadata import entry_points 11 | return entry_points 12 | 13 | 14 | class PluginEntry(object): 15 | """A Plugin entry point 16 | 17 | The entry-point is not loaded/imported on creation. 18 | Use the method `get()` to import the module and get the attribute. 19 | """ 20 | 21 | class Sentinel(object): 22 | pass 23 | 24 | # indicate the entry-point object is not loaded yet 25 | NOT_LOADED = Sentinel() 26 | 27 | def __init__(self, category, name, location): 28 | """ 29 | :param category str: plugin category name 30 | :param name str: plugin name (as used by doit) 31 | :param location str: python object location as : 32 | """ 33 | self.obj = self.NOT_LOADED 34 | self.category = category 35 | self.name = name 36 | self.location = location 37 | 38 | def __repr__(self): 39 | return "PluginEntry('{}', '{}', '{}')".format( 40 | self.category, self.name, self.location) 41 | 42 | def get(self): 43 | """return obj, get from cache or load""" 44 | if self.obj is self.NOT_LOADED: 45 | self.obj = self.load() 46 | return self.obj 47 | 48 | def load(self): 49 | """load/import reference to obj from named module/obj""" 50 | module_name, obj_name = self.location.split(':') 51 | try: 52 | module = importlib.import_module(module_name) 53 | except ImportError: 54 | raise Exception('Plugin {} module `{}` not found.'.format( 55 | self.category, module_name)) 56 | try: 57 | obj = getattr(module, obj_name) 58 | except AttributeError: 59 | raise Exception('Plugin {}:{} module `{}` has no {}.'.format( 60 | self.category, self.name, module_name, obj_name)) 61 | return obj 62 | 63 | 64 | class PluginDict(dict): 65 | """Item values *might* be a PluginEntry or a direct reference to class/obj. 66 | 67 | Values should not be accessed directly, use `get_plugin()` 68 | to make sure the plugin is loaded. 69 | 70 | Typically, one dict is created for each kind of plugin. 71 | doit supports 4 categories: 72 | - COMMAND 73 | - LOADER 74 | - BACKEND 75 | - REPORTER 76 | """ 77 | 78 | entry_point_prefix = 'doit' 79 | 80 | def add_plugins(self, cfg_data, category): 81 | """read all items from a ConfigParser section containing plugins & entry-points. 82 | 83 | Plugins from entry-point have higher priority 84 | """ 85 | self.update(self._from_ini(cfg_data, category)) 86 | self.update(self._from_entry_points(category)) 87 | 88 | def _from_ini(self, cfg_data, category): 89 | """plugins from INI file 90 | 91 | INI `section` names map exactly to plugin `category`. 92 | """ 93 | result = {} 94 | if category in cfg_data: 95 | for name, location in cfg_data[category].items(): 96 | result[name] = PluginEntry(category, name, location) 97 | return result 98 | 99 | def _from_entry_points(self, category): 100 | """get all plugins from setuptools entry_points""" 101 | result = {} 102 | group = f"{self.entry_point_prefix}.{category}" 103 | entry_points = entry_points_impl() 104 | for point in entry_points(group=group): 105 | name = point.name 106 | location = "{}:{}".format(point.module, point.attr) 107 | result[name] = PluginEntry(category, name, location) 108 | return result 109 | 110 | 111 | def get_plugin(self, key): 112 | """load and return a single plugin""" 113 | val = self[key] 114 | if isinstance(val, PluginEntry): 115 | val.name = key # overwrite obj name attribute 116 | return val.get() 117 | else: 118 | return val 119 | 120 | def to_dict(self): 121 | """return a standard dict with all plugins values loaded (no PluginEntry)""" 122 | return {k: self.get_plugin(k) for k in self.keys()} 123 | -------------------------------------------------------------------------------- /doit/version.py: -------------------------------------------------------------------------------- 1 | """doit version, defined out of __init__.py to avoid circular reference""" 2 | VERSION = (0, 37, 'dev0') 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | 3 | # E266 too many leading '#' for block comment 4 | # E301 expected 1 blank line 5 | # E302: expected 2 blank lines 6 | # E303: too many blank lines 7 | # E305: expected 2 blank lines after class or function definition 8 | # E306: expected 1 blank line before a nested definition 9 | # E731: do not assign lambda 10 | # W503: line break before binary operator 11 | ignore = E266, E301, E302, E303, E305, E306, E731, W503 12 | max-line-length = 90 13 | exclude = doit/cmd_completion.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import sys 4 | 5 | from setuptools import setup 6 | 7 | 8 | long_description = ''' 9 | *doit* comes from the idea of bringing the power of build-tools to execute any 10 | kind of task 11 | 12 | *doit* can be uses as a simple **Task Runner** allowing you to easily define ad hoc 13 | tasks, helping you to organize all your project related tasks in an unified 14 | easy-to-use & discoverable way. 15 | 16 | *doit* scales-up with an efficient execution model like a **build-tool**. 17 | *doit* creates a DAG (direct acyclic graph) and is able to cache task results. 18 | It ensures that only required tasks will be executed and in the correct order 19 | (aka incremental-builds). 20 | 21 | The *up-to-date* check to cache task results is not restricted to looking for 22 | file modification on dependencies. Nor it requires "target" files. 23 | So it is also suitable to handle **workflows** not handled by traditional build-tools. 24 | 25 | Tasks' dependencies and creation can be done dynamically during it is execution 26 | making it suitable to drive complex workflows and **pipelines**. 27 | 28 | *doit* is build with a plugin architecture allowing extensible commands, custom 29 | output, storage backend and "task loader". It also provides an API allowing 30 | users to create new applications/tools leveraging *doit* functionality like a framework. 31 | 32 | *doit* is a mature project being actively developed for more than 10 years. 33 | It includes several extras like: parallel execution, auto execution (watch for file 34 | changes), shell tab-completion, DAG visualisation, IPython integration, and more. 35 | 36 | 37 | 38 | Sample Code 39 | =========== 40 | 41 | Define functions returning python dict with task's meta-data. 42 | 43 | Snippet from `tutorial `_: 44 | 45 | .. code:: python 46 | 47 | def task_imports(): 48 | """find imports from a python module""" 49 | for name, module in PKG_MODULES.by_name.items(): 50 | yield { 51 | 'name': name, 52 | 'file_dep': [module.path], 53 | 'actions': [(get_imports, (PKG_MODULES, module.path))], 54 | } 55 | 56 | def task_dot(): 57 | """generate a graphviz's dot graph from module imports""" 58 | return { 59 | 'targets': ['requests.dot'], 60 | 'actions': [module_to_dot], 61 | 'getargs': {'imports': ('imports', 'modules')}, 62 | 'clean': True, 63 | } 64 | 65 | def task_draw(): 66 | """generate image from a dot file""" 67 | return { 68 | 'file_dep': ['requests.dot'], 69 | 'targets': ['requests.png'], 70 | 'actions': ['dot -Tpng %(dependencies)s -o %(targets)s'], 71 | 'clean': True, 72 | } 73 | 74 | 75 | Run from terminal:: 76 | 77 | $ doit list 78 | dot generate a graphviz's dot graph from module imports 79 | draw generate image from a dot file 80 | imports find imports from a python module 81 | $ doit 82 | . imports:requests.models 83 | . imports:requests.__init__ 84 | . imports:requests.help 85 | (...) 86 | . dot 87 | . draw 88 | 89 | 90 | Project Details 91 | =============== 92 | 93 | - Website & docs - `http://pydoit.org `_ 94 | - Project management on github - `https://github.com/pydoit/doit `_ 95 | - Discussion group - `https://groups.google.com/forum/#!forum/python-doit `_ 96 | - News/twitter - `https://twitter.com/pydoit `_ 97 | - Plugins, extensions and projects based on doit - `https://github.com/pydoit/doit/wiki/powered-by-doit `_ 98 | 99 | license 100 | ======= 101 | 102 | The MIT License 103 | Copyright (c) 2008-2022 Eduardo Naufel Schettino 104 | ''' 105 | 106 | setup(name = 'doit', 107 | description = 'doit - Automation Tool', 108 | version = '0.37.dev0', 109 | license = 'MIT', 110 | author = 'Eduardo Naufel Schettino', 111 | author_email = 'schettino72@gmail.com', 112 | url = 'http://pydoit.org', 113 | classifiers = [ 114 | 'Development Status :: 5 - Production/Stable', 115 | 'Environment :: Console', 116 | 'License :: OSI Approved :: MIT License', 117 | 'Natural Language :: English', 118 | 'Operating System :: OS Independent', 119 | 'Operating System :: POSIX', 120 | 'Programming Language :: Python :: 3', 121 | 'Programming Language :: Python :: 3.8', 122 | 'Programming Language :: Python :: 3.9', 123 | 'Programming Language :: Python :: 3.10', 124 | 'Intended Audience :: Developers', 125 | 'Intended Audience :: Information Technology', 126 | 'Intended Audience :: Science/Research', 127 | 'Intended Audience :: System Administrators', 128 | 'Topic :: Software Development :: Build Tools', 129 | 'Topic :: Software Development :: Testing', 130 | 'Topic :: Software Development :: Quality Assurance', 131 | 'Topic :: Scientific/Engineering', 132 | ], 133 | keywords = "build make task automation pipeline task-runner", 134 | project_urls = { 135 | 'Documentation': 'https://pydoit.org/', 136 | 'Source': 'https://github.com/pydoit/doit/', 137 | 'Tracker': 'https://github.com/pydoit/doit/issues', 138 | }, 139 | packages = ['doit'], 140 | python_requires='>=3.8', 141 | install_requires = ['importlib-metadata>=4.4'], 142 | extras_require={ 143 | 'toml': ['tomli; python_version<"3.11"'], 144 | # cloudpickle broken on pypy, see #409 145 | 'cloudpickle': ['cloudpickle; platform_python_implementation!="pypy"'], 146 | }, 147 | long_description = long_description, 148 | entry_points = { 149 | 'console_scripts': [ 150 | 'doit = doit.__main__:main' 151 | ] 152 | }, 153 | ) 154 | -------------------------------------------------------------------------------- /tests/Dockerfile: -------------------------------------------------------------------------------- 1 | # Run/test doit on debian unstable 2 | 3 | # docker build -t doit-debian . 4 | # docker run -it --cap-add SYS_PTRACE -v /home/eduardo/work/doit/dev:/root/doit doit-debian 5 | 6 | # pip3 install -e . 7 | # pip3 install -r dev_requirements.txt 8 | 9 | 10 | from debian:unstable 11 | 12 | RUN apt-get update && apt-get install eatmydata --no-install-recommends -y 13 | RUN eatmydata apt-get install python3-pytest python3-pip -y 14 | RUN apt-get install python3-gdbm strace -y 15 | 16 | WORKDIR /root/doit 17 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pydoit/doit/00c136f5dfe7e9039d0fed6dddd6d45c84c307b4/tests/__init__.py -------------------------------------------------------------------------------- /tests/data/README: -------------------------------------------------------------------------------- 1 | this folder is used to keep some temporary files used on tests. 2 | -------------------------------------------------------------------------------- /tests/loader_sample.py: -------------------------------------------------------------------------------- 1 | 2 | DOIT_CONFIG = {'verbose': 2} 3 | 4 | 5 | def task_xxx1(): 6 | """task doc""" 7 | return { 8 | 'actions': ['do nothing'], 9 | 'params': [{'name':'p1', 'default':'1', 'short':'p'}], 10 | } 11 | 12 | def task_yyy2(): 13 | return {'actions':None} 14 | 15 | def bad_seed(): # pragma: no cover 16 | pass 17 | -------------------------------------------------------------------------------- /tests/module_with_tasks.py: -------------------------------------------------------------------------------- 1 | """ModuleLoadTest uses this file to load tasks from module.""" 2 | 3 | DOIT_CONFIG = dict(verbose=2) 4 | 5 | 6 | def task_xxx1(): 7 | return dict(actions=[]) 8 | 9 | 10 | task_no = 'strings are not tasks' 11 | 12 | 13 | def blabla(): 14 | ... # pragma: no cover 15 | -------------------------------------------------------------------------------- /tests/myecho.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # tests on CmdTask will use this script as an external process. 4 | # just print out all arguments 5 | 6 | import sys 7 | 8 | if __name__ == "__main__": 9 | print(" ".join(sys.argv[1:])) 10 | sys.exit(0) 11 | -------------------------------------------------------------------------------- /tests/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.doit] 2 | optx = "2" 3 | opty = "3" 4 | 5 | [tool.doit.plugins.command] 6 | bar = "tests.sample_plugin:MyCmd" 7 | -------------------------------------------------------------------------------- /tests/sample.cfg: -------------------------------------------------------------------------------- 1 | [GLOBAL] 2 | optx = 6 3 | opty = 7 4 | 5 | [COMMAND] 6 | foo = tests.sample_plugin:MyCmd 7 | 8 | -------------------------------------------------------------------------------- /tests/sample.toml: -------------------------------------------------------------------------------- 1 | optx = "6" 2 | opty = "7" 3 | 4 | [plugins.command] 5 | foo = "tests.sample_plugin:MyCmd" 6 | 7 | [commands.foo] 8 | nval = 33 9 | 10 | [tasks.bar] 11 | opt = "baz" 12 | -------------------------------------------------------------------------------- /tests/sample_md5.txt: -------------------------------------------------------------------------------- 1 | MD5SUM(1) User Commands MD5SUM(1) 2 | 3 | 4 | 5 | NAME 6 | md5sum - compute and check MD5 message digest 7 | 8 | SYNOPSIS 9 | md5sum [OPTION] [FILE]... 10 | 11 | DESCRIPTION 12 | Print or check MD5 (128-bit) checksums. With no FILE, or when FILE is 13 | -, read standard input. 14 | 15 | -b, --binary 16 | read in binary mode 17 | 18 | -c, --check 19 | read MD5 sums from the FILEs and check them 20 | 21 | -t, --text 22 | read in text mode (default) 23 | 24 | The following two options are useful only when verifying checksums: 25 | --status 26 | don’t output anything, status code shows success 27 | 28 | -w, --warn 29 | warn about improperly formatted checksum lines 30 | 31 | --help display this help and exit 32 | 33 | --version 34 | output version information and exit 35 | 36 | The sums are computed as described in RFC 1321. When checking, the 37 | input should be a former output of this program. The default mode is 38 | to print a line with checksum, a character indicating type (‘*’ for 39 | binary, ‘ ’ for text), and name for each FILE. 40 | 41 | AUTHOR 42 | Written by Ulrich Drepper, Scott Miller, and David Madore. 43 | 44 | REPORTING BUGS 45 | Report bugs to . 46 | 47 | COPYRIGHT 48 | Copyright © 2006 Free Software Foundation, Inc. 49 | This is free software. You may redistribute copies of it under the 50 | terms of the GNU General Public License 51 | . There is NO WARRANTY, to the 52 | extent permitted by law. 53 | 54 | SEE ALSO 55 | The full documentation for md5sum is maintained as a Texinfo manual. 56 | If the info and md5sum programs are properly installed at your site, 57 | the command 58 | 59 | info md5sum 60 | 61 | should give you access to the complete manual. 62 | 63 | 64 | 65 | md5sum 5.97 September 2007 MD5SUM(1) 66 | -------------------------------------------------------------------------------- /tests/sample_plugin.py: -------------------------------------------------------------------------------- 1 | from doit.cmd_base import Command 2 | 3 | class MyCmd(Command): 4 | name = 'mycmd' 5 | doc_purpose = 'test extending doit commands' 6 | doc_usage = '[XXX]' 7 | doc_description = 'my command description' 8 | 9 | def execute(self, opt_values, pos_args): # pragma: no cover 10 | print("this command does nothing!") 11 | 12 | 13 | ############## 14 | 15 | from doit.task import dict_to_task 16 | from doit.cmd_base import TaskLoader2 17 | 18 | my_builtin_task = { 19 | 'name': 'sample_task', 20 | 'actions': ['echo hello from built in'], 21 | 'doc': 'sample doc', 22 | } 23 | 24 | class MyLoader(TaskLoader2): 25 | 26 | def load_doit_config(self): 27 | return {'verbosity': 2} 28 | 29 | def load_tasks(self, cmd, pos_args): 30 | return [dict_to_task(my_builtin_task)] 31 | 32 | -------------------------------------------------------------------------------- /tests/sample_process.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # tests on CmdTask will use this script as an external process. 4 | # - If you call this script with 3 or more arguments, the process returns 5 | # exit code (166). 6 | # - If you call this script with arguments "please fail", it returns exit 7 | # code (11). 8 | # - If you call this script with arguments "check env", it verifies the 9 | # existence of an environment variable called "GELKIPWDUZLOVSXE", with 10 | # value "1". If the variable is not found, the process returns exit code (99). 11 | # - Otherwise, any first argument gets written to STDOUT. Any second argument 12 | # gets written to STDERR. 13 | 14 | import os 15 | import sys 16 | 17 | if __name__ == "__main__": 18 | # error 19 | if len(sys.argv) > 3: 20 | sys.exit(166) 21 | # fail 22 | if len(sys.argv) == 3 and sys.argv[1]=='please' and sys.argv[2]=='fail': 23 | sys.stdout.write("out ouch") 24 | sys.stderr.write("err output on failure") 25 | sys.exit(11) 26 | # check env 27 | if len(sys.argv) == 3 and sys.argv[1]=='check' and sys.argv[2]=='env': 28 | if os.environ.get('GELKIPWDUZLOVSXE') == '1': 29 | sys.exit(0) 30 | else: 31 | sys.exit(99) 32 | # ok 33 | if len(sys.argv) > 1: 34 | sys.stdout.write(sys.argv[1]) 35 | if len(sys.argv) > 2: 36 | sys.stderr.write(sys.argv[2]) 37 | sys.exit(0) 38 | -------------------------------------------------------------------------------- /tests/test___init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import doit 4 | from doit.loader import get_module 5 | 6 | 7 | def test_get_initial_workdir(restore_cwd): 8 | initial_wd = os.getcwd() 9 | fileName = os.path.join(os.path.dirname(__file__),"loader_sample.py") 10 | cwd = os.path.normpath(os.path.join(os.path.dirname(__file__), "data")) 11 | assert cwd != initial_wd # make sure test is not too easy 12 | get_module(fileName, cwd) 13 | assert os.getcwd() == cwd, os.getcwd() 14 | assert doit.get_initial_workdir() == initial_wd 15 | 16 | -------------------------------------------------------------------------------- /tests/test___main__.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from sys import executable 3 | 4 | 5 | def test_execute(depfile_name): 6 | assert 0 == subprocess.call([executable, '-m', 'doit', 'list', 7 | '--db-file', depfile_name]) 8 | -------------------------------------------------------------------------------- /tests/test_api.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from doit.cmd_base import ModuleTaskLoader 4 | from doit.api import run, run_tasks 5 | 6 | 7 | def test_run(monkeypatch, depfile_name): 8 | monkeypatch.setattr(sys, 'argv', ['did', '--db-file', depfile_name]) 9 | try: 10 | def hi(): 11 | print('hi') 12 | def task_hi(): 13 | return {'actions': [hi]} 14 | run(locals()) 15 | except SystemExit as err: 16 | assert err.code == 0 17 | else: # pragma: no cover 18 | assert False 19 | 20 | 21 | 22 | def _dodo(): 23 | """sample tasks""" 24 | def hi(opt=None): 25 | print('hi', opt) 26 | 27 | def task_hi(): 28 | return { 29 | 'actions': [hi], 30 | 'params': [{'name': 'opt', 'default': '1'}], 31 | } 32 | def task_two(): 33 | def my_error(): 34 | return False 35 | return { 36 | 'actions': [my_error], 37 | } 38 | 39 | return { 40 | 'task_hi': task_hi, 41 | 'task_two': task_two, 42 | } 43 | 44 | 45 | def test_run_tasks_success(capsys, depfile_name): 46 | result = run_tasks( 47 | ModuleTaskLoader(_dodo()), 48 | {'hi': {'opt': '3'}}, 49 | extra_config = { 50 | 'GLOBAL': { 51 | 'verbosity': 2, 52 | 'dep_file': depfile_name, 53 | }, 54 | }, 55 | ) 56 | assert result == 0 57 | out = capsys.readouterr().out 58 | assert out.strip() == 'hi 3' 59 | 60 | 61 | def test_run_tasks_error(capsys, depfile_name): 62 | result = run_tasks( 63 | ModuleTaskLoader(_dodo()), 64 | {'two': None}, 65 | extra_config = { 66 | 'GLOBAL': { 67 | 'verbosity': 2, 68 | 'dep_file': depfile_name, 69 | }, 70 | }, 71 | ) 72 | assert result == 1 73 | 74 | 75 | 76 | 77 | def test_run_tasks_pos(capsys, depfile_name): 78 | def _dodo_pos(): 79 | """sample tasks""" 80 | def hi(opt, pos): 81 | print(f'hi:{opt}--{pos}') 82 | 83 | def task_hi(): 84 | return { 85 | 'actions': [hi], 86 | 'params': [{'name': 'opt', 'default': '1'}], 87 | 'pos_arg': 'pos', 88 | } 89 | return { 90 | 'task_hi': task_hi, 91 | } 92 | 93 | tasks_selection = {'hi': {'opt': '3', 'pos': 'foo bar baz'}} 94 | extra_config = { 95 | 'GLOBAL': { 96 | 'verbosity': 2, 97 | 'dep_file': depfile_name, 98 | }, 99 | } 100 | result = run_tasks(ModuleTaskLoader(_dodo_pos()), tasks_selection, extra_config=extra_config) 101 | assert result == 0 102 | out = capsys.readouterr().out 103 | assert out.strip() == 'hi:3--foo bar baz' 104 | -------------------------------------------------------------------------------- /tests/test_cmd_completion.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | import pytest 3 | 4 | from doit.exceptions import InvalidCommand 5 | from doit.cmdparse import CmdOption 6 | from doit.plugin import PluginDict 7 | from doit.task import Task 8 | from doit.cmd_base import Command, DodoTaskLoader, TaskLoader2 9 | from doit.cmd_completion import TabCompletion 10 | from doit.cmd_help import Help 11 | from .conftest import CmdFactory 12 | 13 | # doesnt test the shell scripts. just test its creation! 14 | 15 | 16 | class FakeLoader2(TaskLoader2): 17 | def load_doit_config(self): 18 | return {} 19 | 20 | def load_tasks(self, cmd, pos_args): 21 | task_list = [ 22 | Task("t1", None, ), 23 | Task("t2", None, task_dep=['t2:a'], has_subtask=True, ), 24 | Task("t2:a", None, subtask_of='t2'), 25 | ] 26 | return task_list 27 | 28 | @pytest.fixture 29 | def commands(request): 30 | sub_cmds = {} 31 | sub_cmds['tabcompletion'] = TabCompletion 32 | sub_cmds['help'] = Help 33 | return PluginDict(sub_cmds) 34 | 35 | def test_invalid_shell_option(): 36 | cmd = CmdFactory(TabCompletion) 37 | pytest.raises(InvalidCommand, cmd.execute, 38 | {'shell':'another_shell', 'hardcode_tasks': False}, []) 39 | 40 | 41 | class TestCmdCompletionBash(object): 42 | 43 | def test_with_dodo__dynamic_tasks(self, commands): 44 | output = StringIO() 45 | cmd = CmdFactory(TabCompletion, task_loader=DodoTaskLoader(), 46 | outstream=output, cmds=commands) 47 | cmd.execute({'shell':'bash', 'hardcode_tasks': False}, []) 48 | got = output.getvalue() 49 | assert 'dodof' in got 50 | assert 't1' not in got 51 | assert 'tabcompletion' in got 52 | 53 | @pytest.mark.parametrize('loader_class', [FakeLoader2]) 54 | def test_no_dodo__hardcoded_tasks(self, commands, loader_class): 55 | output = StringIO() 56 | cmd = CmdFactory(TabCompletion, task_loader=loader_class(), 57 | outstream=output, cmds=commands) 58 | cmd.execute({'shell':'bash', 'hardcode_tasks': True}, []) 59 | got = output.getvalue() 60 | assert 'dodo.py' not in got 61 | assert 't1' in got 62 | 63 | def test_cmd_takes_file_args(self, commands): 64 | output = StringIO() 65 | cmd = CmdFactory(TabCompletion, task_loader=FakeLoader2(), 66 | outstream=output, cmds=commands) 67 | cmd.execute({'shell':'bash', 'hardcode_tasks': False}, []) 68 | got = output.getvalue() 69 | assert """help) 70 | COMPREPLY=( $(compgen -W "${tasks} ${sub_cmds}" -- $cur) ) 71 | return 0""" in got 72 | assert """tabcompletion) 73 | COMPREPLY=( $(compgen -f -- $cur) ) 74 | return 0""" in got 75 | 76 | 77 | class TestCmdCompletionZsh(object): 78 | 79 | def test_zsh_arg_line(self): 80 | opt1 = CmdOption({'name':'o1', 'default':'', 'help':'my desc'}) 81 | assert '' == TabCompletion._zsh_arg_line(opt1) 82 | 83 | opt2 = CmdOption({'name':'o2', 'default':'', 'help':'my desc', 84 | 'short':'s'}) 85 | assert '"-s[my desc]" \\' == TabCompletion._zsh_arg_line(opt2) 86 | 87 | opt3 = CmdOption({'name':'o3', 'default':'', 'help':'my desc', 88 | 'long':'lll'}) 89 | assert '"--lll[my desc]" \\' == TabCompletion._zsh_arg_line(opt3) 90 | 91 | opt4 = CmdOption({'name':'o4', 'default':'', 'help':'my desc [b]a', 92 | 'short':'s', 'long':'lll'}) 93 | assert ('"(-s|--lll)"{-s,--lll}"[my desc [b\\]a]" \\' == 94 | TabCompletion._zsh_arg_line(opt4)) 95 | 96 | # escaping `"` test 97 | opt5 = CmdOption({'name':'o5', 'default':'', 98 | 'help':'''my "des'c [b]a''', 99 | 'short':'s', 'long':'lll'}) 100 | assert ('''"(-s|--lll)"{-s,--lll}"[my \\"des'c [b\\]a]" \\''' == 101 | TabCompletion._zsh_arg_line(opt5)) 102 | 103 | 104 | def test_cmd_arg_list(self): 105 | no_args = TabCompletion._zsh_arg_list(Command()) 106 | assert "'*::task:(($tasks))'" not in no_args 107 | assert "'::cmd:(($commands))'" not in no_args 108 | 109 | class CmdTakeTasks(Command): 110 | doc_usage = '[TASK ...]' 111 | with_task_args = TabCompletion._zsh_arg_list(CmdTakeTasks()) 112 | assert "'*::task:(($tasks))'" in with_task_args 113 | assert "'::cmd:(($commands))'" not in with_task_args 114 | 115 | class CmdTakeCommands(Command): 116 | doc_usage = '[COMMAND ...]' 117 | with_cmd_args = TabCompletion._zsh_arg_list(CmdTakeCommands()) 118 | assert "'*::task:(($tasks))'" not in with_cmd_args 119 | assert "'::cmd:(($commands))'" in with_cmd_args 120 | 121 | 122 | def test_cmds_with_params(self, commands): 123 | output = StringIO() 124 | cmd = CmdFactory(TabCompletion, task_loader=DodoTaskLoader(), 125 | outstream=output, cmds=commands) 126 | cmd.execute({'shell':'zsh', 'hardcode_tasks': False}, []) 127 | got = output.getvalue() 128 | assert "tabcompletion: generate script" in got 129 | 130 | @pytest.mark.parametrize('loader_class', [FakeLoader2]) 131 | def test_hardcoded_tasks(self, commands, loader_class): 132 | output = StringIO() 133 | cmd = CmdFactory(TabCompletion, task_loader=loader_class(), 134 | outstream=output, cmds=commands) 135 | cmd.execute({'shell':'zsh', 'hardcode_tasks': True}, []) 136 | got = output.getvalue() 137 | assert 't1' in got 138 | -------------------------------------------------------------------------------- /tests/test_cmd_dumpdb.py: -------------------------------------------------------------------------------- 1 | from dbm import whichdb 2 | 3 | import pytest 4 | 5 | from doit.cmd_dumpdb import DumpDB 6 | 7 | class TestCmdDumpDB(object): 8 | 9 | def testDefault(self, capsys, dep_manager): 10 | # cmd_main(["help", "task"]) 11 | dep_manager._set('tid', 'my_dep', 'xxx') 12 | dep_manager.close() 13 | dbm_kind = whichdb(dep_manager.name) 14 | if dbm_kind in ('dbm', 'dbm.ndbm'): # pragma: no cover 15 | pytest.skip(f'"{dbm_kind}" not supported for this operation') 16 | cmd_dump = DumpDB() 17 | cmd_dump.execute({'dep_file': dep_manager.name}, []) 18 | out, err = capsys.readouterr() 19 | assert 'tid' in out 20 | assert 'my_dep' in out 21 | assert 'xxx' in out 22 | -------------------------------------------------------------------------------- /tests/test_cmd_forget.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | 3 | import pytest 4 | 5 | from doit.exceptions import InvalidCommand 6 | from doit.dependency import DbmDB, Dependency 7 | from doit.cmd_forget import Forget 8 | from .conftest import tasks_sample, CmdFactory 9 | 10 | 11 | class TestCmdForget(object): 12 | 13 | @pytest.fixture 14 | def tasks(self, request): 15 | return tasks_sample() 16 | 17 | @staticmethod 18 | def _add_task_deps(tasks, testdb): 19 | """put some data on testdb""" 20 | dep = Dependency(DbmDB, testdb) 21 | for task in tasks: 22 | dep._set(task.name,"dep","1") 23 | dep.close() 24 | 25 | dep2 = Dependency(DbmDB, testdb) 26 | assert "1" == dep2._get("g1.a", "dep") 27 | dep2.close() 28 | 29 | 30 | def testForgetDefault(self, tasks, depfile_name): 31 | self._add_task_deps(tasks, depfile_name) 32 | output = StringIO() 33 | cmd_forget = CmdFactory(Forget, outstream=output, dep_file=depfile_name, 34 | backend='dbm', task_list=tasks, sel_tasks=['t1', 't2'], ) 35 | cmd_forget._execute(False, False, False) 36 | got = output.getvalue().split("\n")[:-1] 37 | assert ["forgetting t1", "forgetting t2"] == got, repr(output.getvalue()) 38 | dep = Dependency(DbmDB, depfile_name) 39 | assert None == dep._get('t1', "dep") 40 | assert None == dep._get('t2', "dep") 41 | assert '1' == dep._get('t3', "dep") 42 | 43 | def testForgetAll(self, tasks, depfile_name): 44 | self._add_task_deps(tasks, depfile_name) 45 | output = StringIO() 46 | cmd_forget = CmdFactory(Forget, outstream=output, dep_file=depfile_name, 47 | backend='dbm', task_list=tasks, sel_tasks=[]) 48 | cmd_forget._execute(False, False, True) 49 | got = output.getvalue().split("\n")[:-1] 50 | assert ["forgetting all tasks"] == got, repr(output.getvalue()) 51 | dep = Dependency(DbmDB, depfile_name) 52 | for task in tasks: 53 | assert None == dep._get(task.name, "dep") 54 | 55 | def testDisableDefault(self, tasks, depfile_name): 56 | self._add_task_deps(tasks, depfile_name) 57 | output = StringIO() 58 | cmd_forget = CmdFactory(Forget, outstream=output, dep_file=depfile_name, 59 | backend='dbm', task_list=tasks, 60 | sel_tasks=['t1', 't2'], sel_default_tasks=True) 61 | cmd_forget._execute(False, True, False) 62 | got = output.getvalue().split("\n")[:-1] 63 | assert ["no tasks specified, pass task name, --enable-default or --all"] == got, ( 64 | repr(output.getvalue())) 65 | dep = Dependency(DbmDB, depfile_name) 66 | for task in tasks: 67 | assert "1" == dep._get(task.name, "dep") 68 | 69 | def testForgetOne(self, tasks, depfile_name): 70 | self._add_task_deps(tasks, depfile_name) 71 | output = StringIO() 72 | cmd_forget = CmdFactory(Forget, outstream=output, dep_file=depfile_name, 73 | backend='dbm', task_list=tasks, 74 | sel_tasks=["t2", "t1"]) 75 | cmd_forget._execute(False, True, False) 76 | got = output.getvalue().split("\n")[:-1] 77 | assert ["forgetting t2", "forgetting t1"] == got 78 | dep = Dependency(DbmDB, depfile_name) 79 | assert None == dep._get("t1", "dep") 80 | assert None == dep._get("t2", "dep") 81 | assert "1" == dep._get("g1.a", "dep") 82 | 83 | def testForgetGroup(self, tasks, depfile_name): 84 | self._add_task_deps(tasks, depfile_name) 85 | output = StringIO() 86 | cmd_forget = CmdFactory( 87 | Forget, outstream=output, dep_file=depfile_name, 88 | backend='dbm', task_list=tasks, sel_tasks=["g1"]) 89 | cmd_forget._execute(False, False, False) 90 | got = output.getvalue().split("\n")[:-1] 91 | assert "forgetting g1" == got[0] 92 | 93 | dep = Dependency(DbmDB, depfile_name) 94 | assert "1" == dep._get("t1", "dep") 95 | assert "1" == dep._get("t2", "dep") 96 | assert None == dep._get("g1", "dep") 97 | assert None == dep._get("g1.a", "dep") 98 | assert None == dep._get("g1.b", "dep") 99 | 100 | 101 | def testForgetTaskDependency(self, tasks, depfile_name): 102 | self._add_task_deps(tasks, depfile_name) 103 | output = StringIO() 104 | cmd_forget = CmdFactory( 105 | Forget, outstream=output, dep_file=depfile_name, 106 | backend='dbm', task_list=tasks, sel_tasks=["t3"]) 107 | cmd_forget._execute(True, False, False) 108 | dep = Dependency(DbmDB, depfile_name) 109 | assert None == dep._get("t3", "dep") 110 | assert None == dep._get("t1", "dep") 111 | 112 | # if task dependency not from a group dont forget it 113 | def testDontForgetTaskDependency(self, tasks, depfile_name): 114 | self._add_task_deps(tasks, depfile_name) 115 | output = StringIO() 116 | cmd_forget = CmdFactory( 117 | Forget, outstream=output, dep_file=depfile_name, 118 | backend='dbm', task_list=tasks, sel_tasks=["t3"]) 119 | cmd_forget._execute(False, False, False) 120 | dep = Dependency(DbmDB, depfile_name) 121 | assert None == dep._get("t3", "dep") 122 | assert "1" == dep._get("t1", "dep") 123 | 124 | def testForgetInvalid(self, tasks, depfile_name): 125 | self._add_task_deps(tasks, depfile_name) 126 | output = StringIO() 127 | cmd_forget = CmdFactory( 128 | Forget, outstream=output, dep_file=depfile_name, 129 | backend='dbm', task_list=tasks, sel_tasks=["XXX"]) 130 | pytest.raises(InvalidCommand, cmd_forget._execute, False, False, False) 131 | -------------------------------------------------------------------------------- /tests/test_cmd_help.py: -------------------------------------------------------------------------------- 1 | from doit.doit_cmd import DoitMain 2 | 3 | 4 | def cmd_main(args, extra_config=None, bin_name='doit'): 5 | if extra_config: 6 | extra_config = {'GLOBAL': extra_config} 7 | main = DoitMain(extra_config=extra_config) 8 | main.BIN_NAME = bin_name 9 | return main.run(args) 10 | 11 | 12 | class TestHelp(object): 13 | def test_help_usage(self, capsys): 14 | returned = cmd_main(["help"]) 15 | assert returned == 0 16 | out, err = capsys.readouterr() 17 | assert "doit list" in out 18 | 19 | def test_help_usage_custom_name(self, capsys): 20 | returned = cmd_main(["help"], bin_name='mytool') 21 | assert returned == 0 22 | out, err = capsys.readouterr() 23 | assert "mytool list" in out 24 | 25 | def test_help_plugin_name(self, capsys): 26 | plugin = {'XXX': 'tests.sample_plugin:MyCmd'} 27 | main = DoitMain(extra_config={'COMMAND':plugin}) 28 | main.BIN_NAME = 'doit' 29 | returned = main.run(["help"]) 30 | assert returned == 0 31 | out, err = capsys.readouterr() 32 | assert "doit XXX " in out 33 | assert "test extending doit commands" in out, out 34 | 35 | def test_help_task_params(self, capsys): 36 | returned = cmd_main(["help", "task"]) 37 | assert returned == 0 38 | out, err = capsys.readouterr() 39 | assert "Task Dictionary parameters" in out 40 | 41 | def test_help_cmd(self, capsys): 42 | returned = cmd_main(["help", "list"], {'dep_file': 'foo.db'}) 43 | assert returned == 0 44 | out, err = capsys.readouterr() 45 | assert "PURPOSE" in out 46 | assert "list tasks from dodo file" in out 47 | # overwritten defaults, are shown as default 48 | assert "file used to save successful runs [default: foo.db]" in out 49 | 50 | def test_help_task_name(self, capsys, restore_cwd, depfile_name): 51 | returned = cmd_main(["help", "-f", "tests/loader_sample.py", 52 | "--db-file", depfile_name, "xxx1"]) 53 | assert returned == 0 54 | out, err = capsys.readouterr() 55 | assert "xxx1" in out # name 56 | assert "task doc" in out # doc 57 | assert "-p" in out # params 58 | 59 | def test_help_wrong_name(self, capsys, restore_cwd, depfile_name): 60 | returned = cmd_main(["help", "-f", "tests/loader_sample.py", 61 | "--db-file", depfile_name, "wrong_name"]) 62 | assert returned == 0 # TODO return different value? 63 | out, err = capsys.readouterr() 64 | assert "doit list" in out 65 | 66 | def test_help_no_dodo_file(self, capsys): 67 | returned = cmd_main(["help", "-f", "no_dodo", "wrong_name"]) 68 | assert returned == 0 # TODO return different value? 69 | out, err = capsys.readouterr() 70 | assert "doit list" in out 71 | 72 | -------------------------------------------------------------------------------- /tests/test_cmd_ignore.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | 3 | import pytest 4 | 5 | from doit.exceptions import InvalidCommand 6 | from doit.dependency import DbmDB, Dependency 7 | from doit.cmd_ignore import Ignore 8 | from .conftest import tasks_sample, CmdFactory 9 | 10 | 11 | class TestCmdIgnore(object): 12 | 13 | @pytest.fixture 14 | def tasks(self, request): 15 | return tasks_sample() 16 | 17 | def testIgnoreAll(self, tasks, dep_manager): 18 | output = StringIO() 19 | cmd = CmdFactory(Ignore, outstream=output, dep_manager=dep_manager, 20 | task_list=tasks) 21 | cmd._execute([]) 22 | got = output.getvalue().split("\n")[:-1] 23 | assert ["You cant ignore all tasks! Please select a task."] == got, got 24 | for task in tasks: 25 | assert None == dep_manager._get(task.name, "ignore:") 26 | 27 | def testIgnoreOne(self, tasks, dep_manager): 28 | output = StringIO() 29 | cmd = CmdFactory(Ignore, outstream=output, dep_manager=dep_manager, 30 | task_list=tasks) 31 | cmd._execute(["t2", "t1"]) 32 | got = output.getvalue().split("\n")[:-1] 33 | assert ["ignoring t2", "ignoring t1"] == got 34 | dep = Dependency(DbmDB, dep_manager.name) 35 | assert '1' == dep._get("t1", "ignore:") 36 | assert '1' == dep._get("t2", "ignore:") 37 | assert None == dep._get("t3", "ignore:") 38 | 39 | def testIgnoreGroup(self, tasks, dep_manager): 40 | output = StringIO() 41 | cmd = CmdFactory(Ignore, outstream=output, dep_manager=dep_manager, 42 | task_list=tasks) 43 | cmd._execute(["g1"]) 44 | got = output.getvalue().split("\n")[:-1] 45 | 46 | dep = Dependency(DbmDB, dep_manager.name) 47 | assert None == dep._get("t1", "ignore:"), got 48 | assert None == dep._get("t2", "ignore:") 49 | assert '1' == dep._get("g1", "ignore:") 50 | assert '1' == dep._get("g1.a", "ignore:") 51 | assert '1' == dep._get("g1.b", "ignore:") 52 | 53 | # if task dependency not from a group dont ignore it 54 | def testDontIgnoreTaskDependency(self, tasks, dep_manager): 55 | output = StringIO() 56 | cmd = CmdFactory(Ignore, outstream=output, dep_manager=dep_manager, 57 | task_list=tasks) 58 | cmd._execute(["t3"]) 59 | dep = Dependency(DbmDB, dep_manager.name) 60 | assert '1' == dep._get("t3", "ignore:") 61 | assert None == dep._get("t1", "ignore:") 62 | 63 | def testIgnoreInvalid(self, tasks, dep_manager): 64 | output = StringIO() 65 | cmd = CmdFactory(Ignore, outstream=output, dep_manager=dep_manager, 66 | task_list=tasks) 67 | pytest.raises(InvalidCommand, cmd._execute, ["XXX"]) 68 | -------------------------------------------------------------------------------- /tests/test_cmd_info.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | 3 | import pytest 4 | 5 | from doit.exceptions import InvalidCommand 6 | from doit.task import Task 7 | from doit.cmd_info import Info 8 | from .conftest import CmdFactory 9 | 10 | 11 | class TestCmdInfo(object): 12 | 13 | def test_info_basic_attrs(self, dep_manager): 14 | output = StringIO() 15 | task = Task("t1", [], file_dep=['tests/data/dependency1'], 16 | doc="task doc", getargs={'a': ('x', 'y')}, verbosity=2, 17 | meta={'a': ['b', 'c']}) 18 | cmd = CmdFactory(Info, outstream=output, 19 | dep_file=dep_manager.name, task_list=[task]) 20 | cmd._execute(['t1'], hide_status=True) 21 | assert "t1" in output.getvalue() 22 | assert "task doc" in output.getvalue() 23 | assert "- tests/data/dependency1" in output.getvalue() 24 | assert "verbosity : 2" in output.getvalue() 25 | assert "getargs : {'a': ('x', 'y')}" in output.getvalue() 26 | assert "meta : {'a': ['b', 'c']}" in output.getvalue() 27 | 28 | def test_invalid_command_args(self, dep_manager): 29 | output = StringIO() 30 | task = Task("t1", [], file_dep=['tests/data/dependency1']) 31 | cmd = CmdFactory(Info, outstream=output, 32 | dep_file=dep_manager.name, task_list=[task]) 33 | # fails if number of args != 1 34 | pytest.raises(InvalidCommand, cmd._execute, []) 35 | pytest.raises(InvalidCommand, cmd._execute, ['t1', 't2']) 36 | 37 | def test_execute_status_run(self, dep_manager, dependency1): 38 | output = StringIO() 39 | task = Task("t1", [], file_dep=['tests/data/dependency1']) 40 | cmd = CmdFactory(Info, outstream=output, 41 | dep_file=dep_manager.name, task_list=[task], 42 | dep_manager=dep_manager) 43 | return_val = cmd._execute(['t1']) 44 | assert "t1" in output.getvalue() 45 | assert return_val == 1 # indicates task is not up-to-date 46 | assert "status" in output.getvalue() 47 | assert ": run" in output.getvalue() 48 | assert " - tests/data/dependency1" in output.getvalue() 49 | 50 | def test_hide_execute_status(self, dep_manager, dependency1): 51 | output = StringIO() 52 | task = Task("t1", [], file_dep=['tests/data/dependency1']) 53 | cmd = CmdFactory(Info, outstream=output, 54 | dep_manager=dep_manager, task_list=[task]) 55 | return_val = cmd._execute(['t1'], hide_status=True) 56 | assert "t1" in output.getvalue() 57 | assert return_val == 0 # always zero if not showing status 58 | assert "status" not in output.getvalue() 59 | assert ": run" not in output.getvalue() 60 | 61 | def test_execute_status_uptodate(self, dep_manager, dependency1): 62 | output = StringIO() 63 | task = Task("t1", [], file_dep=['tests/data/dependency1']) 64 | cmd = CmdFactory(Info, outstream=output, 65 | dep_manager=dep_manager, task_list=[task]) 66 | cmd.dep_manager.save_success(task) 67 | return_val = cmd._execute(['t1']) 68 | assert "t1" in output.getvalue() 69 | assert return_val == 0 # indicates task is not up-to-date 70 | assert ": up-to-date" in output.getvalue() 71 | 72 | 73 | def test_get_reasons_str(self): 74 | reasons = { 75 | 'has_no_dependencies': True, 76 | 'uptodate_false': [('func', 'arg', 'kwarg')], 77 | 'checker_changed': ['foo', 'bar'], 78 | 'missing_target': ['f1', 'f2'], 79 | } 80 | 81 | got = Info.get_reasons(reasons).splitlines() 82 | assert len(got) == 7 83 | assert got[0] == ' * The task has no dependencies.' 84 | assert got[1] == ' * The following uptodate objects evaluate to false:' 85 | assert got[2] == ' - func (args=arg, kwargs=kwarg)' 86 | assert got[3] == ' * The file_dep checker changed from foo to bar.' 87 | assert got[4] == ' * The following targets do not exist:' 88 | assert got[5] == ' - f1' 89 | assert got[6] == ' - f2' 90 | -------------------------------------------------------------------------------- /tests/test_cmd_resetdep.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | 3 | import pytest 4 | 5 | from doit.cmd_resetdep import ResetDep 6 | from doit.dependency import TimestampChecker, get_md5, get_file_md5 7 | from doit.exceptions import InvalidCommand 8 | from doit.task import Task 9 | from tests.conftest import tasks_sample, CmdFactory, get_abspath 10 | 11 | 12 | class TestCmdResetDep(object): 13 | 14 | def test_execute(self, dep_manager, dependency1): 15 | output = StringIO() 16 | tasks = tasks_sample() 17 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks, 18 | dep_manager=dep_manager) 19 | cmd_reset._execute() 20 | got = [line.strip() for line in output.getvalue().split('\n') if line] 21 | expected = ["processed %s" % t.name for t in tasks] 22 | assert sorted(expected) == sorted(got) 23 | 24 | def test_file_dep(self, dep_manager, dependency1): 25 | my_task = Task("t2", [""], file_dep=['tests/data/dependency1']) 26 | output = StringIO() 27 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task], 28 | dep_manager=dep_manager) 29 | cmd_reset._execute() 30 | got = output.getvalue() 31 | assert "processed t2\n" == got 32 | 33 | dep = list(my_task.file_dep)[0] 34 | timestamp, size, md5 = dep_manager._get(my_task.name, dep) 35 | assert get_file_md5(get_abspath("data/dependency1")) == md5 36 | 37 | def test_file_dep_up_to_date(self, dep_manager, dependency1): 38 | my_task = Task("t2", [""], file_dep=['tests/data/dependency1']) 39 | dep_manager.save_success(my_task) 40 | output = StringIO() 41 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task], 42 | dep_manager=dep_manager) 43 | cmd_reset._execute() 44 | got = output.getvalue() 45 | assert "skip t2\n" == got 46 | 47 | def test_file_dep_change_checker(self, dep_manager, dependency1): 48 | my_task = Task("t2", [""], file_dep=['tests/data/dependency1']) 49 | dep_manager.save_success(my_task) 50 | dep_manager.checker = TimestampChecker() 51 | output = StringIO() 52 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task], 53 | dep_manager=dep_manager) 54 | cmd_reset._execute() 55 | got = output.getvalue() 56 | assert "processed t2\n" == got 57 | 58 | def test_filter(self, dep_manager, dependency1): 59 | output = StringIO() 60 | tasks = tasks_sample() 61 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks, 62 | dep_manager=dep_manager) 63 | cmd_reset._execute(pos_args=['t2']) 64 | got = output.getvalue() 65 | assert "processed t2\n" == got 66 | 67 | def test_invalid_task(self, dep_manager): 68 | output = StringIO() 69 | tasks = tasks_sample() 70 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks, 71 | dep_manager=dep_manager) 72 | pytest.raises(InvalidCommand, cmd_reset._execute, pos_args=['xxx']) 73 | 74 | def test_missing_file_dep(self, dep_manager): 75 | my_task = Task("t2", [""], file_dep=['tests/data/missing']) 76 | output = StringIO() 77 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task], 78 | dep_manager=dep_manager) 79 | cmd_reset._execute() 80 | got = output.getvalue() 81 | assert ("failed t2 (Dependent file 'tests/data/missing' does not " 82 | "exist.)\n") == got 83 | 84 | def test_missing_dep_and_target(self, dep_manager, dependency1, dependency2): 85 | 86 | task_a = Task("task_a", [""], 87 | file_dep=['tests/data/dependency1'], 88 | targets=['tests/data/dependency2']) 89 | task_b = Task("task_b", [""], 90 | file_dep=['tests/data/dependency2'], 91 | targets=['tests/data/dependency3']) 92 | task_c = Task("task_c", [""], 93 | file_dep=['tests/data/dependency3'], 94 | targets=['tests/data/dependency4']) 95 | 96 | output = StringIO() 97 | tasks = [task_a, task_b, task_c] 98 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks, 99 | dep_manager=dep_manager) 100 | cmd_reset._execute() 101 | 102 | got = output.getvalue() 103 | assert ("processed task_a\n" 104 | "processed task_b\n" 105 | "failed task_c (Dependent file 'tests/data/dependency3'" 106 | " does not exist.)\n") == got 107 | 108 | def test_values_and_results(self, dep_manager, dependency1): 109 | my_task = Task("t2", [""], file_dep=['tests/data/dependency1']) 110 | my_task.result = "result" 111 | my_task.values = {'x': 5, 'y': 10} 112 | dep_manager.save_success(my_task) 113 | dep_manager.checker = TimestampChecker() # trigger task update 114 | 115 | reseted = Task("t2", [""], file_dep=['tests/data/dependency1']) 116 | output = StringIO() 117 | cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[reseted], 118 | dep_manager=dep_manager) 119 | cmd_reset._execute() 120 | got = output.getvalue() 121 | assert "processed t2\n" == got 122 | assert {'x': 5, 'y': 10} == dep_manager.get_values(reseted.name) 123 | assert get_md5('result') == dep_manager.get_result(reseted.name) 124 | -------------------------------------------------------------------------------- /tests/test_cmd_strace.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | from io import StringIO 3 | 4 | import pytest 5 | 6 | from doit.cmd_base import TaskLoader2 7 | from doit.exceptions import InvalidCommand 8 | from doit.cmdparse import DefaultUpdate 9 | from doit.dependency import JSONCodec 10 | from doit.task import Task 11 | from doit.cmd_strace import Strace 12 | from .conftest import CmdFactory 13 | 14 | @pytest.mark.skipif( 15 | "os.system('strace -V') != 0 or sys.platform in ['win32', 'cygwin']") 16 | class TestCmdStrace(object): 17 | 18 | @staticmethod 19 | def loader_for_task(task): 20 | 21 | class MyTaskLoader(TaskLoader2): 22 | def load_doit_config(self): 23 | return {} 24 | 25 | def load_tasks(self, cmd, pos_args): 26 | return [task] 27 | 28 | return MyTaskLoader() 29 | 30 | def test_dep(self, dependency1, depfile_name): 31 | output = StringIO() 32 | task = Task("tt", ["cat %(dependencies)s"], 33 | file_dep=['tests/data/dependency1']) 34 | cmd = CmdFactory(Strace, outstream=output) 35 | cmd.loader = self.loader_for_task(task) 36 | params = DefaultUpdate(dep_file=depfile_name, show_all=False, 37 | keep_trace=False, backend='dbm', 38 | check_file_uptodate='md5', codec_cls=JSONCodec) 39 | result = cmd.execute(params, ['tt']) 40 | assert 0 == result 41 | got = output.getvalue().split("\n") 42 | dep_path = os.path.abspath("tests/data/dependency1") 43 | assert "R %s" % dep_path in got 44 | 45 | 46 | def test_opt_show_all(self, dependency1, depfile_name): 47 | output = StringIO() 48 | task = Task("tt", ["cat %(dependencies)s"], 49 | file_dep=['tests/data/dependency1']) 50 | cmd = CmdFactory(Strace, outstream=output) 51 | cmd.loader = self.loader_for_task(task) 52 | params = DefaultUpdate(dep_file=depfile_name, show_all=True, 53 | keep_trace=False, backend='dbm', 54 | check_file_uptodate='md5', codec_cls=JSONCodec) 55 | result = cmd.execute(params, ['tt']) 56 | assert 0 == result 57 | got = output.getvalue().split("\n") 58 | assert "cat" in got[0] 59 | 60 | def test_opt_keep_trace(self, dependency1, depfile_name): 61 | output = StringIO() 62 | task = Task("tt", ["cat %(dependencies)s"], 63 | file_dep=['tests/data/dependency1']) 64 | cmd = CmdFactory(Strace, outstream=output) 65 | cmd.loader = self.loader_for_task(task) 66 | params = DefaultUpdate(dep_file=depfile_name, show_all=True, 67 | keep_trace=True, backend='dbm', 68 | check_file_uptodate='md5', codec_cls=JSONCodec) 69 | result = cmd.execute(params, ['tt']) 70 | assert 0 == result 71 | got = output.getvalue().split("\n") 72 | assert "cat" in got[0] 73 | assert os.path.exists(cmd.TRACE_OUT) 74 | os.unlink(cmd.TRACE_OUT) 75 | 76 | 77 | def test_target(self, dependency1, depfile_name): 78 | output = StringIO() 79 | task = Task("tt", ["touch %(targets)s"], 80 | targets=['tests/data/dependency1']) 81 | cmd = CmdFactory(Strace, outstream=output) 82 | cmd.loader = self.loader_for_task(task) 83 | params = DefaultUpdate(dep_file=depfile_name, show_all=False, 84 | keep_trace=False, backend='dbm', 85 | check_file_uptodate='md5', codec_cls=JSONCodec) 86 | result = cmd.execute(params, ['tt']) 87 | assert 0 == result 88 | got = output.getvalue().split("\n") 89 | tgt_path = os.path.abspath("tests/data/dependency1") 90 | assert "W %s" % tgt_path in got 91 | 92 | def test_ignore_python_actions(self, dependency1, depfile_name): 93 | output = StringIO() 94 | def py_open(): 95 | with open(dependency1) as ignore: 96 | ignore 97 | task = Task("tt", [py_open]) 98 | cmd = CmdFactory(Strace, outstream=output) 99 | cmd.loader = self.loader_for_task(task) 100 | params = DefaultUpdate(dep_file=depfile_name, show_all=False, 101 | keep_trace=False, backend='dbm', 102 | check_file_uptodate='md5', codec_cls=JSONCodec) 103 | result = cmd.execute(params, ['tt']) 104 | assert 0 == result 105 | 106 | def test_invalid_command_args(self): 107 | output = StringIO() 108 | cmd = CmdFactory(Strace, outstream=output) 109 | # fails if number of args != 1 110 | pytest.raises(InvalidCommand, cmd.execute, {}, []) 111 | pytest.raises(InvalidCommand, cmd.execute, {}, ['t1', 't2']) 112 | 113 | 114 | -------------------------------------------------------------------------------- /tests/test_exceptions.py: -------------------------------------------------------------------------------- 1 | from doit import exceptions 2 | 3 | 4 | class TestInvalidCommand(object): 5 | def test_just_string(self): 6 | exception = exceptions.InvalidCommand('whatever string') 7 | assert 'whatever string' == str(exception) 8 | 9 | def test_task_not_found(self): 10 | exception = exceptions.InvalidCommand(not_found='my_task') 11 | exception.cmd_used = 'build' 12 | assert 'command `build` invalid parameter: "my_task".' in str(exception) 13 | 14 | def test_param_not_found(self): 15 | exception = exceptions.InvalidCommand(not_found='my_task') 16 | exception.cmd_used = None 17 | want = 'Invalid parameter: "my_task". Must be a command,' 18 | assert want in str(exception) 19 | assert 'Type "doit help" to see' in str(exception) 20 | 21 | def test_custom_binary_name(self): 22 | exception = exceptions.InvalidCommand(not_found='my_task') 23 | exception.cmd_used = None 24 | exception.bin_name = 'my_tool' 25 | assert 'Type "my_tool help" to see ' in str(exception) 26 | 27 | 28 | 29 | class TestBaseFail(object): 30 | def test_name(self): 31 | class XYZ(exceptions.BaseFail): 32 | pass 33 | my_excp = XYZ("hello") 34 | assert 'XYZ' == my_excp.get_name() 35 | assert 'XYZ' in str(my_excp) 36 | assert 'XYZ' in repr(my_excp) 37 | 38 | def test_msg_notraceback(self): 39 | my_excp = exceptions.BaseFail('got you') 40 | msg = my_excp.get_msg() 41 | assert 'got you' in msg 42 | 43 | def test_exception(self): 44 | try: 45 | raise IndexError('too big') 46 | except Exception as e: 47 | my_excp = exceptions.BaseFail('got this', e) 48 | msg = my_excp.get_msg() 49 | assert 'got this' in msg 50 | assert 'too big' in msg 51 | assert 'IndexError' in msg 52 | 53 | def test_caught(self): 54 | try: 55 | raise IndexError('too big') 56 | except Exception as e: 57 | my_excp = exceptions.BaseFail('got this', e) 58 | my_excp2 = exceptions.BaseFail('handle that', my_excp) 59 | msg = my_excp2.get_msg() 60 | assert 'handle that' in msg 61 | assert 'got this' not in msg # could be there too... 62 | assert 'too big' in msg 63 | assert 'IndexError' in msg 64 | 65 | 66 | class TestAllCaught(object): 67 | def test(self): 68 | assert issubclass(exceptions.TaskFailed, exceptions.BaseFail) 69 | assert issubclass(exceptions.TaskError, exceptions.BaseFail) 70 | assert issubclass(exceptions.SetupError, exceptions.BaseFail) 71 | assert issubclass(exceptions.DependencyError, exceptions.BaseFail) 72 | -------------------------------------------------------------------------------- /tests/test_plugin.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from unittest.mock import Mock 3 | 4 | import pytest 5 | 6 | from doit.plugin import PluginEntry, PluginDict 7 | 8 | 9 | if sys.version_info < (3, 10): 10 | from importlib_metadata import EntryPoint 11 | else: 12 | from importlib.metadata import EntryPoint 13 | 14 | 15 | class TestPluginEntry(object): 16 | def test_repr(self): 17 | plugin = PluginEntry('category1', 'name1', 'mock:Mock') 18 | assert "PluginEntry('category1', 'name1', 'mock:Mock')" == repr(plugin) 19 | 20 | def test_get(self): 21 | plugin = PluginEntry('category1', 'name1', 'unittest.mock:Mock') 22 | got = plugin.get() 23 | assert got is Mock 24 | 25 | def test_load_error_module_not_found(self): 26 | plugin = PluginEntry('category1', 'name1', 'i_dont:exist') 27 | with pytest.raises(Exception) as exc_info: 28 | plugin.load() 29 | assert 'Plugin category1 module `i_dont`' in str(exc_info.value) 30 | 31 | def test_load_error_obj_not_found(self): 32 | plugin = PluginEntry('category1', 'name1', 33 | 'unittest.mock:i_dont_exist') 34 | with pytest.raises(Exception) as exc_info: 35 | plugin.load() 36 | assert ('Plugin category1:name1 module `unittest.mock`' in 37 | str(exc_info.value)) 38 | assert 'i_dont_exist' in str(exc_info.value) 39 | 40 | 41 | class TestPluginDict(object): 42 | 43 | @pytest.fixture 44 | def plugins(self): 45 | plugins = PluginDict() 46 | config_dict = {'name1': 'pytest:raises', 47 | 'name2': 'unittest.mock:Mock'} 48 | plugins.add_plugins({'category1': config_dict}, 'category1') 49 | return plugins 50 | 51 | def test_add_plugins_from_dict(self, plugins): 52 | assert len(plugins) == 2 53 | name1 = plugins['name1'] 54 | assert isinstance(name1, PluginEntry) 55 | assert name1.category == 'category1' 56 | assert name1.name == 'name1' 57 | assert name1.location == 'pytest:raises' 58 | 59 | def test_add_plugins_from_pkg_resources(self, monkeypatch): 60 | # mock entry points 61 | from doit import plugin 62 | def fake_entries(group): 63 | yield EntryPoint(name='name1', value='pytest:raises', group=group) 64 | monkeypatch.setattr(plugin, 'entry_points_impl', lambda: fake_entries) 65 | 66 | plugins = PluginDict() 67 | plugins.add_plugins({}, 'category2') 68 | name1 = plugins['name1'] 69 | assert isinstance(name1, PluginEntry) 70 | assert name1.category == 'category2' 71 | assert name1.name == 'name1' 72 | assert name1.location == 'pytest:raises' 73 | 74 | def test_get_plugin_actual_plugin(self, plugins): 75 | assert plugins.get_plugin('name2') is Mock 76 | 77 | def test_get_plugin_not_a_plugin(self, plugins): 78 | my_val = 4 79 | plugins['builtin-item'] = my_val 80 | assert plugins.get_plugin('builtin-item') is my_val 81 | 82 | def test_to_dict(self, plugins): 83 | expected = {'name1': pytest.raises, 84 | 'name2': Mock} 85 | assert plugins.to_dict() == expected 86 | --------------------------------------------------------------------------------