├── .circleci └── config.yml ├── .docs ├── Makefile ├── _build │ ├── doctrees │ │ ├── changelog.doctree │ │ ├── environment.pickle │ │ ├── index.doctree │ │ └── source │ │ │ ├── modules.doctree │ │ │ ├── watchme.client.doctree │ │ │ ├── watchme.command.doctree │ │ │ ├── watchme.config.doctree │ │ │ ├── watchme.doctree │ │ │ ├── watchme.logger.doctree │ │ │ ├── watchme.tasks.doctree │ │ │ ├── watchme.tests.doctree │ │ │ ├── watchme.utils.doctree │ │ │ ├── watchme.watchers.doctree │ │ │ ├── watchme.watchers.gpu.doctree │ │ │ ├── watchme.watchers.psutils.doctree │ │ │ ├── watchme.watchers.results.doctree │ │ │ └── watchme.watchers.urls.doctree │ └── html │ │ ├── .buildinfo │ │ ├── .nojekyll │ │ ├── _sources │ │ ├── changelog.md.txt │ │ ├── index.rst.txt │ │ └── source │ │ │ ├── modules.rst.txt │ │ │ ├── watchme.client.rst.txt │ │ │ ├── watchme.command.rst.txt │ │ │ ├── watchme.config.rst.txt │ │ │ ├── watchme.logger.rst.txt │ │ │ ├── watchme.rst.txt │ │ │ ├── watchme.tasks.rst.txt │ │ │ ├── watchme.tests.rst.txt │ │ │ ├── watchme.utils.rst.txt │ │ │ ├── watchme.watchers.gpu.rst.txt │ │ │ ├── watchme.watchers.psutils.rst.txt │ │ │ ├── watchme.watchers.results.rst.txt │ │ │ ├── watchme.watchers.rst.txt │ │ │ └── watchme.watchers.urls.rst.txt │ │ ├── changelog.html │ │ ├── genindex.html │ │ ├── index.html │ │ ├── objects.inv │ │ ├── py-modindex.html │ │ ├── search.html │ │ ├── searchindex.js │ │ └── source │ │ ├── modules.html │ │ ├── watchme.client.html │ │ ├── watchme.command.html │ │ ├── watchme.config.html │ │ ├── watchme.html │ │ ├── watchme.logger.html │ │ ├── watchme.tasks.html │ │ ├── watchme.tests.html │ │ ├── watchme.utils.html │ │ ├── watchme.watchers.gpu.html │ │ ├── watchme.watchers.html │ │ ├── watchme.watchers.psutils.html │ │ ├── watchme.watchers.results.html │ │ └── watchme.watchers.urls.html ├── apidoc.sh ├── assets │ ├── logo.gif │ └── logo.png ├── changelog.md ├── conf.py ├── index.rst └── source │ ├── modules.rst │ ├── watchme.client.rst │ ├── watchme.command.rst │ ├── watchme.config.rst │ ├── watchme.logger.rst │ ├── watchme.rst │ ├── watchme.tasks.rst │ ├── watchme.tests.rst │ ├── watchme.utils.rst │ ├── watchme.watchers.gpu.rst │ ├── watchme.watchers.psutils.rst │ ├── watchme.watchers.results.rst │ ├── watchme.watchers.rst │ └── watchme.watchers.urls.rst ├── .github ├── AUTHORS.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── FUNDING.yml └── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── question.md ├── .gitignore ├── .pylintrc ├── CHANGELOG.md ├── CITATION.cff ├── LICENSE ├── MANIFEST.in ├── README.md ├── docker ├── Dockerfile └── Dockerfile.py2 ├── docs ├── Gemfile ├── _config.yml ├── _data │ └── links.yml ├── _docs │ ├── _defaults.md │ ├── contributing │ │ ├── code.md │ │ ├── docs.md │ │ ├── index.md │ │ └── watcher.md │ ├── examples │ │ └── index.md │ ├── getting-started │ │ ├── concepts.md │ │ ├── config.md │ │ ├── debugging.md │ │ ├── environment.md │ │ ├── index.md │ │ └── python.md │ ├── install │ │ └── index.md │ └── watcher-tasks │ │ ├── gpu.md │ │ ├── index.md │ │ ├── psutils.md │ │ ├── results.md │ │ └── urls.md ├── _includes │ ├── head.html │ └── navigation.html ├── _layouts │ └── default.html ├── _posts │ └── _defaults.md ├── api │ ├── _sources │ │ ├── changelog.md.txt │ │ ├── index.rst.txt │ │ └── source │ │ │ ├── modules.rst.txt │ │ │ ├── watchme.client.rst.txt │ │ │ ├── watchme.command.rst.txt │ │ │ ├── watchme.config.rst.txt │ │ │ ├── watchme.logger.rst.txt │ │ │ ├── watchme.rst.txt │ │ │ ├── watchme.tasks.rst.txt │ │ │ ├── watchme.tests.rst.txt │ │ │ ├── watchme.utils.rst.txt │ │ │ ├── watchme.watchers.gpu.rst.txt │ │ │ ├── watchme.watchers.psutils.rst.txt │ │ │ ├── watchme.watchers.results.rst.txt │ │ │ ├── watchme.watchers.rst.txt │ │ │ └── watchme.watchers.urls.rst.txt │ ├── assets │ │ ├── basic.css │ │ ├── css │ │ │ ├── badge_only.css │ │ │ └── theme.css │ │ ├── doctools.js │ │ ├── documentation_options.js │ │ ├── file.png │ │ ├── fonts │ │ │ ├── Inconsolata-Bold.ttf │ │ │ ├── Inconsolata-Regular.ttf │ │ │ ├── Inconsolata.ttf │ │ │ ├── Lato-Bold.ttf │ │ │ ├── Lato-Regular.ttf │ │ │ ├── Lato │ │ │ │ ├── lato-bold.eot │ │ │ │ ├── lato-bold.ttf │ │ │ │ ├── lato-bold.woff │ │ │ │ ├── lato-bold.woff2 │ │ │ │ ├── lato-bolditalic.eot │ │ │ │ ├── lato-bolditalic.ttf │ │ │ │ ├── lato-bolditalic.woff │ │ │ │ ├── lato-bolditalic.woff2 │ │ │ │ ├── lato-italic.eot │ │ │ │ ├── lato-italic.ttf │ │ │ │ ├── lato-italic.woff │ │ │ │ ├── lato-italic.woff2 │ │ │ │ ├── lato-regular.eot │ │ │ │ ├── lato-regular.ttf │ │ │ │ ├── lato-regular.woff │ │ │ │ └── lato-regular.woff2 │ │ │ ├── RobotoSlab-Bold.ttf │ │ │ ├── RobotoSlab-Regular.ttf │ │ │ ├── RobotoSlab │ │ │ │ ├── roboto-slab-v7-bold.eot │ │ │ │ ├── roboto-slab-v7-bold.ttf │ │ │ │ ├── roboto-slab-v7-bold.woff │ │ │ │ ├── roboto-slab-v7-bold.woff2 │ │ │ │ ├── roboto-slab-v7-regular.eot │ │ │ │ ├── roboto-slab-v7-regular.ttf │ │ │ │ ├── roboto-slab-v7-regular.woff │ │ │ │ └── roboto-slab-v7-regular.woff2 │ │ │ ├── fontawesome-webfont.eot │ │ │ ├── fontawesome-webfont.svg │ │ │ ├── fontawesome-webfont.ttf │ │ │ ├── fontawesome-webfont.woff │ │ │ └── fontawesome-webfont.woff2 │ │ ├── jquery-3.4.1.js │ │ ├── jquery.js │ │ ├── js │ │ │ ├── modernizr.min.js │ │ │ └── theme.js │ │ ├── language_data.js │ │ ├── logo.gif │ │ ├── logo.png │ │ ├── minus.png │ │ ├── plus.png │ │ ├── pygments.css │ │ ├── searchtools.js │ │ ├── underscore-1.3.1.js │ │ └── underscore.js │ ├── changelog.html │ ├── genindex.html │ ├── index.html │ ├── modules │ │ ├── index.html │ │ ├── watchme.html │ │ └── watchme │ │ │ ├── client.html │ │ │ ├── client │ │ │ ├── activate.html │ │ │ ├── add.html │ │ │ ├── create.html │ │ │ ├── deactivate.html │ │ │ ├── edit.html │ │ │ ├── export.html │ │ │ ├── get.html │ │ │ ├── init.html │ │ │ ├── inspect.html │ │ │ ├── ls.html │ │ │ ├── protect.html │ │ │ ├── remove.html │ │ │ ├── run.html │ │ │ └── schedule.html │ │ │ ├── command │ │ │ ├── commit.html │ │ │ ├── create.html │ │ │ └── utils.html │ │ │ ├── config.html │ │ │ ├── defaults.html │ │ │ ├── logger │ │ │ ├── message.html │ │ │ ├── namer.html │ │ │ ├── progress.html │ │ │ └── spinner.html │ │ │ ├── tasks.html │ │ │ ├── tasks │ │ │ └── worker.html │ │ │ ├── tests │ │ │ ├── test_client.html │ │ │ └── test_utils.html │ │ │ ├── utils │ │ │ ├── fileio.html │ │ │ └── terminal.html │ │ │ ├── watchers.html │ │ │ └── watchers │ │ │ ├── data.html │ │ │ ├── gpu.html │ │ │ ├── gpu │ │ │ ├── decorators.html │ │ │ ├── pynvml.html │ │ │ └── tasks.html │ │ │ ├── psutils.html │ │ │ ├── psutils │ │ │ └── tasks.html │ │ │ ├── results.html │ │ │ ├── results │ │ │ └── tasks.html │ │ │ ├── schedule.html │ │ │ ├── settings.html │ │ │ ├── urls.html │ │ │ └── urls │ │ │ └── tasks.html │ ├── objects.inv │ ├── py-modindex.html │ ├── search.html │ ├── searchindex.js │ └── source │ │ ├── modules.html │ │ ├── watchme.client.html │ │ ├── watchme.command.html │ │ ├── watchme.config.html │ │ ├── watchme.html │ │ ├── watchme.logger.html │ │ ├── watchme.tasks.html │ │ ├── watchme.tests.html │ │ ├── watchme.utils.html │ │ ├── watchme.watchers.gpu.html │ │ ├── watchme.watchers.html │ │ ├── watchme.watchers.psutils.html │ │ ├── watchme.watchers.results.html │ │ └── watchme.watchers.urls.html ├── assets │ ├── css │ │ └── watchme.css │ ├── fonts │ │ ├── helveticaneueout-webfont.woff │ │ └── helveticaneueout-webfont.woff2 │ ├── img │ │ ├── apple-touch-icon.png │ │ ├── emblem.svg │ │ ├── favicon.ico │ │ ├── favicon.png │ │ ├── logo.gif │ │ ├── logo_small.gif │ │ ├── menu.svg │ │ ├── siteicon.png │ │ ├── sprout-cute-1.gif │ │ └── touch-icon.png │ └── js │ │ ├── lunr.min.js │ │ └── search.js └── pages │ ├── 404.md │ ├── changelog.html │ ├── index.md │ ├── robots.txt │ └── search.html ├── paper ├── img │ ├── battery.png │ ├── core-temp.png │ └── virtual-memory-used.png ├── paper.bib └── paper.md ├── requirements.txt ├── setup.cfg ├── setup.py └── watchme ├── __init__.py ├── client ├── __init__.py ├── activate.py ├── add.py ├── create.py ├── deactivate.py ├── edit.py ├── export.py ├── get.py ├── init.py ├── inspect.py ├── ls.py ├── monitor.py ├── protect.py ├── remove.py ├── run.py └── schedule.py ├── command ├── __init__.py ├── commit.py ├── create.py └── utils.py ├── config ├── __init__.py └── watchme.cfg ├── defaults.py ├── logger ├── __init__.py ├── message.py ├── namer.py ├── progress.py └── spinner.py ├── tasks ├── __init__.py ├── decorators.py └── worker.py ├── tests ├── __init__.py ├── helpers.sh ├── test_client.py ├── test_client.sh ├── test_decorators.py ├── test_psutils_decorator.py ├── test_utils.py ├── test_utils.sh └── test_watchers.sh ├── utils ├── __init__.py ├── fileio.py └── terminal.py ├── version.py └── watchers ├── README.md ├── __init__.py ├── data.py ├── gpu ├── README.md ├── __init__.py ├── decorators.py ├── pynvml.py └── tasks.py ├── psutils ├── README.md ├── __init__.py ├── decorators.py └── tasks.py ├── results ├── __init__.py └── tasks.py ├── schedule.py ├── settings.py └── urls ├── __init__.py ├── helpers.py └── tasks.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # CircleCI build config to test different versions of Singuarity 2 | version: 2.1 3 | 4 | orbs: 5 | # https://circleci.com/orbs/registry/orb/circleci/docker-publish 6 | docker-publish: circleci/docker-publish@0.1.3 7 | 8 | workflows: 9 | version: 2 10 | 11 | # This workflow will be run on all branches but master (to test) 12 | build_without_publishing_job: 13 | jobs: 14 | 15 | - docker-publish/publish: 16 | image: vanessa/watchme 17 | registry: quay.io 18 | dockerfile: docker/Dockerfile 19 | deploy: false 20 | tag: latest 21 | filters: 22 | branches: 23 | ignore: 24 | - master 25 | after_build: 26 | - run: 27 | name: Preview Containers that will be Deployed 28 | command: | 29 | # Here we preview the Docker Tag 30 | for string in $(cat watchme/version.py | grep __version__); do 31 | DOCKER_TAG="${string//\"}" 32 | done 33 | echo "Version for Docker tag is ${DOCKER_TAG}" 34 | echo "This build will deploy the following containers:" 35 | echo "quay.io/vanessa/watchme:latest" 36 | echo "quay.io/vanessa/watchme:py3" 37 | echo "quay.io/vanessa/watchme:v${DOCKER_TAG}" 38 | echo "quay.io/vanessa/watchme:py3-v${DOCKER_TAG}" 39 | 40 | 41 | # This workflow will deploy images on merge to master only 42 | docker_with_lifecycle: 43 | jobs: 44 | 45 | - docker-publish/publish: 46 | image: vanessa/watchme 47 | registry: quay.io 48 | dockerfile: docker/Dockerfile 49 | tag: latest 50 | filters: 51 | branches: 52 | only: master 53 | after_build: 54 | - run: 55 | name: Publish Docker Containers with Python Version 3 56 | command: | 57 | # Here we preview the Docker Tag 58 | for string in $(cat watchme/version.py | grep __version__); do 59 | DOCKER_TAG="${string//\"}" 60 | done 61 | echo "Version for Docker tag is ${DOCKER_TAG}" 62 | docker tag quay.io/vanessa/watchme:latest quay.io/vanessa/watchme:v${DOCKER_TAG} 63 | docker tag quay.io/vanessa/watchme:latest quay.io/vanessa/watchme:py3-v${DOCKER_TAG} 64 | docker tag quay.io/vanessa/watchme:latest quay.io/vanessa/watchme:py3 65 | 66 | 67 | test: 68 | jobs: 69 | - test-watchme-python-3: 70 | filters: 71 | branches: 72 | ignore: docs/* 73 | 74 | 75 | install_watchme: &install_watchme 76 | name: install watchme 77 | command: | 78 | source "$BASH_ENV" 79 | pip uninstall watchme --yes || echo "Not installed" 80 | pip install .[all] 81 | 82 | 83 | install_python_3: &install_python_3 84 | name: install Python 3.5 dependencies 85 | command: | 86 | ls $HOME 87 | CONDA_PATH="$HOME/conda/Python3" 88 | echo 'export PATH="'"$CONDA_PATH"'/bin:$PATH"' >> "$BASH_ENV" 89 | source "$BASH_ENV" 90 | if [ ! -d "${CONDA_PATH}" ]; then 91 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh 92 | /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $CONDA_PATH 93 | $HOME/conda/Python3/bin/python setup.py install 94 | else 95 | echo "Miniconda 3 is already installed, continuing to build." 96 | fi 97 | $HOME/conda/Python3/bin/pip install black 98 | 99 | test_watchme_python: &test_watchme_python 100 | name: Test WatchMe Python 101 | command: | 102 | source "$BASH_ENV" 103 | cd ~/repo/watchme/tests 104 | python -m unittest test_client 105 | python -m unittest test_utils 106 | python -m unittest test_decorators 107 | 108 | test_watchme_bash: &test_watchme_bash 109 | name: Test WatchMe Client 110 | command: | 111 | source "$BASH_ENV" 112 | cd ~/repo/watchme/tests 113 | /bin/bash test_utils.sh 114 | /bin/bash test_client.sh 115 | /bin/bash test_watchers.sh 116 | 117 | run_linter: &run_linter 118 | name: run linter 119 | command: | 120 | cd ~/repo 121 | source "$BASH_ENV" 122 | black --check watchme 123 | 124 | jobs: 125 | test-watchme-python-3: 126 | machine: true 127 | working_directory: ~/repo 128 | steps: 129 | - checkout 130 | - restore_cache: 131 | keys: 132 | - v1-dependencies 133 | - run: *install_python_3 134 | - run: *run_linter 135 | - run: *install_watchme 136 | - save_cache: 137 | paths: 138 | - /home/circleci/conda 139 | key: v1-dependencies 140 | - run: *test_watchme_python 141 | - run: *test_watchme_bash 142 | -------------------------------------------------------------------------------- /.docs/_build/doctrees/changelog.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/changelog.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/environment.pickle -------------------------------------------------------------------------------- /.docs/_build/doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/index.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/modules.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/modules.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.client.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.client.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.command.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.command.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.config.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.config.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.logger.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.logger.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.tasks.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.tasks.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.tests.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.tests.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.utils.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.utils.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.watchers.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.watchers.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.watchers.gpu.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.watchers.gpu.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.watchers.psutils.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.watchers.psutils.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.watchers.results.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.watchers.results.doctree -------------------------------------------------------------------------------- /.docs/_build/doctrees/source/watchme.watchers.urls.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/doctrees/source/watchme.watchers.urls.doctree -------------------------------------------------------------------------------- /.docs/_build/html/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 4a48704d3b6586b31ba7d28931b29a73 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /.docs/_build/html/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/html/.nojekyll -------------------------------------------------------------------------------- /.docs/_build/html/_sources/changelog.md.txt: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | This is a manually generated log to track changes to the repository for each release. 4 | Each section should include general headers such as **Implemented enhancements** 5 | and **Merged pull requests**. All closed issued and bug fixes should be 6 | represented by the pull requests that fixed them. 7 | Critical items to know are: 8 | 9 | - renamed commands 10 | - deprecated / removed commands 11 | - changed defaults 12 | - backward incompatible changes 13 | - changed behaviour 14 | 15 | ## [master](https://github.com/vsoch/watchme/tree/master) 16 | - updating documentation to match library, more notes (0.0.28) 17 | - addition of GPU task, including terminal and process monitor (0.0.27) 18 | - adding linting, cleaning up an error for psutils watcher (0.0.26) 19 | - loop values should be checked for None first (0.0.25) 20 | - Connections aren't being parsed (0.0.24) 21 | - But with printing output to screen for monitor (should be json) (0.0.23) 22 | - Adding terminal monitor command group (0.0.22) 23 | - Print of task should be "add-task" (0.0.21) 24 | - Custom WATCHME_ENV variables added to monitoring decorator/task (0.0.20) 25 | - Adding decorator for system (psutils) monitoring (0.0.19) 26 | - Reorganizing task functions to belong with TaskBase (0.0.17) 27 | - Adding option for regular expression for URL wachers, user agent header (0.0.16) 28 | - requests is missing from install dependencies (0.0.15) 29 | - small bug fixes (0.0.14) 30 | - added headers, params, and json args for post and get urls. (0.0.13) 31 | - added ability to specify URL params for Get and Get with selector functions (0.0.12) 32 | - first beta release of watchme, all commands and docs (0.0.11) 33 | - adding changelog, and original skeleton for client (0.0.1) 34 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. Singularity Python API documentation master file, created by 2 | sphinx-quickstart on Sun Feb 11 12:17:05 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to WatchMe Python documentation! 7 | ================================================== 8 | 9 | WatchMe Python is the first reproducible monitoring tool to help you to easily 10 | collect version controlled data. The module is licensed 11 | under the MPL 2.0 LICENSE. 12 | 13 | Contents: 14 | 15 | .. toctree:: 16 | :maxdepth: 3 17 | 18 | source/watchme.rst 19 | changelog.md 20 | 21 | 22 | Indices and tables 23 | ------------------ 24 | 25 | * :ref:`modindex` 26 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/modules.rst.txt: -------------------------------------------------------------------------------- 1 | watchme 2 | ======= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | watchme 8 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.client.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.client package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.client\.activate module 8 | -------------------------------- 9 | 10 | .. automodule:: watchme.client.activate 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.client\.add module 16 | --------------------------- 17 | 18 | .. automodule:: watchme.client.add 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.client\.create module 24 | ------------------------------ 25 | 26 | .. automodule:: watchme.client.create 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.client\.deactivate module 32 | ---------------------------------- 33 | 34 | .. automodule:: watchme.client.deactivate 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | watchme\.client\.edit module 40 | ---------------------------- 41 | 42 | .. automodule:: watchme.client.edit 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | watchme\.client\.export module 48 | ------------------------------ 49 | 50 | .. automodule:: watchme.client.export 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | watchme\.client\.get module 56 | --------------------------- 57 | 58 | .. automodule:: watchme.client.get 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | watchme\.client\.init module 64 | ---------------------------- 65 | 66 | .. automodule:: watchme.client.init 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | watchme\.client\.inspect module 72 | ------------------------------- 73 | 74 | .. automodule:: watchme.client.inspect 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | watchme\.client\.ls module 80 | -------------------------- 81 | 82 | .. automodule:: watchme.client.ls 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | watchme\.client\.protect module 88 | ------------------------------- 89 | 90 | .. automodule:: watchme.client.protect 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | watchme\.client\.remove module 96 | ------------------------------ 97 | 98 | .. automodule:: watchme.client.remove 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | watchme\.client\.run module 104 | --------------------------- 105 | 106 | .. automodule:: watchme.client.run 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | watchme\.client\.schedule module 112 | -------------------------------- 113 | 114 | .. automodule:: watchme.client.schedule 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | 120 | Module contents 121 | --------------- 122 | 123 | .. automodule:: watchme.client 124 | :members: 125 | :undoc-members: 126 | :show-inheritance: 127 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.command.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.command package 2 | ======================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.command\.commit module 8 | ------------------------------- 9 | 10 | .. automodule:: watchme.command.commit 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.command\.create module 16 | ------------------------------- 17 | 18 | .. automodule:: watchme.command.create 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.command\.utils module 24 | ------------------------------ 25 | 26 | .. automodule:: watchme.command.utils 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: watchme.command 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.config.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.config package 2 | ======================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: watchme.config 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.logger.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.logger package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.logger\.message module 8 | ------------------------------- 9 | 10 | .. automodule:: watchme.logger.message 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.logger\.namer module 16 | ----------------------------- 17 | 18 | .. automodule:: watchme.logger.namer 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.logger\.progress module 24 | -------------------------------- 25 | 26 | .. automodule:: watchme.logger.progress 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.logger\.spinner module 32 | ------------------------------- 33 | 34 | .. automodule:: watchme.logger.spinner 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: watchme.logger 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.rst.txt: -------------------------------------------------------------------------------- 1 | watchme package 2 | =============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | watchme.client 10 | watchme.command 11 | watchme.config 12 | watchme.logger 13 | watchme.tasks 14 | watchme.utils 15 | watchme.watchers 16 | 17 | Submodules 18 | ---------- 19 | 20 | watchme\.defaults module 21 | ------------------------ 22 | 23 | .. automodule:: watchme.defaults 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | 28 | watchme\.version module 29 | ----------------------- 30 | 31 | .. automodule:: watchme.version 32 | :members: 33 | :undoc-members: 34 | :show-inheritance: 35 | 36 | 37 | Module contents 38 | --------------- 39 | 40 | .. automodule:: watchme 41 | :members: 42 | :undoc-members: 43 | :show-inheritance: 44 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.tasks.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.tasks package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.tasks\.worker module 8 | ----------------------------- 9 | 10 | .. automodule:: watchme.tasks.worker 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.tasks 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.tests.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.tests package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.tests\.test\_client module 8 | ----------------------------------- 9 | 10 | .. automodule:: watchme.tests.test_client 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.tests\.test\_utils module 16 | ---------------------------------- 17 | 18 | .. automodule:: watchme.tests.test_utils 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: watchme.tests 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.utils.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.utils package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.utils\.fileio module 8 | ----------------------------- 9 | 10 | .. automodule:: watchme.utils.fileio 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.utils\.terminal module 16 | ------------------------------- 17 | 18 | .. automodule:: watchme.utils.terminal 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: watchme.utils 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.watchers.gpu.rst.txt: -------------------------------------------------------------------------------- 1 | watchme.watchers.gpu package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme.watchers.gpu.decorators module 8 | -------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.gpu.decorators 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme.watchers.gpu.pynvml module 16 | ---------------------------------- 17 | 18 | .. automodule:: watchme.watchers.gpu.pynvml 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme.watchers.gpu.tasks module 24 | --------------------------------- 25 | 26 | .. automodule:: watchme.watchers.gpu.tasks 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: watchme.watchers.gpu 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.watchers.psutils.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.watchers\.psutils package 2 | ================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.watchers\.psutils\.tasks module 8 | ---------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.psutils.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.psutils 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.watchers.results.rst.txt: -------------------------------------------------------------------------------- 1 | watchme.watchers.results package 2 | ================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme.watchers.results.tasks module 8 | ------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.results.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.results 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.watchers.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.watchers package 2 | ========================= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | watchme.watchers.psutils 10 | watchme.watchers.urls 11 | 12 | Submodules 13 | ---------- 14 | 15 | watchme\.watchers\.data module 16 | ------------------------------ 17 | 18 | .. automodule:: watchme.watchers.data 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.watchers\.schedule module 24 | ---------------------------------- 25 | 26 | .. automodule:: watchme.watchers.schedule 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.watchers\.settings module 32 | ---------------------------------- 33 | 34 | .. automodule:: watchme.watchers.settings 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: watchme.watchers 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /.docs/_build/html/_sources/source/watchme.watchers.urls.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.watchers\.urls package 2 | =============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.watchers\.urls\.tasks module 8 | ------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.urls.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.urls 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /.docs/_build/html/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/_build/html/objects.inv -------------------------------------------------------------------------------- /.docs/apidoc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # If the modules changed, the content of "source" should be backed up and 3 | # new files generated (to update) by doing: 4 | # 5 | # sphinx-apidoc -o source/ ../watchme 6 | 7 | HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 8 | BASE=$(dirname $HERE) 9 | cd $HERE 10 | cd $BASE && python setup.py install && cd $HERE && make html 11 | rm -rf $BASE/docs/api 12 | 13 | # Create new folders 14 | mkdir -p $BASE/docs/api 15 | 16 | # Rename folders 17 | find $HERE/_build/html -exec sed -i -e 's/_static/assets/g' {} \; 18 | find $HERE/_build/html -exec sed -i -e 's/_modules/modules/g' {} \; 19 | 20 | # Copy to new locations 21 | mv $HERE/_build/html/_static $BASE/docs/api/assets 22 | mv $HERE/_build/html/_modules $BASE/docs/api/modules 23 | cp -R $HERE/_build/html/* $BASE/docs/api 24 | -------------------------------------------------------------------------------- /.docs/assets/logo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/assets/logo.gif -------------------------------------------------------------------------------- /.docs/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/.docs/assets/logo.png -------------------------------------------------------------------------------- /.docs/changelog.md: -------------------------------------------------------------------------------- 1 | ../CHANGELOG.md -------------------------------------------------------------------------------- /.docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Singularity Python API documentation master file, created by 2 | sphinx-quickstart on Sun Feb 11 12:17:05 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to WatchMe Python documentation! 7 | ================================================== 8 | 9 | WatchMe Python is the first reproducible monitoring tool to help you to easily 10 | collect version controlled data. The module is licensed 11 | under the MPL 2.0 LICENSE. 12 | 13 | Contents: 14 | 15 | .. toctree:: 16 | :maxdepth: 3 17 | 18 | source/watchme.rst 19 | changelog.md 20 | 21 | 22 | Indices and tables 23 | ------------------ 24 | 25 | * :ref:`modindex` 26 | -------------------------------------------------------------------------------- /.docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | watchme 2 | ======= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | watchme 8 | -------------------------------------------------------------------------------- /.docs/source/watchme.client.rst: -------------------------------------------------------------------------------- 1 | watchme\.client package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.client\.activate module 8 | -------------------------------- 9 | 10 | .. automodule:: watchme.client.activate 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.client\.add module 16 | --------------------------- 17 | 18 | .. automodule:: watchme.client.add 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.client\.create module 24 | ------------------------------ 25 | 26 | .. automodule:: watchme.client.create 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.client\.deactivate module 32 | ---------------------------------- 33 | 34 | .. automodule:: watchme.client.deactivate 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | watchme\.client\.edit module 40 | ---------------------------- 41 | 42 | .. automodule:: watchme.client.edit 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | watchme\.client\.export module 48 | ------------------------------ 49 | 50 | .. automodule:: watchme.client.export 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | watchme\.client\.get module 56 | --------------------------- 57 | 58 | .. automodule:: watchme.client.get 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | watchme\.client\.init module 64 | ---------------------------- 65 | 66 | .. automodule:: watchme.client.init 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | watchme\.client\.inspect module 72 | ------------------------------- 73 | 74 | .. automodule:: watchme.client.inspect 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | watchme\.client\.ls module 80 | -------------------------- 81 | 82 | .. automodule:: watchme.client.ls 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | watchme\.client\.protect module 88 | ------------------------------- 89 | 90 | .. automodule:: watchme.client.protect 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | watchme\.client\.remove module 96 | ------------------------------ 97 | 98 | .. automodule:: watchme.client.remove 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | watchme\.client\.run module 104 | --------------------------- 105 | 106 | .. automodule:: watchme.client.run 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | watchme\.client\.schedule module 112 | -------------------------------- 113 | 114 | .. automodule:: watchme.client.schedule 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | 120 | Module contents 121 | --------------- 122 | 123 | .. automodule:: watchme.client 124 | :members: 125 | :undoc-members: 126 | :show-inheritance: 127 | -------------------------------------------------------------------------------- /.docs/source/watchme.command.rst: -------------------------------------------------------------------------------- 1 | watchme\.command package 2 | ======================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.command\.commit module 8 | ------------------------------- 9 | 10 | .. automodule:: watchme.command.commit 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.command\.create module 16 | ------------------------------- 17 | 18 | .. automodule:: watchme.command.create 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.command\.utils module 24 | ------------------------------ 25 | 26 | .. automodule:: watchme.command.utils 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: watchme.command 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /.docs/source/watchme.config.rst: -------------------------------------------------------------------------------- 1 | watchme\.config package 2 | ======================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: watchme.config 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /.docs/source/watchme.logger.rst: -------------------------------------------------------------------------------- 1 | watchme\.logger package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.logger\.message module 8 | ------------------------------- 9 | 10 | .. automodule:: watchme.logger.message 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.logger\.namer module 16 | ----------------------------- 17 | 18 | .. automodule:: watchme.logger.namer 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.logger\.progress module 24 | -------------------------------- 25 | 26 | .. automodule:: watchme.logger.progress 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.logger\.spinner module 32 | ------------------------------- 33 | 34 | .. automodule:: watchme.logger.spinner 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: watchme.logger 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /.docs/source/watchme.rst: -------------------------------------------------------------------------------- 1 | watchme package 2 | =============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | watchme.client 10 | watchme.command 11 | watchme.config 12 | watchme.logger 13 | watchme.tasks 14 | watchme.utils 15 | watchme.watchers 16 | 17 | Submodules 18 | ---------- 19 | 20 | watchme\.defaults module 21 | ------------------------ 22 | 23 | .. automodule:: watchme.defaults 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | 28 | watchme\.version module 29 | ----------------------- 30 | 31 | .. automodule:: watchme.version 32 | :members: 33 | :undoc-members: 34 | :show-inheritance: 35 | 36 | 37 | Module contents 38 | --------------- 39 | 40 | .. automodule:: watchme 41 | :members: 42 | :undoc-members: 43 | :show-inheritance: 44 | -------------------------------------------------------------------------------- /.docs/source/watchme.tasks.rst: -------------------------------------------------------------------------------- 1 | watchme\.tasks package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.tasks\.worker module 8 | ----------------------------- 9 | 10 | .. automodule:: watchme.tasks.worker 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.tasks 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /.docs/source/watchme.tests.rst: -------------------------------------------------------------------------------- 1 | watchme\.tests package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.tests\.test\_client module 8 | ----------------------------------- 9 | 10 | .. automodule:: watchme.tests.test_client 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.tests\.test\_utils module 16 | ---------------------------------- 17 | 18 | .. automodule:: watchme.tests.test_utils 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: watchme.tests 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /.docs/source/watchme.utils.rst: -------------------------------------------------------------------------------- 1 | watchme\.utils package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.utils\.fileio module 8 | ----------------------------- 9 | 10 | .. automodule:: watchme.utils.fileio 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.utils\.terminal module 16 | ------------------------------- 17 | 18 | .. automodule:: watchme.utils.terminal 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: watchme.utils 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /.docs/source/watchme.watchers.gpu.rst: -------------------------------------------------------------------------------- 1 | watchme.watchers.gpu package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme.watchers.gpu.decorators module 8 | -------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.gpu.decorators 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme.watchers.gpu.pynvml module 16 | ---------------------------------- 17 | 18 | .. automodule:: watchme.watchers.gpu.pynvml 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme.watchers.gpu.tasks module 24 | --------------------------------- 25 | 26 | .. automodule:: watchme.watchers.gpu.tasks 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: watchme.watchers.gpu 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /.docs/source/watchme.watchers.psutils.rst: -------------------------------------------------------------------------------- 1 | watchme\.watchers\.psutils package 2 | ================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.watchers\.psutils\.tasks module 8 | ---------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.psutils.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.psutils 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /.docs/source/watchme.watchers.results.rst: -------------------------------------------------------------------------------- 1 | watchme.watchers.results package 2 | ================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme.watchers.results.tasks module 8 | ------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.results.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.results 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /.docs/source/watchme.watchers.rst: -------------------------------------------------------------------------------- 1 | watchme\.watchers package 2 | ========================= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | watchme.watchers.psutils 10 | watchme.watchers.urls 11 | 12 | Submodules 13 | ---------- 14 | 15 | watchme\.watchers\.data module 16 | ------------------------------ 17 | 18 | .. automodule:: watchme.watchers.data 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.watchers\.schedule module 24 | ---------------------------------- 25 | 26 | .. automodule:: watchme.watchers.schedule 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.watchers\.settings module 32 | ---------------------------------- 33 | 34 | .. automodule:: watchme.watchers.settings 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: watchme.watchers 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /.docs/source/watchme.watchers.urls.rst: -------------------------------------------------------------------------------- 1 | watchme\.watchers\.urls package 2 | =============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.watchers\.urls\.tasks module 8 | ------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.urls.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.urls 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /.github/AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Maintainers 2 | 3 | - [@vsoch](https://www.github.com/vsoch) 4 | 5 | # Contributors 6 | 7 | - [@SCHKN](https://www.github.com/SCHKN) 8 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # WatchMe Code of Conduct v1.0 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 26 | * Trolling, insulting/derogatory comments, and personal or political attacks 27 | * Public or private harassment 28 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 29 | * Other conduct which could reasonably be considered inappropriate in a professional setting 30 | 31 | ## Our Responsibilities 32 | 33 | Project maintainers are responsible for clarifying the standards of acceptable 34 | behavior and are expected to take appropriate and fair corrective action in 35 | response to any instances of unacceptable behavior. 36 | 37 | Project maintainers have the right and responsibility to remove, edit, or 38 | reject comments, commits, code, wiki edits, issues, and other contributions 39 | that are not aligned to this Code of Conduct, or to ban temporarily or 40 | permanently any contributor for other behaviors that they deem inappropriate, 41 | threatening, offensive, or harmful. 42 | 43 | ## Scope 44 | 45 | This Code of Conduct applies both within project spaces and in public spaces 46 | when an individual is representing the project or its community. Examples of 47 | representing a project or community include using an official project e-mail 48 | address, posting via an official social media account, or acting as an appointed 49 | representative at an online or offline event. Representation of a project may be 50 | further defined and clarified by project maintainers. 51 | 52 | ## Enforcement 53 | 54 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting @vsoch directly. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. @vsoch is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 55 | 56 | ## Thanks 57 | 58 | This code of conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org/), version 1.4, 59 | available at http://contributor-covenant.org/version/1/4. 60 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributor's Agreement 2 | This code is licensed under the MPL 2.0 [LICENSE](LICENSE). 3 | 4 | # Contributing 5 | When contributing to the WatchMe Python Client, it is important to properly communicate the 6 | gist of the contribution. If it is a simple code or editorial fix, simply 7 | explaining this within the GitHub Pull Request (PR) will suffice. But if this 8 | is a larger fix or Enhancement, it should be first discussed with the project 9 | leader or developers. Please also note that we have a [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) 10 | that should be followed for all interactions with the project members and users. 11 | 12 | ## Pull Request Process 13 | 14 | 1. Send PRs to the master branch for merge as a pypi release, or development 15 | if the intention is to add to next pypi release. 16 | 2. Follow the existing code style precedent. This does not need to be strictly 17 | defined as there are many thousands of lines of examples. Note the lack 18 | of tabs anywhere in the project, parentheses and spacing, documentation 19 | style, source code layout, variable scoping, and follow the project's 20 | standards. 21 | 3. Test your PR locally, and provide the steps necessary to test for the 22 | reviewers. 23 | 4. The project's default copyright and header have been included in any new 24 | source files. 25 | 5. All (major) changes to SIF Python Client must be documented in 26 | [docs](docs). If your PR changes a core functionality, please 27 | include clear description of the changes in your PR so that the docs 28 | can be updated, or better, submit another PR to update the docs directly. 29 | 6. If necessary, update the README.md. 30 | 7. The pull request will be reviewed by others, and the final merge must be 31 | done by the project lead, @vsoch (or approved by her). 32 | 33 | 34 | When you contribute to the project, you agree to add code under the provided 35 | licensing terms, and also we ask that you add your name to the [AUTHORS](AUTHORS.md) 36 | file. Any contribution in the way of documentation, features, or bug fixes is 37 | greatly appreciated. 38 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: vsoch 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve watchme 4 | --- 5 | 6 | ## Expected Behavior 7 | 8 | [ Please describe the behavior that are expecting ] 9 | 10 | ## Actual Behavior 11 | 12 | ## Steps to Reproduce 13 | 14 | [ Please provide detailed steps for reproducing the issue and reproducible Code Snippet ] 15 | 16 | ## Context 17 | 18 | [provide more detailed introduction to the issue itself . This is for make a reproducible issue.] 19 | * Operating System: 20 | * watchme version: 21 | * python version: 22 | 23 | ## Failure Logs 24 | 25 | [ log or files here. If relevant, include a screenshot] 26 | 27 | ## Possible Fix 28 | 29 | [ the suggest fixes or reasons for the bug] 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for watchme 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Ask a question, or prompt an update of the docs 4 | --- 5 | 6 | ## What is your question? 7 | 8 | [write your question here, and answer the prompts below] 9 | 10 | - [ ] I found the documentation at https://vsoch.github.io/watchme/ 11 | - [ ] The documentation does not answer my question 12 | 13 | If the documentation doesn't answer your question, where do you think it would be appropriate to add (i.e., where did you go looking for it?) 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | watchme.egg-info/ 2 | dist 3 | build 4 | _site 5 | .eggs 6 | __pycache__ 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | This is a manually generated log to track changes to the repository for each release. 4 | Each section should include general headers such as **Implemented enhancements** 5 | and **Merged pull requests**. All closed issued and bug fixes should be 6 | represented by the pull requests that fixed them. 7 | Critical items to know are: 8 | 9 | - renamed commands 10 | - deprecated / removed commands 11 | - changed defaults 12 | - backward incompatible changes 13 | - changed behaviour 14 | 15 | ## [master](https://github.com/vsoch/watchme/tree/master) 16 | - ensuring a better error message is given when watchme config missing (0.0.29) 17 | - updating documentation to match library, more notes (0.0.28) 18 | - addition of GPU task, including terminal and process monitor (0.0.27) 19 | - adding linting, cleaning up an error for psutils watcher (0.0.26) 20 | - loop values should be checked for None first (0.0.25) 21 | - Connections aren't being parsed (0.0.24) 22 | - But with printing output to screen for monitor (should be json) (0.0.23) 23 | - Adding terminal monitor command group (0.0.22) 24 | - Print of task should be "add-task" (0.0.21) 25 | - Custom WATCHME_ENV variables added to monitoring decorator/task (0.0.20) 26 | - Adding decorator for system (psutils) monitoring (0.0.19) 27 | - Reorganizing task functions to belong with TaskBase (0.0.17) 28 | - Adding option for regular expression for URL wachers, user agent header (0.0.16) 29 | - requests is missing from install dependencies (0.0.15) 30 | - small bug fixes (0.0.14) 31 | - added headers, params, and json args for post and get urls. (0.0.13) 32 | - added ability to specify URL params for Get and Get with selector functions (0.0.12) 33 | - first beta release of watchme, all commands and docs (0.0.11) 34 | - adding changelog, and original skeleton for client (0.0.1) 35 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.1.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: Sochat 5 | given-names: Vanessa 6 | orcid: https://orcid.org/0000-0002-4387-3819 7 | title: "WatchMe: Software for Reproducible Monitoring and Data Collection" 8 | version: 0.0.28 9 | doi: 10.21105/joss.01388 10 | date-released: 2019-05-09 11 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md LICENSE 2 | recursive-include watchme * 3 | graft watchme 4 | recursive-exclude * __pycache__ 5 | recursive-exclude * *.pyc 6 | recursive-exclude * *.pyo 7 | recursive-exclude .docs * 8 | recursive-exclude docs * 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # WatchMe 2 | 3 | ![https://raw.githubusercontent.com/vsoch/watchme/master/docs/assets/img/logo_small.gif](https://raw.githubusercontent.com/vsoch/watchme/master/docs/assets/img/logo_small.gif) 4 | 5 | [![DOI](https://zenodo.org/badge/177837425.svg)](https://zenodo.org/badge/latestdoi/177837425) 6 | [![DOI](http://joss.theoj.org/papers/10.21105/joss.01388/status.svg)](https://doi.org/10.21105/joss.01388) 7 | [![CircleCI](https://circleci.com/gh/vsoch/watchme.svg?style=svg)](https://circleci.com/gh/vsoch/watchme) 8 | 9 | Reproducible watching of web changes. Good for: 10 | 11 | 1. Monitoring system resources (battery, network, memory, cpu, etc.) 12 | 2. Waiting for job postings to change or appear 13 | 3. Monitoring some subset of prices from different vendors 14 | 4. Tracking changes in GitHub repositories (stars, etc.) over time 15 | 16 | - [documentation](https://vsoch.github.io/watchme) 17 | 18 | WatchMe can watch for changes to an entire page, or a specific section of it. 19 | It's appropriate for research use cases where you want to track changes in one 20 | or more pages over time. WatchMe also comes with psutils (system tasks) built 21 | in to allow for monitoring of system resources. Importantly, it is a tool that 22 | implements *reproducible monitoring*, as all your watches, are stored in a 23 | configuration file that can easily be shared with others 24 | to reproduce your watching protocol. For more information, see the 25 | [documentation](https://vsoch.github.io/watchme). 26 | [Docker bases](https://quay.io/repository/vanessa/watchme?tab=tags) are 27 | also available for monitoring processes inside containers. 28 | 29 | ## Limitations 30 | 31 | Watchme uses [cron](http://man7.org/linux/man-pages/man5/crontab.5.html) for 32 | scheduling jobs. This means that if a system was shutdown and then started again 33 | after some time, watchme will not recover missing jobs during that period. If 34 | you have ideas for how to better schedule jobs that you want added to the library, 35 | please [open an issue](https://github.com/vsoch/watchme)! 36 | 37 | ## Licenses 38 | 39 | This code is licensed under the MPL 2.0 [LICENSE](LICENSE). 40 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM continuumio/miniconda3 2 | 3 | # docker build -t vanessa/watchme . 4 | 5 | RUN apt-get update && apt-get install -y git gcc && \ 6 | mkdir -p /code 7 | ADD . /code 8 | WORKDIR /code 9 | RUN pip install .[all] 10 | ENTRYPOINT ["watchme"] 11 | -------------------------------------------------------------------------------- /docker/Dockerfile.py2: -------------------------------------------------------------------------------- 1 | FROM continuumio/miniconda 2 | 3 | # docker build -t vanessa/watchme . 4 | 5 | RUN apt-get update && apt-get install -y git gcc && \ 6 | mkdir -p /code 7 | ADD . /code 8 | WORKDIR /code 9 | RUN pip install .[all] 10 | ENTRYPOINT ["watchme"] 11 | -------------------------------------------------------------------------------- /docs/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'jekyll', '3.8.4' 4 | 5 | group :jekyll_plugins do 6 | gem 'jekyll-feed', '0.11.0' 7 | gem 'jekyll-seo-tag', '2.5.0' 8 | gem 'jekyll-sitemap', '1.2.0' 9 | end 10 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | #---- 2 | # Site 3 | 4 | title: watchme 5 | url: "https://vsoch.github.io/watchme" 6 | baseurl: "/watchme" 7 | show_full_navigation: false 8 | 9 | logo: assets/img/logo.gif 10 | description: Documentation for WatchMe Client 11 | author: 12 | name: 13 | email: vsochat@stanford.edu 14 | twitter: # twitter username without the @ symbol 15 | social: 16 | name: vsoch 17 | links: 18 | - https://github.com/vsoch 19 | 20 | # The current hosting location of the docs 21 | repo: vsoch/watchme 22 | reponame: watchme 23 | 24 | # The current Docker container 25 | docker: quay.io/vanessa/watchme 26 | 27 | # ----- 28 | # Build 29 | 30 | timezone: Etc/UTC 31 | 32 | permalink: pretty 33 | 34 | plugins: 35 | - jekyll-sitemap 36 | - jekyll-seo-tag 37 | - jekyll-feed 38 | 39 | exclude: 40 | - Gemfile 41 | - Gemfile.lock 42 | - README.md 43 | - LICENSE 44 | 45 | collections: 46 | docs: 47 | title: Documentation 48 | permalink: /:path/ 49 | output: true 50 | 51 | defaults: 52 | - 53 | scope: 54 | path: "" 55 | values: 56 | layout: default 57 | - 58 | scope: 59 | path: "" 60 | type: "docs" 61 | values: 62 | seo: 63 | type: Article 64 | _comments: 65 | category: Group navigation links with this field 66 | order: Used to sort links in the navigation 67 | _options: 68 | content: 69 | width: 800 70 | height: 2000 71 | - 72 | scope: 73 | path: "" 74 | type: "posts" 75 | values: 76 | _comments: 77 | type: Marks the impact of this release 78 | 79 | # --------- 80 | # ChangeLog 81 | 82 | types: 83 | - minor 84 | - major 85 | -------------------------------------------------------------------------------- /docs/_data/links.yml: -------------------------------------------------------------------------------- 1 | footer: 2 | - name: ChangeLog 3 | url: changelog 4 | - name: Python API 5 | url: api 6 | - name: GitHub 7 | external_url: https://www.github.com/vsoch/watchme 8 | navigation: 9 | - name: Installation 10 | url: install/ 11 | - name: Getting Started 12 | url: getting-started/ 13 | - name: Watcher Tasks 14 | url: watchers/ 15 | - name: Examples 16 | url: examples/ 17 | - name: Contributing 18 | url: contributing/ 19 | -------------------------------------------------------------------------------- /docs/_docs/_defaults.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 3 | category: 4 | order: 1 5 | --- 6 | -------------------------------------------------------------------------------- /docs/_docs/contributing/code.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Github Contribution 3 | category: Contributing 4 | order: 3 5 | --- 6 | 7 | 8 | To contribute to code, you should first *fork* the {{ site.title }} repository by clicking the *fork* button on the top right of the page. Once forked, you will want to clone the fork of the repository to your computer: 9 | 10 | ```bash 11 | git clone https://github.com//{{ site.reponame }} 12 | cd {{ site.reponame }}/ 13 | ``` 14 | 15 | The main python module, watchme, is in the top level folder. You should checkout a branch, 16 | push the branch to your remote, and when you are ready, open a pull request against 17 | the master branch of vsoch/watchme. 18 | 19 | ```bash 20 | git checkout -b add/my-feature 21 | git commit -a -m 'adding my new feature!' 22 | git push origin add/my-feature 23 | ``` 24 | -------------------------------------------------------------------------------- /docs/_docs/contributing/docs.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Contributing to Documentation 3 | category: Contributing 4 | order: 3 5 | --- 6 | 7 | It's so great that you want to contribute! The documentation here includes information 8 | about using and developing {{ site.title }}, and they are hosted on Github, meaning that you 9 | can easily contribute via a [pull request](https://help.github.com/articles/about-pull-requests/). 10 | 11 | ## Getting Started 12 | 13 | ### Installing Dependencies 14 | 15 | Initially (on OS X), you will need to setup [Brew](http://brew.sh/) which is a 16 | package manager for OS X and [Git](https://git-scm.com/). To install Brew and Git, 17 | run the following commands: 18 | 19 | ```bash 20 | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" 21 | brew install git 22 | ``` 23 | 24 | If you are on Debian/Ubuntu, then you can easily install git with `apt-get` 25 | 26 | ```bash 27 | apt-get update && apt-get install -y git 28 | ``` 29 | 30 | ### Fork the repo 31 | 32 | To contribute to the web based documentation, you should obtain a GitHub account and *fork* the {{ site.title }} Documentation repository by clicking the *fork* button on the top right of the page. Once forked, you will want to clone the fork of the repo to your computer. Let's say my GitHub username is *meatball*: 33 | 34 | ```bash 35 | git clone https://github.com/meatball/{{ site.reponame }} 36 | cd {{ site.reponame }}/ 37 | ``` 38 | 39 | ### Install a local Jekyll server 40 | This step is required if you want to render your work locally before committing the changes. This is highly recommended to ensure that your changes will render properly and will be accepted. 41 | 42 | ```bash 43 | brew install ruby 44 | gem install jekyll 45 | gem install bundler 46 | bundle install 47 | ``` 48 | 49 | The documentation is located in the "docs" subfolder, so after cloning the repository, 50 | you can change the directory to there and then run the jekyll serve. 51 | Specifically, you can see the site locally by running the server with jekyll: 52 | 53 | ```bash 54 | cd docs 55 | bundle exec jekyll serve 56 | ``` 57 | 58 | This will make the site viewable at http://localhost:4000/{{ site.title }}/. 59 | -------------------------------------------------------------------------------- /docs/_docs/contributing/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Contributing 3 | category: Contributing 4 | permalink: /contributing/index.html 5 | order: 1 6 | --- 7 | 8 | It's so great that you want to contribute! There are several ways to contribute. 9 | 10 | - [Contribute to Documentation]({{ site.baseurl }}/contributing/docs/) 11 | - [Create a new Watcher]({{ site.baseurl }}/contributing/watcher/) 12 | - [Contribute Code]({{ site.baseurl }}/contributing/code/) 13 | - Ask a question, report a bug, or suggest a feature by opening up an [issue](https://www.github.com/{{ site.repo }}/issues) 14 | 15 | Whether you ask a question, help to update the code with a bug fix, or 16 | simply want to say hello, your contribution is appreciated. 17 | 18 | -------------------------------------------------------------------------------- /docs/_docs/examples/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Example Watchers 3 | category: Examples 4 | permalink: /examples/index.html 5 | order: 1 6 | --- 7 | 8 | ## Repository Examples 9 | 10 | Here you can find example watcher repos: 11 | 12 | - [system](https://github.com/vsoch/watchme-system) for system, sensors, users, and networking monitoring using psutils tasks. 13 | - [results](https://github.com/vsoch/results-watcher) watcher can be run to quickly save any set of prefixed environment variables to a task folder. 14 | - [sklearn](https://github.com/vsoch/watchme-sklearn) demonstrates using WatchMe decorators monitor different sklearn functions on the fly. 15 | - [air-quality](https://github.com/vsoch/watchme-air-quality) for watching a metric across a few cities. 16 | - [prices](https://github.com/vsoch/watchme-pusheen) an example for monitoring Pusheen prices across several vendors. 17 | - [repository](https://github.com/vsoch/watchme-github-repos) to show using WatchMe to track GitHub repository metadata. 18 | 19 | For either of the above, you can easily install and activate the watcher to run on 20 | your machine! See [here](https://vsoch.github.io/watchme/getting-started/#how-do-i-get-a-watcher). 21 | For specific details about creating the watchers in question, see the README markdowns 22 | in the repositories. 23 | 24 | ## Configuration Examples 25 | 26 | The following example configurations are contributed by users over time. If you 27 | have an example to contribute, please [open an issue](https://www.github.com/{{ site.repo }}/issues) 28 | to share it. 29 | 30 | ### URL Watchers 31 | 32 | The following are examples for [URL watchers](https://vsoch.github.io/watchme/watchers/urls/). 33 | In the following example, the user is using the `get_url_selection` task to extract 34 | a number (note the regular expression) from the text resulting from selecting the 35 | class `.local-temp`. For this version of WatchMe the User-Agent header was not 36 | automatically added, so he added it here as a `header_*` parameter. 37 | 38 | ``` 39 | [task-temperature] 40 | url = https://www.accuweather.com/en/lu/luxembourg/228714/weather-forecast/228714 41 | selection = .local-temp 42 | get_text = true 43 | func = get_url_selection 44 | active = true 45 | type = urls 46 | regex = [0-9]+ 47 | header_user-agent = Mozilla/5.0 48 | ``` 49 | -------------------------------------------------------------------------------- /docs/_docs/getting-started/concepts.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Concepts 3 | category: Getting Started 4 | permalink: /getting-started/concepts/ 5 | order: 2 6 | --- 7 | 8 | 9 | What does WatchMe actually do? It comes down to helping you to create configurations 10 | to check web pages or other system content for changes. The checks are done with 11 | cron jobs, and the changes tracked with version control. 12 | 13 | ### Watcher 14 | 15 | A watcher comes down to a GitHub repository (stored in the WatchMe home directory, 16 | discussed under [Environment](#environment) that tracks changes for one or more 17 | items of interest. You can create as many watchers as you like, and you can choose 18 | to push to a service like GitHub (or not). The watcher has a particular configuration 19 | file, one called `watcher.cfg`, that will tell the software how to run the watcher. 20 | What does this mean? It means you can push your watcher configuration and items 21 | to GitHub, and someone else can install the software and reproduce your watching. 22 | Cool! 23 | 24 | ### Watcher Types 25 | 26 | So far, we've talked about looking for changes in "pages," or websites. 27 | This is the default watcher type, however there are other kinds of 28 | tasks that a watcher might run. When you create a new watcher, you can 29 | specify the type with the `--type` argument. 30 | 31 | ### Decorators 32 | 33 | A watchme decorator is a monitor that can be used to run some task over 34 | one of your python functions. For example, the `psutils` watcher type 35 | has a decorator to record resource information about a running process (your 36 | function) while it's running. 37 | 38 | > Where should I go next? 39 | 40 | Read about the [Environment]({{ site.baseurl }}/getting-started/environment/) or go back to the 41 | [Quick Start]({{ site.baseurl }}/getting-started/) 42 | -------------------------------------------------------------------------------- /docs/_docs/getting-started/config.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Watcher Configuration 3 | category: Getting Started 4 | permalink: /getting-started/watcher-config/ 5 | order: 4 6 | --- 7 | 8 | ## The Watcher 9 | 10 | When you create a watcher, you'll notice it's configuration file, `watchme.cfg` 11 | in the watcher folder: 12 | 13 | ```bash 14 | $ tree /home/vanessa/.watchme/ 15 | /home/vanessa/.watchme/ 16 | └── watcher 17 | └── watchme.cfg 18 | ``` 19 | 20 | and what you can't see is that there is a Git repo (a .git folder) in the 21 | watcher directory too. 22 | 23 | ### Active 24 | 25 | By default, the watcher simply has a status, active or not. 26 | 27 | ``` 28 | [watcher] 29 | active = false 30 | ``` 31 | 32 | When you first create a watcher, it will not be active. When you activate a watcher, 33 | this coincides with generating a cron job to run the command for it, and the cron 34 | job will run a watchme command to read this configuration file. This means that you 35 | can change the active status at any time that you need (manually or via `watchme activate ` and the cron job will run and essentially do nothing. 36 | 37 | 38 | ### Protected 39 | 40 | You can optionally protect a watcher, meaning that you can't delete it. Protected 41 | means that you can't delete the watcher, but you can edit the tasks. 42 | 43 | ``` 44 | [watcher] 45 | active = true 46 | protected = "on" 47 | ``` 48 | 49 | If the parameter is missing or defined with value "off", both coincide with 50 | not protected. 51 | 52 | ### Frozen 53 | 54 | If you want to freeze the configuration entirely, you should freeze it. This 55 | means that you cannot delete the watcher folder, or edit anything in the 56 | configuration file (other than the frozen status). 57 | 58 | ``` 59 | [watcher] 60 | active = true 61 | frozen = "on" 62 | ``` 63 | 64 | If the parameter is missing or defined with value "off", both coincide with 65 | not frozen. 66 | 67 | ### Tasks 68 | 69 | Each action done by a watcher is called a task. A task is identified by 70 | starting with "task" in the configuration: 71 | 72 | ``` 73 | [task-reddit] 74 | url = https://www.reddit.com/r/hpc 75 | active = false 76 | ``` 77 | 78 | Akin to the watcher itself, each task can also be active or not. The 79 | variables within the task will vary based on the kind of watcher. For example, 80 | the above is from the urls watcher type, since it will watch web pages for changes. 81 | 82 | ## Licenses 83 | 84 | This code is licensed under the Mozilla, version 2.0 or later [LICENSE](LICENSE). 85 | -------------------------------------------------------------------------------- /docs/_docs/getting-started/debugging.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Debugging 3 | category: Getting Started 4 | permalink: /getting-started/debugging/ 5 | order: 5 6 | --- 7 | 8 | Let's say that you create a new watcher called tasks: 9 | 10 | ```bash 11 | $ watchme create tasks 12 | Adding watcher /home/vanessa/.watchme/tasks... 13 | Generating watcher config /home/vanessa/.watchme/tasks/watchme.cfg 14 | ``` 15 | 16 | You add a task to watch a URL: 17 | 18 | ``` 19 | $ watchme add-task tasks task-watch-belta url@http://stopcovid.belta.by 20 | [task-watch-belta] 21 | url = http://stopcovid.belta.by 22 | active = true 23 | type = urls 24 | ``` 25 | 26 | You activate the watcher: 27 | 28 | ``` 29 | $ watchme activate tasks 30 | [watcher|tasks] active: true 31 | ``` 32 | 33 | And inspect it: 34 | 35 | ``` 36 | $ watchme inspect tasks 37 | [watcher] 38 | active = true 39 | type = urls 40 | 41 | [task-watch-belta] 42 | url = http://stopcovid.belta.by 43 | active = true 44 | type = urls 45 | ``` 46 | 47 | You want to manually run the task to test it, but you get an error: 48 | 49 | ``` 50 | $ watchme run tasks task-watch-belta 51 | Found 1 contender tasks. 52 | [1/1] |===================================| 100.0% 53 | ERROR Error running task. 54 | ``` 55 | 56 | You realize that you should have done a test run (that doesn't try to save data) 57 | 58 | ``` 59 | $ watchme run tasks task-watch-belta --test 60 | Found 1 contender tasks. 61 | [1/1] |===================================| 100.0% 62 | ERROR Error running task. 63 | ``` 64 | 65 | but that gives you the same thing! And this is logical, because there is some error 66 | with retrieving the URL and getting the result in the first place. The reason 67 | that we aren't able to easily see the error output above is due to the fact 68 | that watchme is using multiprocessing to run tasks. 69 | 70 | The way that you can get around this to debug the issue is to add the `--serial` 71 | flag, which will run the task in serial: 72 | 73 | ``` 74 | $ watchme run tasks task-watch-belta --serial --test 75 | Found 1 contender tasks. 76 | [task-watch-belta:1/1] |===================================| 100.0% 77 | Traceback (most recent call last): 78 | File "/home/vanessa/anaconda3/bin/watchme", line 10, in 79 | sys.exit(main()) 80 | File "/home/vanessa/anaconda3/lib/python3.7/site-packages/watchme/client/__init__.py", line 350, in main 81 | main(args, extras) 82 | File "/home/vanessa/anaconda3/lib/python3.7/site-packages/watchme/client/run.py", line 29, in main 83 | show_progress=not args.no_progress) 84 | File "/home/vanessa/anaconda3/lib/python3.7/site-packages/watchme/watchers/__init__.py", line 791, in run 85 | results = self.run_tasks(tasks, parallel, show_progress) 86 | File "/home/vanessa/anaconda3/lib/python3.7/site-packages/watchme/watchers/__init__.py", line 731, in run_tasks 87 | results[task.name] = task.run() 88 | File "/home/vanessa/anaconda3/lib/python3.7/site-packages/watchme/tasks/__init__.py", line 110, in run 89 | return func(**params) 90 | File "/home/vanessa/anaconda3/lib/python3.7/site-packages/watchme/watchers/urls/tasks.py", line 47, in get_task 91 | result = parse_success_response(response, kwargs) 92 | File "/home/vanessa/anaconda3/lib/python3.7/site-packages/watchme/watchers/urls/helpers.py", line 75, in parse_success_response 93 | result = response.json() 94 | File "/home/vanessa/anaconda3/lib/python3.7/site-packages/requests/models.py", line 897, in json 95 | return complexjson.loads(self.text, **kwargs) 96 | File "/home/vanessa/anaconda3/lib/python3.7/json/__init__.py", line 348, in loads 97 | return _default_decoder.decode(s) 98 | File "/home/vanessa/anaconda3/lib/python3.7/json/decoder.py", line 337, in decode 99 | obj, end = self.raw_decode(s, idx=_w(s, 0).end()) 100 | File "/home/vanessa/anaconda3/lib/python3.7/json/decoder.py", line 355, in raw_decode 101 | raise JSONDecodeError("Expecting value", s, err.value) from None 102 | json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) 103 | ``` 104 | 105 | In this case, we see that there is a bug that the result is expected to be json (but it's not!) 106 | We need to change the save_as parameter to be "text" since it defaults to json. Let's do that: 107 | 108 | 109 | ``` 110 | $ watchme add-task tasks task-watch-belta url@http://stopcovid.belta.by 111 | [task-watch-belta] 112 | url = http://stopcovid.belta.by 113 | active = true 114 | type = urls 115 | save_as = text 116 | ``` 117 | 118 | You should then be able to run the same job in serial (or parallel) and get a successful run. 119 | 120 | ```bash 121 | $ watchme run tasks task-watch-belta 122 | Found 1 contender tasks. 123 | [1/1] |===================================| 100.0% 124 | ``` 125 | -------------------------------------------------------------------------------- /docs/_docs/getting-started/environment.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Environment 3 | category: Getting Started 4 | permalink: /getting-started/environment/ 5 | order: 3 6 | --- 7 | 8 | WatchMe has a set of environment variables that can alter default base organization 9 | and setup, along with variables that can be set across watcher tasks to change how the outputs 10 | are saved there. 11 | 12 | ## Organization 13 | 14 | WatchMe will default to saving your configuration in a subfolder of your home, 15 | $HOME/.watchme, and with the configurations for your watchers 16 | (the groups of pages to watch, and the frequencies). For example, if you have 17 | a watcher called jobs, one called watcher (this is the default) and one called 18 | pizzas, your .watchme folder would looklike this: 19 | 20 | ```bash 21 | .watchme 22 | jobs/ 23 | pizzas/ 24 | watcher/ 25 | ``` 26 | 27 | What goes in each folder? It's actually a git repository, and this helps 28 | to track changes for pages. We do this instead of saving duplicates of 29 | the pages, and if you are doing research, you have a GitHub repository 30 | that updates with each. 31 | 32 | ### Watcher Folder 33 | 34 | Thus, if you want to change the default 35 | folder base ($HOME/.watchme) you can do the following: 36 | 37 | ```bash 38 | export WATCHME_BASE_DIR=$HOME/.creepywatcher 39 | ``` 40 | 41 | If you decide to make this change, it's important to write this into your .bashrc 42 | or equivalent, otherwise the default will be used. 43 | 44 | ### Default Watcher 45 | 46 | If you want to set a default watcher (different than "watcher") to 47 | interact with, you can do that too: 48 | 49 | ```bash 50 | export WATCHME_WATCHER=pizzas 51 | ``` 52 | 53 | And remember that you don't have to do this - you can specify the watcher 54 | name on the command line to setup and create commands. 55 | 56 | 57 | ## Variables 58 | 59 | By default, when you run a task for a watcher, it will save the result 60 | in the watcher folder, in a folder named for the task. For example, if we run 61 | the task "task-harvard-hpc" for a watcher called "watcher," the result 62 | will (without any special change) save like this: 63 | 64 | ```bash 65 | .watchme/ 66 | watcher/ 67 | watchme.cfg 68 | task-harvard-hpc/ 69 | result.txt 70 | ``` 71 | 72 | Of course this isn't ideal for all use cases! If you want a custom file name for the 73 | task, just use the variable `file_name` when you create the watcher. Let's say we want to 74 | use the default urls watcher to post to some endpoint, and save the result as 75 | json with a custom file name: 76 | 77 | ```bash 78 | $ watchme add-task watcher task-api-post url@https://singularityhub.github.io/registry/vanessa/fortune/manifests/latest/ file_name@manifest-latest.json save_as@json func@post_task 79 | ``` 80 | 81 | "save_as" is custom for the urls watcher (so json is returned) but for any watcher 82 | that is saving some file, you can customize the `file_name` variable: 83 | 84 | | Name | Required | Default | Example | Notes| 85 | |------|----------|---------|---------|-----------| 86 | | file_name | No | unset |file_name@image.png| the filename to save, only for download_task | 87 | 88 | 89 | > Where should I go next? 90 | 91 | Read about [Concepts]({{ site.baseurl }}/getting-started/concepts/) or go back to the 92 | [Quick Start]({{ site.baseurl }}/getting-started/) 93 | -------------------------------------------------------------------------------- /docs/_docs/getting-started/python.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Interactive Python 3 | category: Getting Started 4 | permalink: /getting-started/python/ 5 | order: 4 6 | --- 7 | 8 | If you are a developer (or just want to use python to interact with your watchers 9 | and tasks) here are some ways that you can do that. 10 | 11 | ## Instantiate a Watcher 12 | 13 | Let's say we have a watcher named "watcher" in the default base at $HOME/.watchme. 14 | We can instantiate it easily (this is from an interactive Python terminal): 15 | 16 | 17 | ```python 18 | from watchme import get_watcher 19 | watcher = get_watcher('watcher') 20 | # [watcher|watcher] 21 | ``` 22 | 23 | ## Inspection 24 | 25 | You can check the tasks that the watcher has: 26 | 27 | ```python 28 | watcher.inspect() 29 | [watcher] 30 | active = false 31 | 32 | [task-harvard-hpc] 33 | url = https://www.rc.fas.harvard.edu/about/people/ 34 | active = true 35 | type = urls 36 | ``` 37 | 38 | or for a specific task: 39 | 40 | ```python 41 | watcher.inspect('task-reddit-hpc') 42 | [task-harvard-hpc] 43 | url = https://www.rc.fas.harvard.edu/about/people/ 44 | active = true 45 | type = urls 46 | ``` 47 | 48 | or use a list to inspect more than one. 49 | 50 | ## Get tasks 51 | 52 | You can get a task instance by asking for it by name: 53 | 54 | ```python 55 | task = watcher.get_task('task-harvard-hpc') 56 | [task|task-reddit-hpc] 57 | ``` 58 | 59 | If the task doesn't exist, or isn't valid, you'll get an error message and 60 | None will be returned. You can also ask for all tasks that the watcher has, 61 | that are valid: 62 | 63 | ```python 64 | tasks = watcher.get_tasks() 65 | Found 1 contender tasks. 66 | ``` 67 | 68 | ## Run a task 69 | 70 | To run a task manually, you can either run all tasks from the watcher (done with 71 | multiprocessing): 72 | 73 | ```bash 74 | watcher.run() 75 | ``` 76 | 77 | And when you do this, the watcher's repository will be updated with data 78 | and the running timestamp. If you want to run a task and return a result 79 | without updating / interacting with the watcher, just run the task directly: 80 | 81 | ```python 82 | task.run() 83 | ``` 84 | 85 | ## Get function to run it 86 | 87 | To hand the task off to multiprocessing, we need to hand over a function, 88 | and a set of variables to provide to it. You can do this with two functions: 89 | 90 | ```python 91 | params = task.export_params() 92 | func = task.export_func(**params) 93 | ``` 94 | 95 | The export_func function should take all of the (exploded) params. 96 | 97 | **under development** 98 | 99 | 100 | ## Export Data 101 | 102 | Let's say that we want to export data for a particular task. First, initialize 103 | the watcher. Here is watcher `watchme-air-quality`: 104 | 105 | ```bash 106 | from watchme import get_watcher 107 | watcher = get_watcher('watchme-air-quality') 108 | ``` 109 | 110 | Next, export based on a task and a file in the folder: 111 | 112 | ```bash 113 | watcher.export_dataframe('task-air-oakland', 'oakland.txt') 114 | git log --all --oneline --pretty=tformat:"%H" --grep "ADD results" cca3c4eb84d9c38527ec93a9a620bfab07d798f2..86ba4c775fab86ea47d3b96b4477d4aaf6bdde83 -- task-air-oakland/oakland.txt 115 | ``` 116 | 117 | The command shown is how to get the list of commits that are relevant for your 118 | task and file. Notice how we filter to the git message to ADD results, which is used 119 | when we add results. 120 | 121 | 122 | Read about [Concepts]({{ site.baseurl }}/getting-started/concepts/), 123 | [Environment]({{ site.baseurl }}/getting-started/environment/) or go back to the 124 | [Quick Start]({{ site.baseurl }}/getting-started/) 125 | 126 | -------------------------------------------------------------------------------- /docs/_docs/install/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Installation 3 | category: Installation 4 | permalink: /install/index.html 5 | order: 1 6 | --- 7 | 8 | 9 | ## Dependencies 10 | 11 | The only dependency for watchme is to have [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) 12 | and [crontab](https://www.digitalocean.com/community/tutorials/how-to-use-cron-to-automate-tasks-on-a-vps) on your system. Git is used for version control of the pages you are watching, and crontab is 13 | used for scheduling your watches. If you want to install a custom watcher type, 14 | see [installing extras](#installing-extras) below. 15 | 16 | ## Install 17 | 18 | WatchMe can be installed natively (python 3 recommended) with pip: 19 | 20 | ```bash 21 | pip install watchme 22 | ``` 23 | 24 | or you can clone and install from source: 25 | 26 | ```bash 27 | $ git clone https://www.github.com/vsoch/watchme 28 | $ cd watchme 29 | $ python setup.py install 30 | ``` 31 | 32 | When you have installed WatchMe, there will be an executable "watchme" 33 | placed in your bin folder: 34 | 35 | ```bash 36 | which watchme 37 | /home/vanessa/anaconda3/bin/watchme 38 | ``` 39 | 40 | and you should be able to run the executable and see the usage: 41 | 42 | ```bash 43 | $ watchme 44 | 45 | [WatchMe] Command Line Tool v0.0.1 46 | usage: watchme [-h] [--debug] [--version] [--quiet] {init,create} ... 47 | 48 | WatchMe Command Line Tool 49 | 50 | optional arguments: 51 | -h, --help show this help message and exit 52 | --debug use verbose logging to debug. 53 | --version show version and exit. 54 | --quiet suppress additional output. 55 | 56 | actions: 57 | actions for HelpMe Command Line Tool 58 | 59 | {init,create} watchme actions 60 | init initialize watchme 61 | create create a new watcher 62 | ``` 63 | 64 | 65 | If you have any questions or issues, please [open an issue]({{ site.repo }}/issues). 66 | 67 | ## Installing Extras 68 | 69 | If you want to install all of watchme's exporters and watchers: 70 | 71 | ```bash 72 | $ pip install watchme[all] 73 | ``` 74 | 75 | To install all watchers only: 76 | 77 | ```bash 78 | $ pip install watchme[watchers] 79 | ``` 80 | 81 | or a specific watcher task group: 82 | 83 | ```bash 84 | $ pip install watchme[watcher-urls-dynamic] 85 | $ pip install watchme[watcher-psutils] 86 | ``` 87 | 88 | To see all of the choices, see [here](https://github.com/vsoch/watchme/blob/master/setup.py#L109) in the setup file. 89 | -------------------------------------------------------------------------------- /docs/_docs/watcher-tasks/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Watcher Tasks 3 | category: Watcher Tasks 4 | permalink: /watchers/index.html 5 | order: 1 6 | --- 7 | 8 | At this point, you've followed the [getting started]({{ site.baseurl }}/getting-started/) 9 | guide to [install]({{ site.baseurl }}/install/) watchme and create your first 10 | set of watchers. Now, it's time to configure your new watcher, meaning 11 | defining a set of tasks for it to do. Your watcher can have one or more tasks, 12 | and each has a defined type that the watcher knows how to run. 13 | We just have support for watching web urls, task type "urls": 14 | 15 | - [gpu]({{ site.baseurl }}/watchers/gpu/) to monitor one or more gpu devices 16 | - [urls]({{ site.baseurl }}/watchers/urls/) to watch for changes in web content 17 | - [psutils]({{ site.baseurl }}/watchers/psutils) to monitor system, sensors, python, and network 18 | - [results]({{ site.baseurl }}/watchers/results) to record results 19 | -------------------------------------------------------------------------------- /docs/_docs/watcher-tasks/results.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: results 3 | category: Watcher Tasks 4 | permalink: /watchers/results/ 5 | order: 3 6 | --- 7 | 8 | The results watcher is optimized to use WatchMe as a database. The various 9 | functions coincide with where data is expected to come from, and there 10 | is flexibility to customize format and file naming. You don't need 11 | any extra dependencies to use it, other than having watchme installled. 12 | 13 | ```bash 14 | $ pip install watchme 15 | ``` 16 | 17 | Next, create a watcher for your tasks to live under: 18 | 19 | ```bash 20 | $ watchme create results-watcher 21 | ``` 22 | 23 | Now let's walk through each of the tasks. If you aren't familiar with how 24 | to add a task and otherwise manage them, see the [getting started]({{ site.baseurl }}/getting-started/) 25 | docs. Here are the functions exposed by the results group: 26 | 27 | - [From Environment](#1-the-from-env-task) 28 | 29 | ## Add a Task 30 | 31 | We are going to be added tasks to our watcher, "results-watcher" above. 32 | The general format to add a task looks like this: 33 | 34 | ```bash 35 | $ watchme add-task key1@value1 key2@value2 36 | ``` 37 | 38 | The key and value pairs are going to vary based on the watcher task. 39 | 40 | ### 1. The From Env Task 41 | 42 | This task does exactly what it sounds like - it finds results from the environment. 43 | You can specify it to watchme via `func@from_env_task` 44 | 45 | ```bash 46 | $ watchme add-task results-watcher task-hpc-job --type results func@from_env_task 47 | func = from_env_task 48 | active = true 49 | type = results 50 | ``` 51 | 52 | The task itself is really simple - it's going to scrape the environment 53 | for variables that begin with `WATCHMEENV_`. Let's say that I'm running a job 54 | on my research cluster, the process could export some number of results, let's 55 | say one is a density and the other is a weight. 56 | 57 | ``` 58 | export WATCHMEENV_density=0.45 59 | export WATCHMEENV_weight=32 60 | ``` 61 | 62 | The export makes the variables available to any child processes of the job, but 63 | they won't leak into other running processes. I might finish running the job, 64 | and then have watchme issue a command directly to save the result. Let's 65 | activate the watcher to test it out: 66 | 67 | ```bash 68 | $ watchme activate results-watcher 69 | ``` 70 | 71 | And here you can export any number of `WATCHMEENV_` variables in your environment, and then 72 | run watchme like this to test: 73 | 74 | ```bash 75 | $ watchme run results-watcher task-hpc-job 76 | ``` 77 | 78 | The task will find the environment variables, and then save the results to 79 | the git repository according to the environment variable name under 80 | the task folder. In this case we would see: 81 | 82 | ``` 83 | $ tree task-hpc-job/ 84 | task-hpc-job/ 85 | ├── density 86 | ├── TIMESTAMP 87 | └── weight 88 | ``` 89 | 90 | Where the content of density is "0.45" and the content of weight is "32", 91 | respectively. If we push again, git will save our previous values, and 92 | the files will be updated with the current. We can push to GitHub to immediately 93 | share this data as a temporal database (a single entry representing one 94 | metric changing over time) or a collection of results (each timepoint 95 | with multiple variables coinciding with one entity). 96 | 97 | How cool is that! If each commit timepoint with some number of results files 98 | represents a single database entry, you would want to export a `WATCHMEENV_ID` to 99 | provide an identifier for the unit. 100 | 101 | The cool thing about this task is that you likely wouldn't want to schedule it 102 | to run, you would have it run after or during some job or process that you 103 | want to record. Just for example, let's export another set of values, 104 | and then show you how to export the entire data structure for each file: 105 | 106 | ```bash 107 | export WATCHMEENV_density=0.55 108 | export WATCHMEENV_weight=34 109 | watchme run results-watcher task-hpc-job 110 | ``` 111 | 112 | To export data, the format is: 113 | 114 | ```bash 115 | $ watchme export results-watcher task-hpc-job density 116 | git log --all --oneline --pretty=tformat:"%H" --grep "ADD results" 384d7bdc6e54af6266377b30ff0d47a40c4fc28d..732dee443caa19f0e50ec1e9b89ca3a542459cc7 -- task-hpc-job/density 117 | { 118 | "commits": [ 119 | "732dee443caa19f0e50ec1e9b89ca3a542459cc7", 120 | "c0861ed8ebe473cc3efa1db5f84e10d05d61bbc8" 121 | ], 122 | "dates": [ 123 | "2019-05-08 15:16:14 -0400", 124 | "2019-05-08 15:11:32 -0400" 125 | ], 126 | "content": [ 127 | "0.55", 128 | "0.45" 129 | ] 130 | } 131 | ``` 132 | 133 | There you go! 134 | -------------------------------------------------------------------------------- /docs/_includes/head.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {% seo %} 6 | {% feed_meta %} 7 | 8 | 9 | 10 | 11 | 12 | {% if jekyll.environment == 'production' and site.google_analytics_key %} 13 | 14 | 15 | 22 | {% endif %} 23 | 24 | -------------------------------------------------------------------------------- /docs/_includes/navigation.html: -------------------------------------------------------------------------------- 1 | 30 | -------------------------------------------------------------------------------- /docs/_layouts/default.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include head.html %} 5 | 6 | 7 |
8 |

9 | {{ site.title | downcase }} 10 | 11 | 12 | 13 |

14 | 15 |
16 | 17 | 18 |
19 | 20 | {% include navigation.html %} 21 |
22 | 23 |
24 |
25 |

{% if page.category %}{{ page.category }}{% else %}{{ site.title }}{% endif %}

26 |

{{ page.title }}

27 |
28 |
29 | {{ content }} 30 |
31 |
32 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /docs/_posts/_defaults.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 3 | type: major 4 | --- 5 | 6 | This release introduces 7 | 8 | **Features:** 9 | 10 | * 11 | 12 | **Fixes:** 13 | 14 | * 15 | -------------------------------------------------------------------------------- /docs/api/_sources/changelog.md.txt: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | This is a manually generated log to track changes to the repository for each release. 4 | Each section should include general headers such as **Implemented enhancements** 5 | and **Merged pull requests**. All closed issued and bug fixes should be 6 | represented by the pull requests that fixed them. 7 | Critical items to know are: 8 | 9 | - renamed commands 10 | - deprecated / removed commands 11 | - changed defaults 12 | - backward incompatible changes 13 | - changed behaviour 14 | 15 | ## [master](https://github.com/vsoch/watchme/tree/master) 16 | - updating documentation to match library, more notes (0.0.28) 17 | - addition of GPU task, including terminal and process monitor (0.0.27) 18 | - adding linting, cleaning up an error for psutils watcher (0.0.26) 19 | - loop values should be checked for None first (0.0.25) 20 | - Connections aren't being parsed (0.0.24) 21 | - But with printing output to screen for monitor (should be json) (0.0.23) 22 | - Adding terminal monitor command group (0.0.22) 23 | - Print of task should be "add-task" (0.0.21) 24 | - Custom WATCHME_ENV variables added to monitoring decorator/task (0.0.20) 25 | - Adding decorator for system (psutils) monitoring (0.0.19) 26 | - Reorganizing task functions to belong with TaskBase (0.0.17) 27 | - Adding option for regular expression for URL wachers, user agent header (0.0.16) 28 | - requests is missing from install dependencies (0.0.15) 29 | - small bug fixes (0.0.14) 30 | - added headers, params, and json args for post and get urls. (0.0.13) 31 | - added ability to specify URL params for Get and Get with selector functions (0.0.12) 32 | - first beta release of watchme, all commands and docs (0.0.11) 33 | - adding changelog, and original skeleton for client (0.0.1) 34 | -------------------------------------------------------------------------------- /docs/api/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. Singularity Python API documentation master file, created by 2 | sphinx-quickstart on Sun Feb 11 12:17:05 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to WatchMe Python documentation! 7 | ================================================== 8 | 9 | WatchMe Python is the first reproducible monitoring tool to help you to easily 10 | collect version controlled data. The module is licensed 11 | under the MPL 2.0 LICENSE. 12 | 13 | Contents: 14 | 15 | .. toctree:: 16 | :maxdepth: 3 17 | 18 | source/watchme.rst 19 | changelog.md 20 | 21 | 22 | Indices and tables 23 | ------------------ 24 | 25 | * :ref:`modindex` 26 | -------------------------------------------------------------------------------- /docs/api/_sources/source/modules.rst.txt: -------------------------------------------------------------------------------- 1 | watchme 2 | ======= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | watchme 8 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.client.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.client package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.client\.activate module 8 | -------------------------------- 9 | 10 | .. automodule:: watchme.client.activate 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.client\.add module 16 | --------------------------- 17 | 18 | .. automodule:: watchme.client.add 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.client\.create module 24 | ------------------------------ 25 | 26 | .. automodule:: watchme.client.create 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.client\.deactivate module 32 | ---------------------------------- 33 | 34 | .. automodule:: watchme.client.deactivate 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | watchme\.client\.edit module 40 | ---------------------------- 41 | 42 | .. automodule:: watchme.client.edit 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | watchme\.client\.export module 48 | ------------------------------ 49 | 50 | .. automodule:: watchme.client.export 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | watchme\.client\.get module 56 | --------------------------- 57 | 58 | .. automodule:: watchme.client.get 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | watchme\.client\.init module 64 | ---------------------------- 65 | 66 | .. automodule:: watchme.client.init 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | watchme\.client\.inspect module 72 | ------------------------------- 73 | 74 | .. automodule:: watchme.client.inspect 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | watchme\.client\.ls module 80 | -------------------------- 81 | 82 | .. automodule:: watchme.client.ls 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | watchme\.client\.protect module 88 | ------------------------------- 89 | 90 | .. automodule:: watchme.client.protect 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | watchme\.client\.remove module 96 | ------------------------------ 97 | 98 | .. automodule:: watchme.client.remove 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | watchme\.client\.run module 104 | --------------------------- 105 | 106 | .. automodule:: watchme.client.run 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | watchme\.client\.schedule module 112 | -------------------------------- 113 | 114 | .. automodule:: watchme.client.schedule 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | 120 | Module contents 121 | --------------- 122 | 123 | .. automodule:: watchme.client 124 | :members: 125 | :undoc-members: 126 | :show-inheritance: 127 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.command.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.command package 2 | ======================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.command\.commit module 8 | ------------------------------- 9 | 10 | .. automodule:: watchme.command.commit 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.command\.create module 16 | ------------------------------- 17 | 18 | .. automodule:: watchme.command.create 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.command\.utils module 24 | ------------------------------ 25 | 26 | .. automodule:: watchme.command.utils 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: watchme.command 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.config.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.config package 2 | ======================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: watchme.config 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.logger.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.logger package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.logger\.message module 8 | ------------------------------- 9 | 10 | .. automodule:: watchme.logger.message 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.logger\.namer module 16 | ----------------------------- 17 | 18 | .. automodule:: watchme.logger.namer 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.logger\.progress module 24 | -------------------------------- 25 | 26 | .. automodule:: watchme.logger.progress 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.logger\.spinner module 32 | ------------------------------- 33 | 34 | .. automodule:: watchme.logger.spinner 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: watchme.logger 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.rst.txt: -------------------------------------------------------------------------------- 1 | watchme package 2 | =============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | watchme.client 10 | watchme.command 11 | watchme.config 12 | watchme.logger 13 | watchme.tasks 14 | watchme.utils 15 | watchme.watchers 16 | 17 | Submodules 18 | ---------- 19 | 20 | watchme\.defaults module 21 | ------------------------ 22 | 23 | .. automodule:: watchme.defaults 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | 28 | watchme\.version module 29 | ----------------------- 30 | 31 | .. automodule:: watchme.version 32 | :members: 33 | :undoc-members: 34 | :show-inheritance: 35 | 36 | 37 | Module contents 38 | --------------- 39 | 40 | .. automodule:: watchme 41 | :members: 42 | :undoc-members: 43 | :show-inheritance: 44 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.tasks.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.tasks package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.tasks\.worker module 8 | ----------------------------- 9 | 10 | .. automodule:: watchme.tasks.worker 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.tasks 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.tests.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.tests package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.tests\.test\_client module 8 | ----------------------------------- 9 | 10 | .. automodule:: watchme.tests.test_client 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.tests\.test\_utils module 16 | ---------------------------------- 17 | 18 | .. automodule:: watchme.tests.test_utils 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: watchme.tests 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.utils.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.utils package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.utils\.fileio module 8 | ----------------------------- 9 | 10 | .. automodule:: watchme.utils.fileio 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme\.utils\.terminal module 16 | ------------------------------- 17 | 18 | .. automodule:: watchme.utils.terminal 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: watchme.utils 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.watchers.gpu.rst.txt: -------------------------------------------------------------------------------- 1 | watchme.watchers.gpu package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme.watchers.gpu.decorators module 8 | -------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.gpu.decorators 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | watchme.watchers.gpu.pynvml module 16 | ---------------------------------- 17 | 18 | .. automodule:: watchme.watchers.gpu.pynvml 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme.watchers.gpu.tasks module 24 | --------------------------------- 25 | 26 | .. automodule:: watchme.watchers.gpu.tasks 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: watchme.watchers.gpu 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.watchers.psutils.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.watchers\.psutils package 2 | ================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.watchers\.psutils\.tasks module 8 | ---------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.psutils.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.psutils 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.watchers.results.rst.txt: -------------------------------------------------------------------------------- 1 | watchme.watchers.results package 2 | ================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme.watchers.results.tasks module 8 | ------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.results.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.results 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.watchers.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.watchers package 2 | ========================= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | watchme.watchers.psutils 10 | watchme.watchers.urls 11 | 12 | Submodules 13 | ---------- 14 | 15 | watchme\.watchers\.data module 16 | ------------------------------ 17 | 18 | .. automodule:: watchme.watchers.data 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | watchme\.watchers\.schedule module 24 | ---------------------------------- 25 | 26 | .. automodule:: watchme.watchers.schedule 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | watchme\.watchers\.settings module 32 | ---------------------------------- 33 | 34 | .. automodule:: watchme.watchers.settings 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: watchme.watchers 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/api/_sources/source/watchme.watchers.urls.rst.txt: -------------------------------------------------------------------------------- 1 | watchme\.watchers\.urls package 2 | =============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | watchme\.watchers\.urls\.tasks module 8 | ------------------------------------- 9 | 10 | .. automodule:: watchme.watchers.urls.tasks 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: watchme.watchers.urls 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/api/assets/css/badge_only.css: -------------------------------------------------------------------------------- 1 | .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../fonts/fontawesome-webfont.eot");src:url("../fonts/fontawesome-webfont.eot?#iefix") format("embedded-opentype"),url("../fonts/fontawesome-webfont.woff") format("woff"),url("../fonts/fontawesome-webfont.ttf") format("truetype"),url("../fonts/fontawesome-webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} 2 | -------------------------------------------------------------------------------- /docs/api/assets/documentation_options.js: -------------------------------------------------------------------------------- 1 | var DOCUMENTATION_OPTIONS = { 2 | URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), 3 | VERSION: '11', 4 | LANGUAGE: 'None', 5 | COLLAPSE_INDEX: false, 6 | FILE_SUFFIX: '.html', 7 | HAS_SOURCE: true, 8 | SOURCELINK_SUFFIX: '.txt', 9 | NAVIGATION_WITH_KEYS: false 10 | }; -------------------------------------------------------------------------------- /docs/api/assets/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/file.png -------------------------------------------------------------------------------- /docs/api/assets/fonts/Inconsolata-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Inconsolata-Bold.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Inconsolata-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Inconsolata-Regular.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Inconsolata.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Inconsolata.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato-Bold.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato-Regular.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-bold.eot -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-bold.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-bold.woff -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-bolditalic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-bolditalic.eot -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-bolditalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-bolditalic.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-bolditalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-bolditalic.woff -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-bolditalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-bolditalic.woff2 -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-italic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-italic.eot -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-italic.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-italic.woff -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-italic.woff2 -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-regular.eot -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-regular.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-regular.woff -------------------------------------------------------------------------------- /docs/api/assets/fonts/Lato/lato-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/Lato/lato-regular.woff2 -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab-Bold.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab-Regular.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-bold.eot -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-bold.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-bold.woff -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-regular.eot -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-regular.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-regular.woff -------------------------------------------------------------------------------- /docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/RobotoSlab/roboto-slab-v7-regular.woff2 -------------------------------------------------------------------------------- /docs/api/assets/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/api/assets/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/api/assets/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/api/assets/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/api/assets/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/api/assets/js/theme.js: -------------------------------------------------------------------------------- 1 | /* sphinx_rtd_theme version 0.4.3 | MIT license */ 2 | /* Built 20190212 16:02 */ 3 | require=function r(s,a,l){function c(e,n){if(!a[e]){if(!s[e]){var i="function"==typeof require&&require;if(!n&&i)return i(e,!0);if(u)return u(e,!0);var t=new Error("Cannot find module '"+e+"'");throw t.code="MODULE_NOT_FOUND",t}var o=a[e]={exports:{}};s[e][0].call(o.exports,function(n){return c(s[e][1][n]||n)},o,o.exports,r,s,a,l)}return a[e].exports}for(var u="function"==typeof require&&require,n=0;n"),i("table.docutils.footnote").wrap("
"),i("table.docutils.citation").wrap("
"),i(".wy-menu-vertical ul").not(".simple").siblings("a").each(function(){var e=i(this);expand=i(''),expand.on("click",function(n){return t.toggleCurrent(e),n.stopPropagation(),!1}),e.prepend(expand)})},reset:function(){var n=encodeURI(window.location.hash)||"#";try{var e=$(".wy-menu-vertical"),i=e.find('[href="'+n+'"]');if(0===i.length){var t=$('.document [id="'+n.substring(1)+'"]').closest("div.section");0===(i=e.find('[href="#'+t.attr("id")+'"]')).length&&(i=e.find('[href="#"]'))}0this.docHeight||(this.navBar.scrollTop(i),this.winPosition=n)},onResize:function(){this.winResize=!1,this.winHeight=this.win.height(),this.docHeight=$(document).height()},hashChange:function(){this.linkScroll=!0,this.win.one("hashchange",function(){this.linkScroll=!1})},toggleCurrent:function(n){var e=n.closest("li");e.siblings("li.current").removeClass("current"),e.siblings().find("li.current").removeClass("current"),e.find("> ul li.current").removeClass("current"),e.toggleClass("current")}},"undefined"!=typeof window&&(window.SphinxRtdTheme={Navigation:e.exports.ThemeNav,StickyNav:e.exports.ThemeNav}),function(){for(var r=0,n=["ms","moz","webkit","o"],e=0;e 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/assets/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/assets/img/favicon.ico -------------------------------------------------------------------------------- /docs/assets/img/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/assets/img/favicon.png -------------------------------------------------------------------------------- /docs/assets/img/logo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/assets/img/logo.gif -------------------------------------------------------------------------------- /docs/assets/img/logo_small.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/assets/img/logo_small.gif -------------------------------------------------------------------------------- /docs/assets/img/menu.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /docs/assets/img/siteicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/assets/img/siteicon.png -------------------------------------------------------------------------------- /docs/assets/img/sprout-cute-1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/assets/img/sprout-cute-1.gif -------------------------------------------------------------------------------- /docs/assets/img/touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/docs/assets/img/touch-icon.png -------------------------------------------------------------------------------- /docs/assets/js/search.js: -------------------------------------------------------------------------------- 1 | --- 2 | layout: null 3 | --- 4 | (function () { 5 | function getQueryVariable(variable) { 6 | var query = window.location.search.substring(1), 7 | vars = query.split("&"); 8 | 9 | for (var i = 0; i < vars.length; i++) { 10 | var pair = vars[i].split("="); 11 | 12 | if (pair[0] === variable) { 13 | return decodeURIComponent(pair[1].replace(/\+/g, '%20')).trim(); 14 | } 15 | } 16 | } 17 | 18 | function getPreview(query, content, previewLength) { 19 | previewLength = previewLength || (content.length * 2); 20 | 21 | var parts = query.split(" "), 22 | match = content.toLowerCase().indexOf(query.toLowerCase()), 23 | matchLength = query.length, 24 | preview; 25 | 26 | // Find a relevant location in content 27 | for (var i = 0; i < parts.length; i++) { 28 | if (match >= 0) { 29 | break; 30 | } 31 | 32 | match = content.toLowerCase().indexOf(parts[i].toLowerCase()); 33 | matchLength = parts[i].length; 34 | } 35 | 36 | // Create preview 37 | if (match >= 0) { 38 | var start = match - (previewLength / 2), 39 | end = start > 0 ? match + matchLength + (previewLength / 2) : previewLength; 40 | 41 | preview = content.substring(start, end).trim(); 42 | 43 | if (start > 0) { 44 | preview = "..." + preview; 45 | } 46 | 47 | if (end < content.length) { 48 | preview = preview + "..."; 49 | } 50 | 51 | // Highlight query parts 52 | preview = preview.replace(new RegExp("(" + parts.join("|") + ")", "gi"), "$1"); 53 | } else { 54 | // Use start of content if no match found 55 | preview = content.substring(0, previewLength).trim() + (content.length > previewLength ? "..." : ""); 56 | } 57 | 58 | return preview; 59 | } 60 | 61 | function displaySearchResults(results, query) { 62 | var searchResultsEl = document.getElementById("search-results"), 63 | searchProcessEl = document.getElementById("search-process"); 64 | 65 | if (results.length) { 66 | var resultsHTML = ""; 67 | results.forEach(function (result) { 68 | var item = window.data[result.ref], 69 | contentPreview = getPreview(query, item.content, 170), 70 | titlePreview = getPreview(query, item.title); 71 | 72 | resultsHTML += "

  • " + titlePreview + "

    " + contentPreview + "

    • "; 73 | }); 74 | 75 | searchResultsEl.innerHTML = resultsHTML; 76 | searchProcessEl.innerText = "Showing"; 77 | } else { 78 | searchResultsEl.style.display = "none"; 79 | searchProcessEl.innerText = "No"; 80 | } 81 | } 82 | 83 | window.index = lunr(function () { 84 | this.field("id"); 85 | this.field("title", {boost: 10}); 86 | this.field("category"); 87 | this.field("url"); 88 | this.field("content"); 89 | }); 90 | 91 | var query = decodeURIComponent((getQueryVariable("q") || "").replace(/\+/g, "%20")), 92 | searchQueryContainerEl = document.getElementById("search-query-container"), 93 | searchQueryEl = document.getElementById("search-query"), 94 | searchInputEl = document.getElementById("search-input"); 95 | 96 | searchInputEl.value = query; 97 | searchQueryEl.innerText = query; 98 | searchQueryContainerEl.style.display = "inline"; 99 | 100 | for (var key in window.data) { 101 | window.index.add(window.data[key]); 102 | } 103 | 104 | displaySearchResults(window.index.search(query), query); // Hand the results off to be displayed 105 | })(); -------------------------------------------------------------------------------- /docs/pages/404.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Not Found 3 | permalink: /404.html 4 | sitemap: false 5 | --- 6 | 7 | This page doesn't exist! 8 | -------------------------------------------------------------------------------- /docs/pages/changelog.html: -------------------------------------------------------------------------------- 1 | --- 2 | title: Change Log 3 | permalink: /changelog/ 4 | --- 5 | 6 |

    Subscribe with RSS to keep up with the latest changes. 7 | The most detailed and up to date changes are kept with the CHANGELOG 8 | with the code base.

    9 | 10 |

    ✎ Update Change Log

    11 | 12 |
    13 | {% for change in site.posts %} 14 |
    15 |

    {{ change.title }}

    16 |

    {{ change.date | date: "%B %d, %Y" }} {{ change.type }}

    17 | 18 | {{ change.content }} 19 | 20 |

    ✎ Update Entry

    21 |
    22 | {% endfor %} 23 |
    24 | -------------------------------------------------------------------------------- /docs/pages/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Welcome 3 | permalink: / 4 | --- 5 | 6 | **WatchMe Python Client** 7 | 8 | > What is WatchMe? 9 | 10 | This is a simple tool to help you to reproducibly watch for changes in 11 | web content, system resources, or any task that you might provide to the 12 | software. While there are some (limited functionality) online resources that 13 | will help to track web content changes, there are three problems. 14 | 15 | 1. It's typically the case that you will be charged for more than a few pages 16 | 2. It's not appropriate for a research setting where you would want programmatical parsing 17 | 3. The configuration of your watcher is not reproducible. 18 | 19 | Thus, you should use this tool if you don't want to pay for a service, want to 20 | generate a reproducible watching, want to monitor resources or similar on your host, 21 | or just prefer command line tools. 22 | 23 | > What is a watcher? 24 | 25 | A watcher is a general helper to set regularly scheduled watching of things. 26 | By default, the software will watch for changes to one or more urls, and a 27 | second family of tasks (psutils) will help to monitor system resources (cpu, 28 | memory, temperature, fans, etc.) on your host. 29 | 30 | > Where do I go from here? 31 | 32 | A good place to start is the [getting started]({{ site.baseurl }}/getting-started/) page. 33 | -------------------------------------------------------------------------------- /docs/pages/robots.txt: -------------------------------------------------------------------------------- 1 | --- 2 | layout: null 3 | sitemap: false 4 | permalink: /robots.txt 5 | --- 6 | User-agent: * 7 | Sitemap: {{ site.url }}/sitemap.xml 8 | Disallow: /search/ 9 | -------------------------------------------------------------------------------- /docs/pages/search.html: -------------------------------------------------------------------------------- 1 | --- 2 | title: Search 3 | sitemap: false 4 | permalink: /search/ 5 | --- 6 | 7 |

    Loading results

    8 |
      9 | 10 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /paper/img/battery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/paper/img/battery.png -------------------------------------------------------------------------------- /paper/img/core-temp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/paper/img/core-temp.png -------------------------------------------------------------------------------- /paper/img/virtual-memory-used.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/paper/img/virtual-memory-used.png -------------------------------------------------------------------------------- /paper/paper.bib: -------------------------------------------------------------------------------- 1 | @MISC{psutil, 2 | title = "psutil documentation --- psutil 5.6.2 documentation", 3 | howpublished = "\url{https://psutil.readthedocs.io/en/latest/index.html}", 4 | note = "Accessed: 2019-4-7" 5 | } 6 | 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | python-crontab>=2.3.6 2 | configparser>=3.5.3 3 | requests>=2.21.0 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description_file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import os 3 | 4 | ################################################################################ 5 | # HELPER FUNCTIONS ############################################################# 6 | ################################################################################ 7 | 8 | 9 | def get_lookup(): 10 | """get version by way of singularity.version, returns a 11 | lookup dictionary with several global variables without 12 | needing to import singularity 13 | """ 14 | lookup = dict() 15 | version_file = os.path.join("watchme", "version.py") 16 | with open(version_file) as filey: 17 | exec(filey.read(), lookup) 18 | return lookup 19 | 20 | 21 | # Read in requirements 22 | def get_requirements(lookup=None, key="INSTALL_REQUIRES"): 23 | """get_requirements reads in requirements and versions from 24 | the lookup obtained with get_lookup""" 25 | 26 | if lookup is None: 27 | lookup = get_lookup() 28 | 29 | install_requires = [] 30 | for module in lookup[key]: 31 | module_name = module[0] 32 | module_meta = module[1] 33 | if "exact_version" in module_meta: 34 | dependency = "%s==%s" % (module_name, module_meta["exact_version"]) 35 | elif "min_version" in module_meta: 36 | min_version = module_meta["min_version"] 37 | if min_version is None: 38 | dependency = module_name 39 | else: 40 | dependency = "%s>=%s" % (module_name, min_version) 41 | install_requires.append(dependency) 42 | return install_requires 43 | 44 | 45 | # Make sure everything is relative to setup.py 46 | install_path = os.path.dirname(os.path.abspath(__file__)) 47 | os.chdir(install_path) 48 | 49 | # Get version information from the lookup 50 | lookup = get_lookup() 51 | VERSION = lookup["__version__"] 52 | NAME = lookup["NAME"] 53 | AUTHOR = lookup["AUTHOR"] 54 | AUTHOR_EMAIL = lookup["AUTHOR_EMAIL"] 55 | PACKAGE_URL = lookup["PACKAGE_URL"] 56 | KEYWORDS = lookup["KEYWORDS"] 57 | DESCRIPTION = lookup["DESCRIPTION"] 58 | LICENSE = lookup["LICENSE"] 59 | with open("README.md") as filey: 60 | LONG_DESCRIPTION = filey.read() 61 | 62 | ################################################################################ 63 | # MAIN ######################################################################### 64 | ################################################################################ 65 | 66 | 67 | if __name__ == "__main__": 68 | 69 | # Install all exporters and/or watchers 70 | INSTALL_REQUIRES = get_requirements(lookup) 71 | INSTALL_ALL = get_requirements(lookup, "INSTALL_ALL") 72 | WATCHERS = get_requirements(lookup, "INSTALL_WATCHERS") 73 | 74 | # Watchers 75 | URLS_DYNAMIC = get_requirements(lookup, "INSTALL_URLS_DYNAMIC") 76 | PSUTILS = get_requirements(lookup, "INSTALL_PSUTILS") 77 | 78 | setup( 79 | name=NAME, 80 | version=VERSION, 81 | author=AUTHOR, 82 | author_email=AUTHOR_EMAIL, 83 | maintainer=AUTHOR, 84 | maintainer_email=AUTHOR_EMAIL, 85 | packages=find_packages(), 86 | include_package_data=True, 87 | zip_safe=False, 88 | url=PACKAGE_URL, 89 | license=LICENSE, 90 | description=DESCRIPTION, 91 | long_description=LONG_DESCRIPTION, 92 | long_description_content_type="text/markdown", 93 | keywords=KEYWORDS, 94 | setup_requires=["pytest-runner"], 95 | tests_require=["pytest"], 96 | install_requires=INSTALL_REQUIRES, 97 | extras_require={ 98 | "all": [INSTALL_ALL], 99 | "watchers": [WATCHERS], 100 | "watcher-urls-dynamic": [URLS_DYNAMIC], 101 | "watcher-psutils": [PSUTILS], 102 | }, 103 | classifiers=[ 104 | "Intended Audience :: Science/Research", 105 | "Intended Audience :: Developers", 106 | "Programming Language :: Python", 107 | "Topic :: Software Development", 108 | "Topic :: Scientific/Engineering", 109 | "Operating System :: Unix", 110 | "Programming Language :: Python :: 2.7", 111 | "Programming Language :: Python :: 3", 112 | ], 113 | entry_points={"console_scripts": ["watchme=watchme.client:main"]}, 114 | ) 115 | -------------------------------------------------------------------------------- /watchme/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.version import __version__ 6 | 7 | 8 | def get_watcher(name="watcher", base=None, create=False, quiet=True, **kwargs): 9 | """ 10 | get the correct watcher depending on the environment variable 11 | WATCHME_WATCHER or default to "watcher" 12 | 13 | Parameters 14 | ========== 15 | name: the name of the watcher, will be made all lowercase 16 | base: the watcher base, if not defined, will use WATCHER_BASE_DIR envar 17 | create: if the watcher folder doesn't exist, create it (default False) 18 | for all interactions with a watcher other than create, we should 19 | exit if the watcher the user wants doesn't exist. 20 | quiet: if True, suppress most output about the client (e.g. speak) 21 | """ 22 | from watchme.watchers import Watcher 23 | 24 | Watcher.name = name.lower() 25 | Watcher.quiet = quiet 26 | 27 | # Initialize the watcher 28 | return Watcher(create=create, base=base) 29 | -------------------------------------------------------------------------------- /watchme/client/activate.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | 7 | 8 | def main(args, extra): 9 | """activate one or more watchers""" 10 | # Doesn't work if watcher not provided 11 | watcher = args.watcher[0] 12 | watcher = get_watcher(watcher, base=args.base) 13 | 14 | # The user is deactivating the entire watcher 15 | if extra is None: 16 | watcher.activate() 17 | else: 18 | for name in extra: 19 | watcher.activate(name) 20 | -------------------------------------------------------------------------------- /watchme/client/add.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | from watchme.logger import bot 7 | 8 | 9 | def main(args, extra): 10 | """add a task for a watcher""" 11 | # Required - will print help if not provided 12 | name = args.watcher[0] 13 | task = args.task[0] 14 | 15 | if not task.startswith("task"): 16 | example = "watchme add-task watcher task-cpu func@cpu_task type@psutils" 17 | bot.exit('Task name must start with "task", e.g., %s' % example) 18 | 19 | # Exit if the user doesn't provide any parameters 20 | if extra is None: 21 | bot.exit("Please provide parameters to add to your watcher (key@value)") 22 | 23 | # Type can also be an argument 24 | watcher_type = args.watcher_type 25 | params = [] 26 | for param in extra: 27 | if param.startswith("type@"): 28 | watcher_type = param.replace("type@", "") 29 | else: 30 | params.append(param) 31 | 32 | # Get the watcher to interact with, must already exist 33 | watcher = get_watcher(name, base=args.base, create=False) 34 | 35 | # Add the task. Will exit if not a valid type, or parameters 36 | watcher.add_task( 37 | task=task, 38 | task_type=watcher_type, 39 | params=params, 40 | force=args.force, 41 | active=args.active, 42 | ) 43 | -------------------------------------------------------------------------------- /watchme/client/create.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Copyright (C) 2019-2022 Vanessa Sochat. 4 | 5 | This Source Code Form is subject to the terms of the 6 | Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed 7 | with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | """ 10 | 11 | from watchme.command import create_watcher 12 | 13 | 14 | def main(args, extra): 15 | """create means creating one or more watchers.""" 16 | 17 | watchers = args.watchers 18 | if len(watchers) == 0: 19 | watchers = ["watcher"] 20 | 21 | for watcher in watchers: 22 | create_watcher(watcher) 23 | -------------------------------------------------------------------------------- /watchme/client/deactivate.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | 7 | 8 | def main(args, extra): 9 | """deactivate one or more watchers""" 10 | # Doesn't work if watcher not provided 11 | watcher = args.watcher[0] 12 | watcher = get_watcher(watcher, base=args.base) 13 | 14 | # The user is deactivating the entire watcher 15 | if extra is None: 16 | watcher.deactivate() 17 | else: 18 | for name in extra: 19 | watcher.deactivate(name) 20 | -------------------------------------------------------------------------------- /watchme/client/edit.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | from watchme.logger import bot 7 | 8 | 9 | def main(args, extra): 10 | """edit the configuration for a watcher task""" 11 | # Required - will print help if not provided 12 | name = args.watcher[0] 13 | action = args.action[0] 14 | task = args.task[0] 15 | 16 | # Get the watcher (exits if doesn't exist) 17 | watcher = get_watcher(name, base=args.base) 18 | 19 | # Exit if the user doesn't provide a time 20 | if extra is None: 21 | bot.exit("Please provide one or more items to %s" % action) 22 | 23 | key = extra[0] 24 | value = None 25 | if action in ["add", "update"]: 26 | if len(extra) != 2: 27 | bot.exit("You must do watchme add ") 28 | value = extra[1] 29 | 30 | # Ensure the task exists 31 | watcher.edit_task(task, action, key, value) 32 | -------------------------------------------------------------------------------- /watchme/client/export.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | from watchme.utils import write_json 7 | from watchme.logger import bot 8 | import json 9 | import os 10 | 11 | 12 | def main(args, extra): 13 | """export temporal data for a watcher""" 14 | # Required - will print help if not provided 15 | name = args.watcher[0] 16 | task = args.task[0] 17 | filename = args.filename[0] 18 | 19 | if not task.startswith("task") and not task.startswith("decorator"): 20 | example = "watchme export watcher task-reddit result.txt" 21 | bot.exit('Task name must start with "task" or "decorator": %s' % example) 22 | 23 | # Use the output file, or a temporary file 24 | out = args.out 25 | 26 | # Get the watcher to interact with, must already exist 27 | watcher = get_watcher(name, base=args.base, create=False) 28 | 29 | if out is not None: 30 | if os.path.exists(out) and args.force is False: 31 | bot.exit("%s exists! Use --force to overwrite." % out) 32 | 33 | # Export the data to file 34 | result = watcher.export_dict( 35 | task=task, filename=filename, name=name, export_json=args.json, base=args.base 36 | ) 37 | 38 | if result is not None: 39 | 40 | if out is None: 41 | print(json.dumps(result, indent=4)) 42 | else: 43 | write_json(result, out) 44 | bot.info("Result written to %s" % out) 45 | -------------------------------------------------------------------------------- /watchme/client/get.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.command import git_clone 6 | 7 | 8 | def main(args, extra): 9 | """get a watcher using git, meaning clone to a temporary location and then 10 | copying the entire repo (or a subfolder) to the watcher base. 11 | """ 12 | 13 | # Required - will print help if not provided 14 | repo = args.repo[0] 15 | 16 | # If a custom name is provided, add it 17 | if extra is not None: 18 | extra = extra.pop(0) 19 | 20 | # Clone the watcher, and optionally just one task 21 | git_clone(repo=repo, base=args.base, name=extra, force=args.force) 22 | -------------------------------------------------------------------------------- /watchme/client/init.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.command.create import create_watcher_base, create_watcher 6 | 7 | 8 | def main(args, extra): 9 | """init means creating the watcher base, and the first (default) watcher.""" 10 | 11 | # Create the base 12 | create_watcher_base(args.watcher, args.base) 13 | 14 | # Create the first default watcher 15 | if not args.create_empty: 16 | create_watcher(args.watcher) 17 | -------------------------------------------------------------------------------- /watchme/client/inspect.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | 7 | 8 | def main(args, extra): 9 | """activate one or more watchers""" 10 | # Required - will print help if not provided 11 | name = args.watcher[0] 12 | watcher = get_watcher(name, base=args.base, create=False) 13 | 14 | # If user wants to see create command, must have list of tasks 15 | if args.create_command is True and extra is None: 16 | extra = [x.name for x in watcher.get_tasks()] 17 | 18 | # Show the create command, or inspect a task 19 | watcher.inspect(extra, create_command=args.create_command) 20 | -------------------------------------------------------------------------------- /watchme/client/ls.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.command import get_watchers, list_task, list_watcher, list_watcher_types 6 | from watchme.logger import bot 7 | 8 | 9 | def main(args, extra): 10 | """list installed watchers""" 11 | if args.watchers is True: 12 | list_watcher_types() 13 | 14 | # Otherwise, we are listing installed watchers and tasks 15 | else: 16 | 17 | # If no watchers provided, list the watchers 18 | if extra is None: 19 | get_watchers(args.base) 20 | 21 | # One argument is the name of a watcher 22 | elif len(extra) == 1: 23 | list_watcher(extra[0], args.base) 24 | 25 | # Two arguments must be a watcher and task 26 | elif len(extra) == 2: 27 | list_task(extra[0], extra[1], args.base) 28 | 29 | else: 30 | bot.exit("Please provide none or all of to list.") 31 | -------------------------------------------------------------------------------- /watchme/client/monitor.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.command import get_watchers 6 | from watchme import get_watcher 7 | from watchme.tasks.decorators import TerminalRunner 8 | import json 9 | 10 | 11 | def main(args, extra): 12 | """monitor a task (from the command line), meaning wrapping it with 13 | multiprocessing, getting the id, and returning a result (command line 14 | or written to file) 15 | """ 16 | # The first argument will either be part of a command, or a watcher 17 | watcher = args.watcher 18 | 19 | # The entire user command is the extra arguments 20 | command = extra 21 | 22 | # If the user provides a watcher, we are saving to it 23 | if watcher not in get_watchers(args.base, quiet=True): 24 | command = [watcher] + command 25 | watcher = None 26 | else: 27 | watcher = get_watcher(watcher, base=args.base, create=False) 28 | 29 | command = " ".join(command) 30 | runner = TerminalRunner( 31 | command, 32 | skip=args.skip, 33 | include=args.include, 34 | only=args.only, 35 | seconds=args.seconds, 36 | ) 37 | runner.run() 38 | timepoints = runner.wait(args.func) 39 | 40 | # The output folder depends on the watcher func 41 | prefix = "decorator-psutils" 42 | if args.func == "gpu_task": 43 | prefix = "decorator-gpu" 44 | 45 | # If we don't have a watcher, print to terminal 46 | if watcher is None or args.test is True: 47 | print(json.dumps(timepoints)) 48 | 49 | # Otherwise save to watcher task folder 50 | else: 51 | name = args.name 52 | if name is None: 53 | name = command.replace(" ", "-") 54 | name = "%s-%s" % (prefix, name) 55 | watcher.finish_runs({name: timepoints}) 56 | -------------------------------------------------------------------------------- /watchme/client/protect.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | 7 | 8 | def main(args, extra): 9 | """protect or freeze a watcher, or disable.""" 10 | # Required - will print help if not provided 11 | name = args.watcher[0] 12 | watcher = get_watcher(name, base=args.base, create=False) 13 | 14 | if args.action in ["on", "off"]: 15 | watcher.protect(args.action) 16 | elif args.action == "freeze": 17 | watcher.freeze() 18 | elif args.action == "unfreeze": 19 | watcher.unfreeze() 20 | -------------------------------------------------------------------------------- /watchme/client/remove.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | from watchme.logger import bot 7 | 8 | 9 | def main(args, extra): 10 | """activate one or more watchers""" 11 | # Required - will print help if not provided 12 | name = args.watcher[0] 13 | watcher = get_watcher(name, base=args.base, create=False) 14 | 15 | # If delete is true, remove entire watcher, only if not protected or frozen 16 | if args.delete: 17 | watcher.delete() 18 | 19 | else: 20 | 21 | # Exit if the user doesn't provide any tasks to remove 22 | if extra is None: 23 | bot.exit("Provide tasks to remove, or --delete for entire watcher.") 24 | 25 | for task in extra: 26 | 27 | # Remove the task, if it exists 28 | watcher.remove_task(task) 29 | -------------------------------------------------------------------------------- /watchme/client/run.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | 7 | 8 | def main(args, extra): 9 | """run a watcher, optionally supplying one or more regular expressions to 10 | check for. 11 | """ 12 | # Required - will print help if not provided 13 | name = args.watcher[0] 14 | watcher = get_watcher(name, base=args.base, create=False) 15 | 16 | # If a set of task names or regular expressions are provided: 17 | if extra is not None: 18 | extra = "(%s)" % "|".join(extra) 19 | 20 | # Run the watcher, providing regular expressions to match tasks 21 | watcher.run( 22 | regexp=extra, 23 | parallel=not args.serial, 24 | test=args.test, 25 | show_progress=not args.no_progress, 26 | ) 27 | -------------------------------------------------------------------------------- /watchme/client/schedule.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme import get_watcher 6 | from watchme.logger import bot 7 | 8 | 9 | def main(args, extra): 10 | """activate one or more watchers""" 11 | # Required - will print help if not provided 12 | name = args.watcher[0] 13 | 14 | # Exit if the user doesn't provide a time 15 | if extra is None: 16 | bot.exit("Please provide a time frame (@daily, @hourly, @weekly, etc.)") 17 | 18 | # Determine the time to use 19 | if "@daily" in extra: 20 | minute, hour, month, day, weekday = 0, 0, "*", "*", "*" 21 | elif "@hourly" in extra: 22 | minute, hour, month, day, weekday = 0, "*", "*", "*", "*" 23 | elif "@weekly" in extra: 24 | minute, hour, month, day, weekday = 0, 0, "*", "*", 0 25 | elif "@monthly" in extra: 26 | minute, hour, month, day, weekday = 0, 0, 1, "*", "*" 27 | elif "@yearly" in extra: 28 | minute, hour, month, day, weekday = 0, 0, 1, 1, "*" 29 | else: 30 | if len(extra) != 5: 31 | message = """Please enter a frequency (@weekly) or use a valid 32 | cron timestamp (see https://crontab.guru/).""" 33 | bot.exit(message) 34 | minute, hour, month, day, weekday = extra 35 | 36 | # Schedule the watcher 37 | watcher = get_watcher(name, base=args.base) 38 | watcher.schedule(minute, hour, month, day, weekday, force=args.force) 39 | -------------------------------------------------------------------------------- /watchme/command/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Copyright (C) 2019-2022 Vanessa Sochat. 4 | 5 | This Source Code Form is subject to the terms of the 6 | Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed 7 | with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | 10 | """ 11 | 12 | from .create import create_watcher_base, create_watcher 13 | 14 | from .commit import ( 15 | write_timestamp, 16 | get_earliest_commit, 17 | get_commits, 18 | git_commit, 19 | git_clone, 20 | git_add, 21 | git_date, 22 | git_show, 23 | ) 24 | from .utils import get_watchers, list_task, list_watcher, list_watcher_types 25 | -------------------------------------------------------------------------------- /watchme/command/create.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Copyright (C) 2019-2022 Vanessa Sochat. 4 | 5 | This Source Code Form is subject to the terms of the 6 | Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed 7 | with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | """ 10 | 11 | from watchme.utils import run_command, mkdir_p 12 | from watchme.defaults import WATCHME_WATCHER, WATCHME_BASE_DIR 13 | from watchme.logger import bot 14 | from watchme.config import generate_watcher_config 15 | import os 16 | 17 | 18 | def create_watcher(name=None, watcher_type=None, base=None): 19 | """create a watcher, meaning a folder with a configuration and 20 | initialized git repo. 21 | 22 | Parameters 23 | ========== 24 | name: the watcher to create, uses default or WATCHME_WATCHER 25 | watcher_type: the type of watcher to create. defaults to 26 | WATCHER_DEFAULT_TYPE 27 | base: The watcher base to use (defaults to $HOME/.watchme) 28 | """ 29 | if name is None: 30 | name = WATCHME_WATCHER 31 | 32 | if base is None: 33 | base = WATCHME_BASE_DIR 34 | 35 | # Create the repository folder 36 | repo = os.path.join(base, name) 37 | 38 | if not os.path.exists(repo): 39 | 40 | bot.info("Adding watcher %s..." % repo) 41 | mkdir_p(repo) 42 | 43 | # Ensure no gpg signing happens 44 | run_command("git --git-dir=%s/.git init" % repo) 45 | run_command("git --git-dir=%s/.git config commit.gpgsign false" % repo) 46 | 47 | # Add the watcher configuration file 48 | generate_watcher_config(repo, watcher_type) 49 | run_command("git -C %s add watchme.cfg" % repo) 50 | return repo 51 | 52 | else: 53 | bot.info("%s already exists: %s" % (name, repo)) 54 | 55 | 56 | def create_watcher_base(name=None, base=None): 57 | """create a watch base and default repo, if it doesn't already exist. 58 | 59 | Parameters 60 | ========== 61 | name: the watcher to create, uses default or WATCHME_WATCHER 62 | base: the watcher base, defaults to WATCHME_BASE_DIR 63 | """ 64 | if base is None: 65 | base = WATCHME_BASE_DIR 66 | 67 | if name is None: 68 | name = WATCHME_WATCHER 69 | 70 | if not os.path.exists(base): 71 | bot.info("Creating %s..." % base) 72 | mkdir_p(base) 73 | -------------------------------------------------------------------------------- /watchme/config/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.logger import bot 6 | from watchme.defaults import WATCHME_WATCHER, WATCHME_BASE_DIR, WATCHME_DEFAULT_TYPE 7 | from watchme.utils import get_installdir, read_file 8 | import configparser 9 | import getpass 10 | import shutil 11 | import os 12 | 13 | 14 | # CONFIG TEMPLATES ############################################################# 15 | 16 | 17 | def get_configfile_template(): 18 | """return the full path to the default configuration file""" 19 | return _get_config("watchme.cfg") 20 | 21 | 22 | def _get_config(name): 23 | """shared function to return a file in the config directory""" 24 | return os.path.abspath(os.path.join(get_installdir(), "config", name)) 25 | 26 | 27 | # ACTIVE CONFIG FILES ########################################################## 28 | 29 | 30 | def get_configfile(name, base=None): 31 | """return the full path to a specific watcher configuration""" 32 | if base is None: 33 | base = WATCHME_BASE_DIR 34 | 35 | configfile = os.path.join(base, name, "watchme.cfg") 36 | check_exists(configfile) 37 | return configfile 38 | 39 | 40 | # CONFIG IO #################################################################### 41 | 42 | 43 | def write_config(filename, config, mode="w"): 44 | """use configparser to write a config object to filename""" 45 | with open(filename, mode) as filey: 46 | config.write(filey) 47 | return filename 48 | 49 | 50 | def read_config(filename): 51 | """use configparser to write a config object to filename""" 52 | check_exists(filename) 53 | config = configparser.ConfigParser() 54 | config.read(filename) 55 | return config 56 | 57 | 58 | # WATCHER CONFIG ############################################################### 59 | 60 | 61 | def generate_watcher_config(path, watcher_type=None): 62 | """generate a watcher config, meaning a watcher folder in the watchme 63 | base folder. 64 | 65 | Parameters 66 | ========== 67 | path: the path to the watcher repository 68 | """ 69 | check_exists(path) 70 | configfile = get_configfile_template() 71 | watcher_config = os.path.join(path, "watchme.cfg") 72 | if not os.path.exists(watcher_config): 73 | bot.info("Generating watcher config %s" % watcher_config) 74 | shutil.copyfile(configfile, watcher_config) 75 | 76 | # Complete generation includes the watcher type 77 | if watcher_type is None: 78 | watcher_type = WATCHME_DEFAULT_TYPE 79 | 80 | # The template config has the section, but just in case 81 | config = read_config(configfile) 82 | if "watcher" not in config.sections(): 83 | config.add_section("watcher") 84 | config["watcher"]["type"] = watcher_type 85 | 86 | # Save to file 87 | write_config(watcher_config, config) 88 | 89 | 90 | # CONFIG HELPERS ############################################################### 91 | 92 | 93 | def check_exists(filename): 94 | """a general helper function to check for existence, and exit if not found.""" 95 | if not os.path.exists(filename): 96 | bot.exit("Cannot find %s" % filename) 97 | -------------------------------------------------------------------------------- /watchme/config/watchme.cfg: -------------------------------------------------------------------------------- 1 | [watcher] 2 | active = false 3 | type = urls 4 | 5 | -------------------------------------------------------------------------------- /watchme/defaults.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.logger import bot 6 | from watchme.utils import get_userhome 7 | import os 8 | import sys 9 | 10 | 11 | ################################################################################ 12 | # environment / options 13 | ################################################################################ 14 | 15 | 16 | def getenv(variable_key, default=None, required=False, silent=True): 17 | """attempt to get an environment variable. If the variable 18 | is not found, None is returned. 19 | 20 | Parameters 21 | ========== 22 | variable_key: the variable name 23 | required: exit with error if not found 24 | silent: Do not print debugging information for variable 25 | """ 26 | variable = os.environ.get(variable_key, default) 27 | if variable is None and required: 28 | bot.error("Cannot find environment variable %s, exiting." % variable_key) 29 | sys.exit(1) 30 | 31 | if not silent and variable is not None: 32 | bot.verbose("%s found as %s" % (variable_key, variable)) 33 | 34 | return variable 35 | 36 | 37 | ################################################################################ 38 | # Helpme 39 | 40 | USERHOME = get_userhome() 41 | 42 | WATCHME_WATCHER = getenv("WATCHME_WATCHER", "watcher") 43 | _config = os.path.join(USERHOME, ".watchme") 44 | WATCHME_BASE_DIR = getenv("WATCHME_BASE_DIR", _config) 45 | WATCHME_WORKERS = int(getenv("WATCHME_WORKERS", 9)) 46 | 47 | # The types of valid watchers (currently only urls). Corresponds with 48 | # a folder under "main/watchers" 49 | WATCHME_TASK_TYPES = ["urls", "url", "gpu", "psutils", "results"] 50 | WATCHME_DEFAULT_TYPE = "urls" 51 | 52 | # Parameters not allowed for task clients, set by TaskBase 53 | WATCHME_NOTALLOWED_PARAMS = ["type", "active"] 54 | -------------------------------------------------------------------------------- /watchme/logger/__init__.py: -------------------------------------------------------------------------------- 1 | from .message import bot 2 | from .progress import ProgressBar 3 | from .namer import RobotNamer 4 | -------------------------------------------------------------------------------- /watchme/logger/spinner.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | import sys 6 | import time 7 | import threading 8 | from random import choice 9 | 10 | 11 | class Spinner: 12 | spinning = False 13 | delay = 0.1 14 | 15 | @staticmethod 16 | def spinning_cursor(): 17 | while 1: 18 | for cursor in "|/-\\": 19 | yield cursor 20 | 21 | @staticmethod 22 | def balloons_cursor(): 23 | while 1: 24 | for cursor in ". o O @ *": 25 | yield cursor 26 | 27 | @staticmethod 28 | def changing_arrows(): 29 | while 1: 30 | for cursor in "<^>v": 31 | yield cursor 32 | 33 | def select_generator(self, generator): 34 | if generator is None: 35 | generator = choice(["cursor", "arrow", "balloons"]) 36 | 37 | return generator 38 | 39 | def __init__(self, delay=None, generator=None): 40 | generator = self.select_generator(generator) 41 | 42 | if generator == "cursor": 43 | self.spinner_generator = self.spinning_cursor() 44 | elif generator == "arrow": 45 | self.spinner_generator = self.changing_arrows() 46 | elif generator == "balloons": 47 | self.spinner_generator = self.balloons_cursor() 48 | if delay is None: 49 | delay = 0.2 50 | else: 51 | self.spinner_generator = self.spinning_cursor() 52 | 53 | if delay and float(delay): 54 | self.delay = delay 55 | 56 | def run(self): 57 | while self.spinning: 58 | sys.stdout.write(next(self.spinner_generator)) 59 | sys.stdout.flush() 60 | time.sleep(self.delay) 61 | sys.stdout.write("\b") 62 | sys.stdout.flush() 63 | 64 | def start(self): 65 | self.spinning = True 66 | threading.Thread(target=self.run).start() 67 | 68 | def stop(self): 69 | self.spinning = False 70 | time.sleep(self.delay) 71 | -------------------------------------------------------------------------------- /watchme/tasks/worker.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.logger import bot 6 | from watchme.defaults import WATCHME_WORKERS 7 | import multiprocessing 8 | import itertools 9 | import time 10 | import signal 11 | import sys 12 | 13 | 14 | class Workers: 15 | def __init__(self, workers=None, show_progress=False): 16 | 17 | if workers is None: 18 | workers = WATCHME_WORKERS 19 | self.workers = workers 20 | self.show_progress = show_progress 21 | bot.debug("Using %s workers for multiprocess." % (self.workers)) 22 | 23 | def start(self): 24 | bot.debug("Starting multiprocess") 25 | self.start_time = time.time() 26 | 27 | def end(self): 28 | self.end_time = time.time() 29 | self.runtime = self.runtime = self.end_time - self.start_time 30 | bot.debug("Ending multiprocess, runtime: %s sec" % (self.runtime)) 31 | 32 | def run(self, funcs, tasks): 33 | """run will send a list of tasks, a tuple with arguments, through a function. 34 | the arguments should be ordered correctly. 35 | 36 | Parameters 37 | ========== 38 | funcs: the functions to run with multiprocessing.pool, a dictionary 39 | with lookup by the task name 40 | tasks: a dict of tasks, each task name (key) with a 41 | tuple of arguments to process 42 | """ 43 | # Number of tasks must == number of functions 44 | assert len(funcs) == len(tasks) 45 | 46 | # Keep track of some progress for the user 47 | progress = 1 48 | total = len(tasks) 49 | 50 | # if we don't have tasks, don't run 51 | if not tasks: 52 | return 53 | 54 | # results will also have the same key to look up 55 | finished = dict() 56 | results = [] 57 | 58 | try: 59 | prefix = "[%s/%s]" % (progress, total) 60 | if self.show_progress: 61 | bot.show_progress(0, total, length=35, prefix=prefix) 62 | pool = multiprocessing.Pool(self.workers, init_worker) 63 | 64 | self.start() 65 | for key, params in tasks.items(): 66 | func = funcs[key] 67 | if not self.show_progress: 68 | bot.info("Processing task %s:%s" % (key, params)) 69 | result = pool.apply_async(multi_wrapper, multi_package(func, [params])) 70 | 71 | # Store the key with the result 72 | results.append((key, result)) 73 | 74 | while len(results) > 0: 75 | pair = results.pop() 76 | key, result = pair 77 | result.wait() 78 | if self.show_progress: 79 | bot.show_progress(progress, total, length=35, prefix=prefix) 80 | progress += 1 81 | prefix = "[%s/%s]" % (progress, total) 82 | finished[key] = result.get() 83 | 84 | self.end() 85 | pool.close() 86 | pool.join() 87 | 88 | except (KeyboardInterrupt, SystemExit): 89 | bot.error("Keyboard interrupt detected, terminating workers!") 90 | pool.terminate() 91 | sys.exit(1) 92 | 93 | except: 94 | bot.exit("Error running task.") 95 | 96 | return finished 97 | 98 | 99 | # Supporting functions for MultiProcess Worker 100 | def init_worker(): 101 | signal.signal(signal.SIGINT, signal.SIG_IGN) 102 | 103 | 104 | def multi_wrapper(func_args): 105 | function, kwargs = func_args 106 | return function(**kwargs) 107 | 108 | 109 | def multi_package(func, kwargs): 110 | zipped = zip(itertools.repeat(func), kwargs) 111 | return zipped 112 | -------------------------------------------------------------------------------- /watchme/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vsoch/watchme/f209d3d4bf99a25cd2dcaeaa2431cf3ecfe68585/watchme/tests/__init__.py -------------------------------------------------------------------------------- /watchme/tests/helpers.sh: -------------------------------------------------------------------------------- 1 | runTest() { 2 | 3 | # The first argument is the code we should get 4 | ERROR="${1:-}" 5 | shift 6 | OUTPUT=${1:-} 7 | shift 8 | 9 | "$@" > "${OUTPUT}" 2>&1 10 | RETVAL="$?" 11 | 12 | if [ "$ERROR" = "0" -a "$RETVAL" != "0" ]; then 13 | echo "$@ (retval=$RETVAL) ERROR" 14 | cat ${OUTPUT} 15 | echo "Output in ${OUTPUT}" 16 | exit 1 17 | elif [ "$ERROR" != "0" -a "$RETVAL" = "0" ]; then 18 | echo "$@ (retval=$RETVAL) ERROR" 19 | echo "Output in ${OUTPUT}" 20 | cat ${OUTPUT} 21 | exit 1 22 | else 23 | echo "$@ (retval=$RETVAL) OK" 24 | fi 25 | } 26 | -------------------------------------------------------------------------------- /watchme/tests/test_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Include help functions 4 | . helpers.sh 5 | 6 | echo 7 | echo "************** START: test_client.sh **********************" 8 | 9 | # Create temporary testing directory 10 | echo "Creating temporary directory to work in." 11 | tmpdir=$(mktemp -d) 12 | output=$(mktemp ${tmpdir:-/tmp}/watchme_test.XXXXXX) 13 | 14 | echo "Testing help commands..." 15 | 16 | # Test help for all commands 17 | for command in init get export create add-task inspect list protect remove run activate deactivate schedule edit; 18 | do 19 | runTest 0 $output watchme $command --help 20 | done 21 | 22 | echo "#### Testing WATCHME_BASE_DIR setting" 23 | 24 | # set the watchme base, create watcher 25 | echo "#### Testing watchme create" 26 | export WATCHME_BASE_DIR="${tmpdir}" 27 | runTest 0 $output watchme create watcher 28 | 29 | # Does the watcher directory exist, and the config file? 30 | runTest 0 $output test -d "$tmpdir/watcher" 31 | runTest 0 $output test -d "$tmpdir/watcher/.git" 32 | runTest 0 $output test -f "$tmpdir/watcher/watchme.cfg" 33 | 34 | # Test downloading another watcher 35 | echo "#### Testing watchme get" 36 | runTest 0 $output watchme get https://www.github.com/vsoch/watchme-github-repos 37 | runTest 0 $output test -d "$tmpdir/watchme-github-repos/.git" 38 | runTest 0 $output test -f "$tmpdir/watchme-github-repos/watchme.cfg" 39 | 40 | echo "With a custom name..." 41 | runTest 0 $output watchme get https://www.github.com/vsoch/watchme-github-repos github 42 | runTest 0 $output test -d "$tmpdir/github/.git" 43 | runTest 0 $output test -f "$tmpdir/github/watchme.cfg" 44 | 45 | echo "#### Testing watchme export" 46 | runTest 0 $output watchme export github task-singularity result.json --json 47 | runTest 0 $output watchme export github task-singularity result.json 48 | runTest 0 $output watchme export github task-singularity TIMESTAMP 49 | runTest 255 $output watchme export github task-singularity doesnt-exist.json 50 | 51 | echo "#### Testing watchme inspect" 52 | runTest 0 $output watchme inspect github task-sregistry 53 | runTest 255 $output watchme inspect github task-doesntexist 54 | runTest 255 $output watchme inspect github invalid-task 55 | 56 | echo "#### Testing watchme list" 57 | runTest 0 $output watchme list 58 | runTest 0 $output watchme list github 59 | runTest 0 $output watchme list github task-sregistry 60 | 61 | echo "#### Testing watchme protect and freeze" 62 | runTest 0 $output watchme protect watchme-github-repos on 63 | runTest 255 $output watchme remove watchme-github-repos --delete 64 | runTest 0 $output watchme protect watchme-github-repos off 65 | runTest 0 $output watchme protect watchme-github-repos freeze 66 | runTest 255 $output watchme remove watchme-github-repos --delete 67 | runTest 0 $output watchme protect watchme-github-repos unfreeze 68 | runTest 0 $output watchme protect watchme-github-repos off 69 | runTest 0 $output watchme remove watchme-github-repos --delete 70 | 71 | echo "#### Testing watchme activate and deactivate" 72 | runTest 0 $output watchme deactivate github 73 | runTest 255 $output watchme run github 74 | runTest 0 $output watchme activate github 75 | runTest 0 $output watchme deactivate github task-singularity 76 | 77 | echo "#### Testing watchme run" 78 | runTest 0 $output watchme run github task-spython --test 79 | runTest 0 $output watchme run github task-expfactory 80 | runTest 0 $output watchme run github task-expfactory --no-progress 81 | runTest 0 $output watchme run github task-expfactory --serial 82 | runTest 0 $output watchme run github task-doesntexist 83 | 84 | echo "#### Testing watchme monitor" 85 | runTest 0 $output watchme monitor sleep 2 86 | runTest 0 $output watchme monitor github sleep 2 87 | runTest 0 $output test -d "$tmpdir/github/decorator-psutils-sleep-2" 88 | runTest 0 $output test -f "$tmpdir/github/decorator-psutils-sleep-2/result.json" 89 | runTest 0 $output test -f "$tmpdir/github/decorator-psutils-sleep-2/TIMESTAMP" 90 | runTest 0 $output watchme monitor github sleep 2 --name avocado 91 | runTest 0 $output test -d "$tmpdir/github/decorator-psutils-avocado" 92 | runTest 0 $output test -f "$tmpdir/github/decorator-psutils-avocado/result.json" 93 | runTest 0 $output test -f "$tmpdir/github/decorator-psutils-avocado/TIMESTAMP" 94 | 95 | 96 | echo "Finish testing basic client" 97 | rm -rf ${tmpdir} 98 | -------------------------------------------------------------------------------- /watchme/tests/test_decorators.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | __author__ = "Vanessa Sochat" 4 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 5 | __license__ = "MPL 2.0" 6 | 7 | from watchme import get_watcher 8 | from watchme.command import create_watcher 9 | import unittest 10 | import tempfile 11 | import shutil 12 | 13 | 14 | print("####################################################### test_decorators") 15 | 16 | 17 | class TestDecorators(unittest.TestCase): 18 | def setUp(self): 19 | self.base = tempfile.mkdtemp() 20 | self.repo = create_watcher("pancakes", base=self.base) 21 | self.cli = get_watcher("pancakes", base=self.base) 22 | 23 | def tearDown(self): 24 | shutil.rmtree(self.base) 25 | 26 | def test_psutils_monitor(self): 27 | """test creation function, and basic watcher config""" 28 | print("Testing psutilsc.decorators.TerminalRunner") 29 | from watchme.tasks.decorators import TerminalRunner 30 | 31 | runner = TerminalRunner("sleep 2") 32 | runner.run() 33 | timepoints = runner.wait("monitor_pid_task") 34 | self.assertTrue(len(timepoints) == 1) 35 | 36 | 37 | if __name__ == "__main__": 38 | unittest.main() 39 | -------------------------------------------------------------------------------- /watchme/tests/test_psutils_decorator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from watchme.watchers.psutils.decorators import monitor_resources 4 | from time import sleep 5 | 6 | # Here we create a decorator to monitor "my func." Specifically, we: 7 | # - want to use the watcher "decorator" that already exists. If we want to 8 | # create on the fly, we would set creat=True 9 | # - will record metrics every 3 seconds 10 | # - to have somewhat of an impact on system resources we make a long list 11 | # - we test to ensure that something is returned ("Hello!") 12 | 13 | 14 | @monitor_resources("system", seconds=3) 15 | def myfunc(iters, pause): 16 | long_list = [] 17 | print("Generating a long list, pause is %s and iters is %s" % (pause, iters)) 18 | for i in range(iters): 19 | long_list = long_list + (i * 10) * ["pancakes"] 20 | print("i is %s, sleeping %s seconds" % (i, pause)) 21 | sleep(pause) 22 | 23 | return len(long_list) 24 | 25 | 26 | # ensure the function runs when the file is called 27 | if __name__ == "__main__": 28 | print("Calling myfunc with 2 iters") 29 | result = myfunc(2, 2) 30 | print("Result list has length %s" % result) 31 | -------------------------------------------------------------------------------- /watchme/tests/test_utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Include help functions 4 | . helpers.sh 5 | 6 | echo 7 | echo "************** START: test_utils.sh **********************" 8 | 9 | # Create temporary testing directory 10 | echo "Creating temporary directory to work in." 11 | tmpdir=$(mktemp -d) 12 | output=$(mktemp ${tmpdir:-/tmp}/watchme_test.XXXXXX) 13 | 14 | export WATCHMEENV_density=0.24 15 | 16 | echo "#### Testing watchme get_watchme_env" 17 | runTest 0 $output python -c "from watchme.utils import get_watchme_env as ge;import sys;env = ge();print(env.get('density'))" | grep density || exit 1 18 | 19 | echo "Finish testing utils" 20 | rm -rf ${tmpdir} 21 | -------------------------------------------------------------------------------- /watchme/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .terminal import run_command, get_installdir, get_watchme_env, which 2 | from .fileio import ( 3 | copyfile, 4 | get_userhome, 5 | get_user, 6 | get_host, 7 | generate_temporary_file, 8 | mkdir_p, 9 | get_tmpdir, 10 | print_json, 11 | read_file, 12 | read_json, 13 | write_file, 14 | write_json, 15 | ) 16 | -------------------------------------------------------------------------------- /watchme/utils/fileio.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | import errno 6 | import os 7 | import tempfile 8 | import json 9 | import socket 10 | import shutil 11 | import sys 12 | import getpass 13 | 14 | 15 | from watchme.logger import bot 16 | 17 | # FOLDER OPERATIONS ############################################################ 18 | 19 | 20 | def get_userhome(): 21 | """get the user home based on the effective uid""" 22 | return os.path.expanduser("~") 23 | 24 | 25 | def get_user(): 26 | """return the active user""" 27 | return getpass.getuser() 28 | 29 | 30 | def get_host(): 31 | """return the hostname""" 32 | return socket.gethostname() 33 | 34 | 35 | def mkdir_p(path): 36 | """mkdir_p attempts to get the same functionality as mkdir -p 37 | 38 | Paramters 39 | ========= 40 | param path: the path to create. 41 | """ 42 | try: 43 | os.makedirs(path) 44 | except OSError as e: 45 | if e.errno == errno.EEXIST and os.path.isdir(path): 46 | pass 47 | else: 48 | bot.error("Error creating path %s, exiting." % path) 49 | sys.exit(1) 50 | 51 | 52 | # FILE OPERATIONS ############################################################## 53 | 54 | 55 | def generate_temporary_file(folder=None, prefix="watchme", ext=None): 56 | """write a temporary file, in base directory with a particular extension. 57 | 58 | Parameters 59 | ========== 60 | folder: the base directory to write in. 61 | prefix: the prefix to use 62 | ext: the extension to use. 63 | 64 | """ 65 | if folder is None: 66 | folder = tempfile.gettempdir() 67 | tmp = next(tempfile._get_candidate_names()) 68 | tmp = "%s/%s.%s" % (folder, prefix, tmp) 69 | 70 | # Does the user want an extension? 71 | if ext is not None: 72 | tmp = "%s.%s" % (tmp, ext) 73 | 74 | return tmp 75 | 76 | 77 | def get_tmpdir(prefix="", create=True): 78 | """get a temporary directory for an operation. If SREGISTRY_TMPDIR 79 | is set, return that. Otherwise, return the output of tempfile.mkdtemp 80 | 81 | Parameters 82 | ========== 83 | prefix: Given a need for a sandbox (or similar), we will need to 84 | create a subfolder *within* the SREGISTRY_TMPDIR. 85 | create: boolean to determine if we should create folder (True) 86 | """ 87 | tmpdir = tempfile.gettempdir() 88 | prefix = prefix or "watchme-tmp" 89 | prefix = "%s.%s" % (prefix, next(tempfile._get_candidate_names())) 90 | tmpdir = os.path.join(tmpdir, prefix) 91 | 92 | if not os.path.exists(tmpdir) and create is True: 93 | os.mkdir(tmpdir) 94 | 95 | return tmpdir 96 | 97 | 98 | def copyfile(source, destination, force=True): 99 | """copy a file from a source to its destination.""" 100 | if os.path.exists(destination) and force is True: 101 | os.remove(destination) 102 | shutil.copyfile(source, destination) 103 | return destination 104 | 105 | 106 | def write_file(filename, content, mode="w"): 107 | """write_file will open a file, "filename" and write content, "content" 108 | and properly close the file 109 | """ 110 | with open(filename, mode) as filey: 111 | filey.writelines(content) 112 | return filename 113 | 114 | 115 | def write_json(json_obj, filename, mode="w", print_pretty=True): 116 | """write_json will (optionally,pretty print) a json object to file 117 | 118 | Parameters 119 | ========== 120 | json_obj: the dict to print to json 121 | filename: the output file to write to 122 | pretty_print: if True, will use nicer formatting 123 | """ 124 | with open(filename, mode) as filey: 125 | if print_pretty: 126 | filey.writelines(print_json(json_obj)) 127 | else: 128 | filey.writelines(json.dumps(json_obj)) 129 | return filename 130 | 131 | 132 | def print_json(json_obj): 133 | """just dump the json in a "pretty print" format""" 134 | return json.dumps(json_obj, indent=4, separators=(",", ": ")) 135 | 136 | 137 | def read_file(filename, mode="r", readlines=True): 138 | """write_file will open a file, "filename" and write content, "content" 139 | and properly close the file 140 | """ 141 | with open(filename, mode) as filey: 142 | if readlines is True: 143 | content = filey.readlines() 144 | else: 145 | content = filey.read() 146 | return content 147 | 148 | 149 | def read_json(filename, mode="r"): 150 | """read_json reads in a json file and returns 151 | the data structure as dict. 152 | """ 153 | with open(filename, mode) as filey: 154 | data = json.load(filey) 155 | return data 156 | -------------------------------------------------------------------------------- /watchme/utils/terminal.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | 6 | from subprocess import Popen, PIPE, STDOUT 7 | import os 8 | import re 9 | import shlex 10 | 11 | # Terminal Commands 12 | 13 | 14 | def which(software, strip_newline=True): 15 | """get_install will return the path to where an executable is installed.""" 16 | if software is None: 17 | software = "watchme" 18 | cmd = "which %s" % software 19 | try: 20 | result = run_command(cmd) 21 | if strip_newline is True: 22 | result["message"] = result["message"].strip("\n") 23 | if "message" in result: 24 | return result["message"] 25 | return result 26 | 27 | except: # FileNotFoundError 28 | return None 29 | 30 | 31 | def get_installdir(): 32 | """get_installdir returns the installation directory of the application""" 33 | return os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 34 | 35 | 36 | def run_command(cmd, sudo=False): 37 | """run_command uses subprocess to send a command to the terminal. 38 | 39 | Parameters 40 | ========== 41 | cmd: the command to send, should be a list for subprocess 42 | error_message: the error message to give to user if fails, 43 | if none specified, will alert that command failed. 44 | 45 | """ 46 | 47 | cmd = shlex.split(cmd) 48 | 49 | if sudo is True: 50 | cmd = ["sudo"] + cmd 51 | 52 | try: 53 | output = Popen(cmd, stderr=STDOUT, stdout=PIPE) 54 | 55 | except FileNotFoundError: 56 | cmd.pop(0) 57 | output = Popen(cmd, stderr=STDOUT, stdout=PIPE) 58 | 59 | t = output.communicate()[0], output.returncode 60 | output = {"message": t[0], "return_code": t[1]} 61 | 62 | if isinstance(output["message"], bytes): 63 | output["message"] = output["message"].decode("utf-8") 64 | 65 | return output 66 | 67 | 68 | # Environment 69 | 70 | 71 | def convert2boolean(arg): 72 | """ 73 | convert2boolean is used for environmental variables 74 | that must be returned as boolean 75 | """ 76 | if not isinstance(arg, bool): 77 | return arg.lower() in ("yes", "true", "t", "1", "y") 78 | return arg 79 | 80 | 81 | def get_watchme_env(prefix="WATCHMEENV_"): 82 | """get any environment variables that start with WATCMEENV_, return the 83 | dictionary to the user. 84 | """ 85 | environ = {} 86 | 87 | # First extract variables from the environment 88 | for key, value in os.environ.items(): 89 | 90 | # Variables that are specified, or start with WATCHMEENV included 91 | if key.startswith(prefix): 92 | 93 | # Replace the WATCHMEENV_ if present 94 | key = re.sub("^%s" % prefix, "", key) 95 | 96 | # Don't include empty strings 97 | if value not in ["", None]: 98 | 99 | environ[key] = value 100 | 101 | return environ 102 | -------------------------------------------------------------------------------- /watchme/version.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | 6 | __version__ = "0.0.29" 7 | AUTHOR = "Vanessa Sochat" 8 | AUTHOR_EMAIL = "vsochat@stanford.edu" 9 | NAME = "watchme" 10 | PACKAGE_URL = "http://www.github.com/vsoch/watchme" 11 | KEYWORDS = "web, changes, cron, reproducible, version-control" 12 | DESCRIPTION = "reproducible monitoring client with exporters" 13 | LICENSE = "LICENSE" 14 | 15 | INSTALL_REQUIRES = ( 16 | ("python-crontab", {"min_version": "2.3.6"}), 17 | ("configparser", {"min_version": "3.5.3"}), 18 | ("requests", {"min_version": "2.21.0"}), 19 | ) 20 | 21 | ## beautiful soup selection task 22 | 23 | INSTALL_URLS_DYNAMIC = ( 24 | ("beautifulsoup4", {"min_version": "4.6.0"}), 25 | ("lxml", {"min_version": "4.1.1"}), 26 | ) 27 | 28 | INSTALL_PSUTILS = (("psutil", {"min_version": "5.4.3"}),) 29 | 30 | # Install all watchers and exporters 31 | INSTALL_ALL = INSTALL_REQUIRES + INSTALL_PSUTILS + INSTALL_URLS_DYNAMIC 32 | 33 | # Install all watchers 34 | INSTALL_WATCHERS = INSTALL_REQUIRES + INSTALL_PSUTILS + INSTALL_URLS_DYNAMIC 35 | -------------------------------------------------------------------------------- /watchme/watchers/README.md: -------------------------------------------------------------------------------- 1 | # Watchers 2 | 3 | Each of these is a Watcher that the user can request. 4 | 5 | - [urls](urls) to watch for changes in websites (default) 6 | - [psutils](psutils) to get basic system statistics 7 | 8 | ## Watcher Base 9 | 10 | The watcher base is defined in the [init](__init__.py) file here. 11 | 12 | - [schedules](schedule.py) to interact with cronjobs 13 | - [exporters](data.py) (extras) and the default export of temporal data 14 | - [settings](settings.py) meaning add/remove from the watchme.cfg 15 | -------------------------------------------------------------------------------- /watchme/watchers/data.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2022, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.logger import bot 6 | from watchme.command import get_commits, git_show, git_date 7 | import os 8 | import json 9 | 10 | # Data Exports 11 | 12 | 13 | def export_dict( 14 | self, 15 | task, 16 | filename, 17 | name=None, 18 | export_json=False, 19 | from_commit=None, 20 | to_commit=None, 21 | base=None, 22 | ): 23 | """Export a data frame of changes for a filename over time. 24 | 25 | Parameters 26 | ========== 27 | task: the task folder for the watcher to look in 28 | name: the name of the watcher, defaults to the client's 29 | base: the base of watchme to look for the task folder 30 | from_commit: the commit to start at 31 | to_commit: the commit to go to 32 | grep: the expression to match (not used if None) 33 | filename: the filename to filter to. Includes all files if not specified. 34 | """ 35 | if name is None: 36 | name = self.name 37 | 38 | if base is None: 39 | base = self.base 40 | 41 | # Quit early if the task isn't there 42 | if not self.has_task(task) and not task.startswith("decorator"): 43 | bot.exit("%s is not a valid task or decorator for %s" % (task, name)) 44 | 45 | repo = os.path.join(base, self.name) 46 | if not os.path.exists(repo): 47 | bot.exit("%s does not exist." % repo) 48 | 49 | filepath = os.path.join(base, self.name, task, filename) 50 | 51 | # Ensure that the filename exists in the repository 52 | if not os.path.exists(filepath): 53 | bot.exit("%s does not exist for watcher %s" % (filepath, name)) 54 | 55 | # Now filepath must be relative to the repo 56 | filepath = os.path.join(task, filename) 57 | 58 | commits = get_commits( 59 | repo=repo, 60 | from_commit=from_commit, 61 | to_commit=to_commit, 62 | grep="ADD results %s" % task, 63 | filename=filepath, 64 | ) 65 | 66 | # Keep lists of commits, dates, content 67 | result = {"commits": [], "dates": [], "content": []} 68 | 69 | # Empty content (or other) returns None 70 | for commit in commits: 71 | content = git_show(repo=repo, commit=commit, filename=filepath) 72 | 73 | if export_json is True: 74 | content = json.loads(content) 75 | 76 | # If it's a list, add it to content 77 | if isinstance(content, list): 78 | result["content"] += content 79 | # Otherwise, append 80 | else: 81 | result["content"].append(content) 82 | 83 | result["dates"].append(git_date(repo=repo, commit=commit)) 84 | result["commits"].append(commit) 85 | return result 86 | -------------------------------------------------------------------------------- /watchme/watchers/gpu/README.md: -------------------------------------------------------------------------------- 1 | # GPU 2 | 3 | The psutils has one or more [tasks](tasks.py) that can be added to a watcher. 4 | Unlike a traditional watcher task group, psutils also offers a decorator 5 | to collect system metrics during the runtime of a function. The metrics 6 | are saved to an existing watcher based on the function name, with 7 | a folder starting with `decorator-`. For example, let's create a watcher 8 | folder to use with a new decorator: 9 | 10 | ```bash 11 | $ watchme create decorator 12 | ``` 13 | 14 | And then here is how we would use the psutils decorator with some long running 15 | function: 16 | 17 | ```python 18 | from watchme.watchers.psutils.decorators import monitor_resources 19 | 20 | @monitor_resources('decorator', seconds=3) 21 | def myfunc(): 22 | long_list = [] 23 | for i in range(100): 24 | long_list = long_list + (i*10)*['pancakes'] 25 | sleep 10 26 | ``` 27 | 28 | The first parameter is the watcher name (it must exist), the second is 29 | how often to collect metrics (in seconds) and the third is a single or list 30 | of psutils tasks that you want to run. 31 | -------------------------------------------------------------------------------- /watchme/watchers/gpu/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2021, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from .pynvml import nvmlInit 6 | from watchme.tasks import TaskBase 7 | from watchme.logger import bot 8 | from watchme.utils import get_user, get_host 9 | 10 | 11 | class Task(TaskBase): 12 | 13 | required_params = [] 14 | 15 | def __init__(self, name, params=None, **kwargs): 16 | 17 | if params is None: 18 | params = {} 19 | 20 | self.type = "gpu" 21 | self.assert_gpu() 22 | 23 | # If the user doesn't provide a file name, name based on task 24 | if "_save" in kwargs: 25 | if "file_name" not in params: 26 | params["file_name"] = "%s_%s.json" % ( 27 | get_host().lower(), 28 | get_user().lower(), 29 | ) 30 | 31 | # Handles setting the name, setting params, and validate 32 | super(Task, self).__init__(name, params, **kwargs) 33 | 34 | def assert_gpu(self): 35 | """has_gpu is run from the getgo to see if there are any libraries 36 | for the client to read from. If not, we alert the user and exit. 37 | """ 38 | try: 39 | nvmlInit() 40 | except: 41 | bot.exit("NVML Shared Library Not Found.") 42 | 43 | def export_func(self): 44 | """this function should return the correct task (from the tasks.py 45 | in the same folder) based on some logic of the params that are given 46 | by the user (self.params). If there is only one kind of function for 47 | the task, it's fairly easy to import and return it here. This 48 | function should take no arguments, but instead use the self.params 49 | already provided in the client. 50 | """ 51 | name = self.params.get("func", "gpu_task") 52 | 53 | if name == "gpu_task": 54 | from .tasks import gpu_task as func 55 | else: 56 | func = None 57 | 58 | return func 59 | -------------------------------------------------------------------------------- /watchme/watchers/gpu/decorators.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2021, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from functools import wraps 6 | from watchme.logger import bot 7 | from watchme.tasks.decorators import ProcessRunner 8 | from watchme import get_watcher 9 | 10 | 11 | def monitor_gpu(*args, **kwargs): 12 | """a decorator to monitor a function every 3 (or user specified) seconds. 13 | We include one or more task names that include data we want to extract. 14 | we get the pid of the running function, and then use the 15 | gpu_task from gpu to watch it. The functools "wraps" 16 | ensures that the (fargs, fkwargs) are passed from the calling function 17 | despite the wrapper. The following parameters can be provided to 18 | "monitor resources" 19 | 20 | Parameters 21 | ========== 22 | watcher: the watcher instance to use, used to save data to a "task" 23 | folder that starts with "decorator- folder. If not provided, 31 | defaults to the function name 32 | """ 33 | 34 | def inner(func): 35 | @wraps(func) 36 | def wrapper(*fargs, **fkwargs): 37 | 38 | # Typically the task folder is the index, so we will create 39 | # indices that start with decorator- 40 | result = None 41 | 42 | # The watcher is required, first keyword argument 43 | if not args: 44 | bot.error("A watcher name is required for the gpu decorator.") 45 | return result 46 | 47 | # Get a watcher to save results to 48 | watcher = get_watcher(args[0], create=kwargs.get("create", False)) 49 | 50 | # Start the function 51 | runner = ProcessRunner( 52 | seconds=kwargs.get("seconds", 3), 53 | skip=kwargs.get("skip", []), 54 | include=kwargs.get("include", []), 55 | only=kwargs.get("only", []), 56 | ) 57 | 58 | runner.run(func, *fargs, **fkwargs) 59 | result = runner.wait("gpu_task") 60 | 61 | # Save results (finishing runs) - key is folder created 62 | name = kwargs.get("name", func.__name__) 63 | key = "decorator-gpu-%s" % name 64 | results = {key: runner.timepoints} 65 | watcher.finish_runs(results) 66 | 67 | # Return function result to the user 68 | return result 69 | 70 | return wrapper 71 | 72 | return inner 73 | -------------------------------------------------------------------------------- /watchme/watchers/psutils/README.md: -------------------------------------------------------------------------------- 1 | # Psutils 2 | 3 | The psutils has one or more [tasks](tasks.py) that can be added to a watcher. 4 | Unlike a traditional watcher task group, psutils also offers a decorator 5 | to collect system metrics during the runtime of a function. The metrics 6 | are saved to an existing watcher based on the function name, with 7 | a folder starting with `decorator-`. For example, let's create a watcher 8 | folder to use with a new decorator: 9 | 10 | ```bash 11 | $ watchme create decorator 12 | ``` 13 | 14 | And then here is how we would use the psutils decorator with some long running 15 | function: 16 | 17 | ```python 18 | from watchme.watchers.psutils.decorators import monitor_resources 19 | 20 | @monitor_resources('decorator', seconds=3) 21 | def myfunc(): 22 | long_list = [] 23 | for i in range(100): 24 | long_list = long_list + (i*10)*['pancakes'] 25 | sleep 10 26 | ``` 27 | 28 | The first parameter is the watcher name (it must exist), the second is 29 | how often to collect metrics (in seconds) and the third is a single or list 30 | of psutils tasks that you want to run. 31 | -------------------------------------------------------------------------------- /watchme/watchers/psutils/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2021, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.tasks import TaskBase 6 | from watchme.utils import get_user, get_host 7 | 8 | 9 | class Task(TaskBase): 10 | 11 | required_params = [] 12 | 13 | def __init__(self, name, params=None, **kwargs): 14 | 15 | if params is None: 16 | params = {} 17 | 18 | self.type = "psutils" 19 | 20 | # If the user doesn't provide a file name, name based on task 21 | if "_save" in kwargs: 22 | if "file_name" not in params: 23 | params["file_name"] = "%s_%s.json" % ( 24 | get_host().lower(), 25 | get_user().lower(), 26 | ) 27 | 28 | # Handles setting the name, setting params, and validate 29 | super(Task, self).__init__(name, params, **kwargs) 30 | 31 | def export_func(self): 32 | """this function should return the correct task (from the tasks.py 33 | in the same folder) based on some logic of the params that are given 34 | by the user (self.params). If there is only one kind of function for 35 | the task, it's fairly easy to import and return it here. This 36 | function should take no arguments, but instead use the self.params 37 | already provided in the client. 38 | """ 39 | name = self.params.get("func", "cpu_task") 40 | 41 | if name == "cpu_task": 42 | from .tasks import cpu_task as func 43 | elif name == "memory_task": 44 | from .tasks import memory_task as func 45 | elif name == "monitor_pid_task": 46 | from .tasks import monitor_pid_task as func 47 | elif name == "net_task": 48 | from .tasks import net_task as func 49 | elif name == "python_task": 50 | from .tasks import python_task as func 51 | elif name == "sensors_task": 52 | from .tasks import sensors_task as func 53 | elif name == "system_task": 54 | from .tasks import system_task as func 55 | elif name == "users_task": 56 | from .tasks import users_task as func 57 | else: 58 | func = None 59 | 60 | return func 61 | -------------------------------------------------------------------------------- /watchme/watchers/psutils/decorators.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2021, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from functools import wraps 6 | from watchme.logger import bot 7 | from watchme.tasks.decorators import ProcessRunner 8 | from watchme import get_watcher 9 | 10 | 11 | def monitor_resources(*args, **kwargs): 12 | """a decorator to monitor a function every 3 (or user specified) seconds. 13 | We include one or more task names that include data we want to extract. 14 | we get the pid of the running function, and then use the 15 | monitor_pid_task from psutils to watch it. The functools "wraps" 16 | ensures that the (fargs, fkwargs) are passed from the calling function 17 | despite the wrapper. The following parameters can be provided to 18 | "monitor resources" 19 | 20 | Parameters 21 | ========== 22 | watcher: the watcher instance to use, used to save data to a "task" 23 | folder that starts with "decorator- folder. If not provided, 31 | defaults to the function name 32 | """ 33 | 34 | def inner(func): 35 | @wraps(func) 36 | def wrapper(*fargs, **fkwargs): 37 | 38 | # Typically the task folder is the index, so we will create 39 | # indices that start with decorator- 40 | result = None 41 | 42 | # The watcher is required, first keyword argument 43 | if not args: 44 | bot.error("A watcher name is required for the psutils decorator.") 45 | return result 46 | 47 | # Get a watcher to save results to 48 | watcher = get_watcher(args[0], create=kwargs.get("create", False)) 49 | 50 | # Start the function 51 | runner = ProcessRunner( 52 | seconds=kwargs.get("seconds", 3), 53 | skip=kwargs.get("skip", []), 54 | include=kwargs.get("include", []), 55 | only=kwargs.get("only", []), 56 | ) 57 | 58 | runner.run(func, *fargs, **fkwargs) 59 | result = runner.wait("monitor_pid_task") 60 | 61 | # Save results (finishing runs) - key is folder created 62 | name = kwargs.get("name", func.__name__) 63 | key = "decorator-psutils-%s" % name 64 | results = {key: runner.timepoints} 65 | watcher.finish_runs(results) 66 | 67 | # Return function result to the user 68 | return result 69 | 70 | return wrapper 71 | 72 | return inner 73 | -------------------------------------------------------------------------------- /watchme/watchers/results/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2021, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.tasks import TaskBase 6 | 7 | 8 | class Task(TaskBase): 9 | """a results task aims to use WatchMe as a database, meaning the functions 10 | are optimized to collect and record results.""" 11 | 12 | required_params = [] 13 | 14 | def __init__(self, name, params=None, **kwargs): 15 | 16 | if params is None: 17 | params = [] 18 | 19 | self.type = "results" 20 | 21 | # Handles setting the name, setting params, and validate 22 | super(Task, self).__init__(name, params, **kwargs) 23 | 24 | def export_func(self): 25 | """this function should return the correct task (from the tasks.py 26 | in the same folder) based on some logic of the params that are given 27 | by the user (self.params). If there is only one kind of function for 28 | the task, it's fairly easy to import and return it here. This 29 | function should take no arguments, but instead use the self.params 30 | already provided in the client. 31 | """ 32 | name = self.params.get("func", "from_env_task") 33 | 34 | if name == "from_env_task": 35 | from .tasks import from_env_task as func 36 | else: 37 | func = None 38 | 39 | return func 40 | -------------------------------------------------------------------------------- /watchme/watchers/results/tasks.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2021, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.utils import get_tmpdir, get_watchme_env, write_file 6 | import os 7 | import shutil 8 | 9 | 10 | def from_env_task(**kwargs): 11 | """Get some set of variables from the environment. We look for those 12 | defined in kwargs, but also any in the environment that start with 13 | the prefix WATCHMENEV_. They are saved to files that are equivalently 14 | named, and the idea is that we can track a changing (single) result, 15 | meaning that the timestamp is meaningful, or we can track 16 | values for many different results, so the timestamps just serve to 17 | record when each was recorded. 18 | 19 | Parameters 20 | ========== 21 | *: any number of parameters from the task configuration that will 22 | be looked for in the environment. 23 | """ 24 | results = [] 25 | 26 | # Create a temporary directory for results 27 | tmpdir = get_tmpdir() 28 | 29 | # First extract variables from the environment 30 | environ = get_watchme_env() 31 | for key, value in environ.items(): 32 | 33 | # Write the result to file (don't include extension) 34 | filename = os.path.join(tmpdir, key) 35 | write_file(filename, value) 36 | results.append(filename) 37 | 38 | # If no results, return None 39 | if len(results) == 0: 40 | shutil.rmtree(tmpdir) 41 | 42 | return results 43 | -------------------------------------------------------------------------------- /watchme/watchers/urls/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Vanessa Sochat" 2 | __copyright__ = "Copyright 2020-2021, Vanessa Sochat" 3 | __license__ = "MPL 2.0" 4 | 5 | from watchme.tasks import TaskBase 6 | from watchme.logger import bot 7 | 8 | 9 | class Task(TaskBase): 10 | 11 | required_params = ["url"] 12 | 13 | def __init__(self, name, params=None, **kwargs): 14 | 15 | if params is None: 16 | params = {} 17 | 18 | self.type = "urls" 19 | 20 | # Handles setting the name, setting params, and validate 21 | super(Task, self).__init__(name, params, **kwargs) 22 | 23 | def _validate(self): 24 | """additional validation function, called by validate() of 25 | superclass. Here we assume all required self.params are included. 26 | If an parameter is found to be invalid, self.valid should be set 27 | to False 28 | """ 29 | # The url must begin with http 30 | if not self.params["url"].startswith("http"): 31 | bot.error("%s is not a valid url." % self.params["url"]) 32 | self.valid = False 33 | 34 | def export_func(self): 35 | """this function should return the correct task (from the tasks.py 36 | in the same folder) based on some logic of the params that are given 37 | by the user (self.params). If there is only one kind of function for 38 | the task, it's fairly easy to import and return it here. This 39 | function should take no arguments, but instead use the self.params 40 | already provided in the client. 41 | """ 42 | name = self.params.get("func", "get_task") 43 | 44 | if name == "get_task": 45 | from .tasks import get_task as func 46 | elif name == "download_task": 47 | from .tasks import download_task as func 48 | elif name == "post_task": 49 | from .tasks import post_task as func 50 | elif name == "get_url_selection": 51 | from .tasks import get_url_selection as func 52 | else: 53 | func = None 54 | 55 | bot.debug("function name is %s" % name) 56 | return func 57 | --------------------------------------------------------------------------------