├── .github └── workflows │ └── unittest.yml ├── .gitignore ├── Jenkinsfile ├── LICENSE ├── README.md ├── bin └── optcomplete.bash ├── doc ├── Makefile ├── README ├── baseconf.py ├── conf.py └── index.rst ├── examples ├── qa.py └── simple_option.py ├── lib └── vsc │ ├── __init__.py │ └── utils │ ├── __init__.py │ ├── affinity.py │ ├── asyncprocess.py │ ├── daemon.py │ ├── dateandtime.py │ ├── docs.py │ ├── exceptions.py │ ├── fancylogger.py │ ├── frozendict.py │ ├── generaloption.py │ ├── groups.py │ ├── mail.py │ ├── missing.py │ ├── optcomplete.py │ ├── patterns.py │ ├── rest.py │ ├── run.py │ ├── testing.py │ └── wrapper.py ├── setup.py ├── test ├── 00-import.py ├── __init__.py ├── asyncprocess.py ├── data │ └── mailconfig.ini ├── dateandtime.py ├── docs.py ├── exceptions.py ├── fancylogger.py ├── generaloption.py ├── groups.py ├── mail.py ├── missing.py ├── optcomplete.py ├── rest.py ├── run.py ├── runtests │ ├── qa.py │ ├── run_nested.sh │ ├── simple.py │ └── simple_option.py ├── sandbox │ └── testpkg │ │ ├── __init__.py │ │ ├── testmodule.py │ │ └── testmodulebis.py └── wrapper.py ├── tox.ini └── vsc-ci.ini /.github/workflows/unittest.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/unittest.yml: configuration file for github actions worflow 2 | # This file was automatically generated using 'python -m vsc.install.ci' 3 | # DO NOT EDIT MANUALLY 4 | jobs: 5 | python_unittests: 6 | runs-on: ubuntu-20.04 7 | steps: 8 | - name: Checkout code 9 | uses: actions/checkout@v4 10 | - name: Setup Python 11 | uses: actions/setup-python@v5 12 | with: 13 | python-version: ${{ matrix.python }} 14 | - name: install tox 15 | run: pip install 'virtualenv<20.22.0' 'tox<4.5.0' 16 | - name: add mandatory git remote 17 | run: git remote add hpcugent https://github.com/hpcugent/vsc-base.git 18 | - name: Run tox 19 | run: tox -e py$(echo ${{ matrix.python }} | sed 's/\.//g') 20 | strategy: 21 | matrix: 22 | python: 23 | - 3.6 24 | - 3.9 25 | name: run python tests 26 | 'on': 27 | - push 28 | - pull_request 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.swp 3 | *~ 4 | 5 | # ok to ignore generated file in vsc-base 6 | setup.cfg 7 | 8 | # Packages 9 | .eggs* 10 | *.egg 11 | *.egg-info 12 | dist 13 | build 14 | eggs 15 | parts 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | 28 | #Translations 29 | *.mo 30 | 31 | #Mr Developer 32 | .mr.developer.cfg 33 | 34 | # Ninja 35 | *.nja 36 | 37 | html 38 | test-reports 39 | htmlcov/ 40 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | // Jenkinsfile: scripted Jenkins pipefile 2 | // This file was automatically generated using 'python -m vsc.install.ci' 3 | // DO NOT EDIT MANUALLY 4 | 5 | node { 6 | stage('checkout git') { 7 | checkout scm 8 | // remove untracked files (*.pyc for example) 9 | sh 'git clean -fxd' 10 | } 11 | stage('test') { 12 | sh 'pip3 install --ignore-installed --prefix $PWD/.vsc-tox tox' 13 | sh 'export PATH=$PWD/.vsc-tox/bin:$PATH && export PYTHONPATH=$PWD/.vsc-tox/lib/python$(python3 -c "import sys; print(\\"%s.%s\\" % sys.version_info[:2])")/site-packages:$PYTHONPATH && tox -v -c tox.ini' 14 | sh 'rm -r $PWD/.vsc-tox' 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vsc-base 2 | 3 | # Description 4 | 5 | Common tools used within our organization. 6 | Originally created by the HPC team of Ghent University (https://ugent.be/hpc). 7 | 8 | # Namespaces and tools 9 | 10 | ## lib/utils 11 | python utilities to be used as libraries 12 | 13 | - __fancylogger__: an extention of the default python logger designed to be easy to use and have a couple of `fancy` features. 14 | 15 | - custom specifiers for mpi loggin (the mpirank) with autodetection of mpi 16 | - custom specifier for always showing the calling function's name 17 | - rotating file handler 18 | - a default formatter. 19 | - logging to an UDP server (logdaemon.py f.ex.) 20 | - easily setting loglevel 21 | 22 | - __daemon.py__ : Daemon class written by Sander Marechal (http://www.jejik.com) to start a python script as a daemon. 23 | - __missing.py__: Small functions and tools that are commonly used but not available in the Python (2.x) API. 24 | - ~~__cache.py__ : File cache to store pickled data identified by a key accompanied by a timestamp.~~ (moved to [vsc-utils](https://github.com/hpcugent/vsc-utils)) 25 | - __generaloption.py__ : A general option parser for python. It will fetch options (in this order) from config files, from environment variables and from the command line and parse them in a way compatible with the default python optionparser. Thus allowing a very flexible way to configure your scripts. It also adds a few other useful extras. 26 | - __affinity.py__ : Linux cpu affinity. 27 | 28 | - Based on `sched.h` and `bits/sched.h`, 29 | - see man pages for `sched_getaffinity` and `sched_setaffinity` 30 | - also provides a `cpuset` class to convert between human readable cpusets and the bit version Linux priority 31 | - Based on sys/resources.h and bits/resources.h see man pages for `getpriority` and `setpriority` 32 | 33 | - __asyncprocess.py__ : Module to allow Asynchronous subprocess use on Windows and Posix platforms 34 | 35 | - Based on a [python recipe](http://code.activestate.com/recipes/440554/) by Josiah Carlson 36 | - added STDOUT handle and recv_some 37 | 38 | - __daemon.py__ : [A generic daemon class by Sander Marechal](http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/) 39 | - __dateandtime.py__ : A module with various convenience functions and classes to deal with date, time and timezone. 40 | - __nagios.py__ : This module provides functionality to cache and report results of script executions that can readily be interpreted by nagios/icinga. 41 | - __run.py__ : Python module to execute a command, can make use of asyncprocess, answer questions based on a dictionary 42 | 43 | - supports a whole lot of ways to input, process and output the command. (filehandles, PIPE, pty, stdout, logging...) 44 | 45 | - __mail.py__ : Wrapper around the standard Python mail library. 46 | 47 | - Send a plain text message 48 | - Send an HTML message, with a plain text alternative 49 | 50 | # Acknowledgements 51 | vsc-base was created with support of [Ghent University](https://www.ugent.be/en), 52 | the [Flemish Supercomputer Centre (VSC)](https://vscentrum.be/nl/en), 53 | the [Flemish Research Foundation (FWO)](https://www.fwo.be/en), 54 | and [the Department of Economy, Science and Innovation (EWI)](https://www.ewi-vlaanderen.be/en). 55 | 56 | -------------------------------------------------------------------------------- /bin/optcomplete.bash: -------------------------------------------------------------------------------- 1 | #******************************************************************************\ 2 | # * Copyright (c) 2003-2004, Martin Blais 3 | # * All rights reserved. 4 | # * 5 | # * Redistribution and use in source and binary forms, with or without 6 | # * modification, are permitted provided that the following conditions are 7 | # * met: 8 | # * 9 | # * * Redistributions of source code must retain the above copyright 10 | # * notice, this list of conditions and the following disclaimer. 11 | # * 12 | # * * Redistributions in binary form must reproduce the above copyright 13 | # * notice, this list of conditions and the following disclaimer in the 14 | # * documentation and/or other materials provided with the distribution. 15 | # * 16 | # * * Neither the name of the Martin Blais, Furius, nor the names of its 17 | # * contributors may be used to endorse or promote products derived from 18 | # * this software without specific prior written permission. 19 | # * 20 | # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | #******************************************************************************\ 32 | # 33 | # stdweird: This is a copy of etc/optcomplete.bash (changeset 17:e0a9131a94cc) 34 | # stdweird: source: https://hg.furius.ca/public/optcomplete 35 | # 36 | # optcomplete harness for bash shell. You then need to tell 37 | # bash to invoke this shell function with a command like 38 | # this: 39 | # 40 | # complete -F _optcomplete 41 | # 42 | 43 | _optcomplete() 44 | { 45 | # needed to let it return _filedir based commands 46 | local cur prev quoted 47 | _get_comp_words_by_ref cur prev 48 | _quote_readline_by_ref "$cur" quoted 49 | _expand || return 0 50 | 51 | # call the command with the completion information, then eval it's results so it can call _filedir or similar 52 | # this does have the potential to be a security problem, especially if running as root, but we should trust 53 | # the completions as we're planning to run this script anyway 54 | eval $( \ 55 | COMP_LINE=$COMP_LINE COMP_POINT=$COMP_POINT \ 56 | COMP_WORDS="${COMP_WORDS[*]}" COMP_CWORD=$COMP_CWORD \ 57 | OPTPARSE_AUTO_COMPLETE=1 $1 58 | ) 59 | } 60 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | CUR_DIR = $(PWD) 10 | 11 | # Internal variables. 12 | PAPEROPT_a4 = -D latex_paper_size=a4 13 | PAPEROPT_letter = -D latex_paper_size=letter 14 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 15 | # the i18n builder cannot share the environment and doctrees with the others 16 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 17 | 18 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 19 | 20 | help: 21 | @echo "Please use \`make ' where is one of" 22 | @echo " html to make standalone HTML files" 23 | @echo " api to make .rst files for a recursive python module api" 24 | @echo " dirhtml to make HTML files named index.html in directories" 25 | @echo " singlehtml to make a single large HTML file" 26 | @echo " pickle to make pickle files" 27 | @echo " json to make JSON files" 28 | @echo " htmlhelp to make HTML files and a HTML help project" 29 | @echo " qthelp to make HTML files and a qthelp project" 30 | @echo " devhelp to make HTML files and a Devhelp project" 31 | @echo " epub to make an epub" 32 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 33 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 34 | @echo " text to make text files" 35 | @echo " man to make manual pages" 36 | @echo " texinfo to make Texinfo files" 37 | @echo " info to make Texinfo files and run them through makeinfo" 38 | @echo " gettext to make PO message catalogs" 39 | @echo " changes to make an overview of all changed/added/deprecated items" 40 | @echo " linkcheck to check all external links for integrity" 41 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 42 | 43 | clean: 44 | -rm -rf $(BUILDDIR)/* 45 | 46 | api: 47 | @bash -c 'for d in ../../vsc-*/lib/vsc; do cd $$d && sphinx-apidoc -f -d 8 -T -o $(CUR_DIR) . && cd - ; done' 48 | rm vsc.rst # this will not cointain enought information, we use the glob pattern in index.rst instead of this one 49 | 50 | html: 51 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 52 | @echo 53 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 54 | 55 | dirhtml: 56 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 57 | @echo 58 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 59 | 60 | singlehtml: 61 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 62 | @echo 63 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 64 | 65 | pickle: 66 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 67 | @echo 68 | @echo "Build finished; now you can process the pickle files." 69 | 70 | json: 71 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 72 | @echo 73 | @echo "Build finished; now you can process the JSON files." 74 | 75 | htmlhelp: 76 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 77 | @echo 78 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 79 | ".hhp project file in $(BUILDDIR)/htmlhelp." 80 | 81 | qthelp: 82 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 83 | @echo 84 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 85 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 86 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/vsc-base.qhcp" 87 | @echo "To view the help file:" 88 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/vsc-base.qhc" 89 | 90 | devhelp: 91 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 92 | @echo 93 | @echo "Build finished." 94 | @echo "To view the help file:" 95 | @echo "# mkdir -p $$HOME/.local/share/devhelp/vsc-base" 96 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/vsc-base" 97 | @echo "# devhelp" 98 | 99 | epub: 100 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 101 | @echo 102 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 103 | 104 | latex: 105 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 106 | @echo 107 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 108 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 109 | "(use \`make latexpdf' here to do that automatically)." 110 | 111 | latexpdf: 112 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 113 | @echo "Running LaTeX files through pdflatex..." 114 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 115 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 116 | 117 | text: 118 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 119 | @echo 120 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 121 | 122 | man: 123 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 124 | @echo 125 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 126 | 127 | texinfo: 128 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 129 | @echo 130 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 131 | @echo "Run \`make' in that directory to run these through makeinfo" \ 132 | "(use \`make info' here to do that automatically)." 133 | 134 | info: 135 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 136 | @echo "Running Texinfo files through makeinfo..." 137 | make -C $(BUILDDIR)/texinfo info 138 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 139 | 140 | gettext: 141 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 142 | @echo 143 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 144 | 145 | changes: 146 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 147 | @echo 148 | @echo "The overview file is in $(BUILDDIR)/changes." 149 | 150 | linkcheck: 151 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 152 | @echo 153 | @echo "Link check complete; look for any errors in the above output " \ 154 | "or in $(BUILDDIR)/linkcheck/output.txt." 155 | 156 | doctest: 157 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 158 | @echo "Testing of doctests in the sources finished, look at the " \ 159 | "results in $(BUILDDIR)/doctest/output.txt." 160 | -------------------------------------------------------------------------------- /doc/README: -------------------------------------------------------------------------------- 1 | This directory contains the documentation for vsc-base 2 | 3 | It should be created with sphinx. 4 | 5 | The baseconf.py file contains base configuration for vsc-base and all other vsc-* packages. 6 | This can be extended, as seen in conf.py 7 | 8 | 9 | run make api to generate the .rst files for all vsc- packages 10 | run make html to create the documentation from these .rst packages 11 | -------------------------------------------------------------------------------- /doc/baseconf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # vsc-base documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Apr 29 17:56:48 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['_templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = 'index' 41 | 42 | # General information about the project. 43 | project = u'vsc-base' 44 | copyright = u'2013, Ghent University' 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = '1.0.4' 52 | # The full version, including alpha/beta/rc tags. 53 | release = '1.0.4' 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | #language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | #today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | #today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = ['_build'] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = 'default' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = 'vsc-basedoc' 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | latex_elements = { 173 | # The paper size ('letterpaper' or 'a4paper'). 174 | #'papersize': 'letterpaper', 175 | 176 | # The font size ('10pt', '11pt' or '12pt'). 177 | #'pointsize': '10pt', 178 | 179 | # Additional stuff for the LaTeX preamble. 180 | #'preamble': '', 181 | } 182 | 183 | # Grouping the document tree into LaTeX files. List of tuples 184 | # (source start file, target name, title, author, documentclass [howto/manual]). 185 | latex_documents = [ 186 | ('index', 'vsc-base.tex', u'vsc-base Documentation', 187 | u'Ghent University', 'manual'), 188 | ] 189 | 190 | # The name of an image file (relative to this directory) to place at the top of 191 | # the title page. 192 | #latex_logo = None 193 | 194 | # For "manual" documents, if this is true, then toplevel headings are parts, 195 | # not chapters. 196 | #latex_use_parts = False 197 | 198 | # If true, show page references after internal links. 199 | #latex_show_pagerefs = False 200 | 201 | # If true, show URL addresses after external links. 202 | #latex_show_urls = False 203 | 204 | # Documents to append as an appendix to all manuals. 205 | #latex_appendices = [] 206 | 207 | # If false, no module index is generated. 208 | #latex_domain_indices = True 209 | 210 | 211 | # -- Options for manual page output -------------------------------------------- 212 | 213 | # One entry per manual page. List of tuples 214 | # (source start file, name, description, authors, manual section). 215 | man_pages = [ 216 | ('index', 'vsc-base', u'vsc-base Documentation', 217 | [u'Ghent University'], 1) 218 | ] 219 | 220 | # If true, show URL addresses after external links. 221 | #man_show_urls = False 222 | 223 | 224 | # -- Options for Texinfo output ------------------------------------------------ 225 | 226 | # Grouping the document tree into Texinfo files. List of tuples 227 | # (source start file, target name, title, author, 228 | # dir menu entry, description, category) 229 | texinfo_documents = [ 230 | ('index', 'vsc-base', u'vsc-base Documentation', 231 | u'Ghent University', 'vsc-base', 'One line description of project.', 232 | 'Miscellaneous'), 233 | ] 234 | 235 | # Documents to append as an appendix to all manuals. 236 | #texinfo_appendices = [] 237 | 238 | # If false, no module index is generated. 239 | #texinfo_domain_indices = True 240 | 241 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 242 | #texinfo_show_urls = 'footnote' 243 | 244 | 245 | # Example configuration for intersphinx: refer to the Python standard library. 246 | intersphinx_mapping = {'http://docs.python.org/': None} 247 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('.') 3 | from baseconf import * 4 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. vsc-base documentation master file, created by 2 | sphinx-quickstart on Mon Apr 29 17:56:48 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to vsc-base's documentation! 7 | ==================================== 8 | In this document we will try to explain how to use some of the vsc-base tools. 9 | 10 | Indices and tables 11 | ================== 12 | 13 | * :ref:`genindex` 14 | * :ref:`modindex` 15 | * :ref:`search` 16 | 17 | 18 | Option parsing 19 | -------------- 20 | A lot of python scripts start of with parsing options. :py:mod:`vsc.utils.generaloption` provides a very general way of getting options. 21 | It provides the :py:func:`vsc.utils.generaloption.simple_option` function, which lets you easily parse options from: 22 | * A config file 23 | * Environment variables 24 | * The command line 25 | 26 | This is the simplest way to use generaloption, however, you can easily extend the :py:class:`vsc.utils.generaloption.GeneralOption` class to 27 | provide a lot more fine graned options. 28 | 29 | Example usage 30 | ^^^^^^^^^^^^^ 31 | You can use the :py:func:`vsc.utils.generaloption.simple_option` function to easily get a general option parser. 32 | a general options dict has as key the long option name, and is followed by a list/tuple mandatory are 4 elements: 33 | option help, type, action, default a 5th element is optional and is the short help name (if any):: 34 | 35 | """This is an example on how to use simple_option""" 36 | from vsc.utils.generaloption import simple_option 37 | opts = { 38 | "path": ("Specify the path to look for profiles.", None, "store",'defpath', "p"), 39 | "host": ("Specify a specific hostname (will guess the profile filename).", None, "store", None, "O"), 40 | "pxepath": ("Specify a path for the pxe base dir.", None, "store", 'pxepath', "P"), 41 | "vsmpversion": ("Specify a vsmp version.", None, "store", 'version', "V"), 42 | "imagerbase": ("Specify a path to imager subdirs (imager to be found in this path + /vSMP_version).", None, "store", 'imagerbase', "i"), 43 | "boot": ("Boot from pxe, hdd0", None, "store", 'boot', "b"), 44 | 'licmgr': ("License host:port", None, "store", "%s:%s" % ('lic_host', 'lic_port'), "L") 45 | } 46 | parser = simple_option(opts, config_files=["/etc/myconfig.cfg", "%s/.myconfig.cfg" % os.path.expanduser("~")]) 47 | options = parser.options 48 | parser.log.debug(options.path) 49 | 50 | running this script with the -H option will give:: 51 | Usage: test.py [options] 52 | 53 | 54 | This is an example on how to use simple_option 55 | 56 | Options: 57 | -h, --shorthelp show short help message and exit 58 | -H, --help show full help message and exit 59 | 60 | Main options (configfile section MAIN): 61 | -b BOOT, --boot=BOOT 62 | Boot from pxe, hdd0 (def boot) 63 | -O HOST, --host=HOST 64 | Specify a specific hostname (will guess the profile filename). 65 | -i IMAGERBASE, --imagerbase=IMAGERBASE 66 | Specify a path to imager subdirs (imager to be found in this path + /vSMP_version). (def imagerbase) 67 | -L LICMGR, --licmgr=LICMGR 68 | License host:port (def lic_host:lic_port) 69 | -p PATH, --path=PATH 70 | Specify the path to look for profiles. (def defpath) 71 | -P PXEPATH, --pxepath=PXEPATH 72 | Specify a path for the pxe base dir. (def pxepath) 73 | -V VSMPVERSION, --vsmpversion=VSMPVERSION 74 | Specify a vsmp version. (def version) 75 | 76 | Debug and logging options (configfile section MAIN): 77 | -d, --debug Enable debug log mode (def False) 78 | --info Enable info log mode (def False) 79 | --quiet Enable info quiet/warning mode (def False) 80 | 81 | Configfile options (configfile section MAIN): 82 | --configfiles=CONFIGFILES 83 | Parse (additional) configfiles (type comma-separated list) 84 | --ignoreconfigfiles=IGNORECONFIGFILES 85 | Ignore configfiles (type comma-separated list) 86 | 87 | Boolean options support disable prefix to do the inverse of the action, e.g. option --someopt also supports --disable-someopt. 88 | 89 | All long option names can be passed as environment variables. Variable name is SCRIPT_ eg. --someopt is same as setting SCRIPT_SOMEOPT in the environment. 90 | 91 | 92 | GeneralOption will now use a configfile (in /etc/myconfig.cfg or ~/.myconfig.cfg) to look up options. It uses pythons' :py:class:`ConfigParser` class to parse the config files. 93 | If multiple configfiles can be found, the last one will be used. 94 | The options in the config file can be overwritten when you set an environment variable (e.g., SCRIPT_DEBUG). 95 | You can then furhter overwrite these options on the command line. 96 | 97 | You will automatically have a short and long help, get the docstring in the help and get a :py:class:`vsc.utils.fancylogger.FancyLogger` logger and logging options. 98 | (try running script.py -d) 99 | 100 | Advanced option parsing 101 | ^^^^^^^^^^^^^^^^^^^^^^^ 102 | You can easily extend GeneralOption and GeneralOptionParser to create very complicated option parsers, wich perform postprocessing sanity checks, have multiple optiongroups, 103 | print out extended help, set custom environment variable prefixes etc... 104 | For a real world example, see the EasyBuildOptions in `easybuild/tools/options.py `_ 105 | -------------------------------------------------------------------------------- /examples/qa.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2013-2013 Ghent University 4 | # 5 | # This file is part of vsc-base, 6 | # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), 7 | # with support of Ghent University (http://ugent.be/hpc), 8 | # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), 9 | # the Flemish Research Foundation (FWO) (http://www.fwo.be/en) 10 | # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). 11 | # 12 | # http://github.com/hpcugent/vsc-base 13 | # 14 | # vsc-base is free software: you can redistribute it and/or modify 15 | # it under the terms of the GNU Library General Public License as 16 | # published by the Free Software Foundation, either version 2 of 17 | # the License, or (at your option) any later version. 18 | # 19 | # vsc-base is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | # GNU Library General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Library General Public License 25 | # along with vsc-base. If not, see . 26 | # 27 | """ 28 | Simple QA script 29 | 30 | @author: Stijn De Weirdt (Ghent University) 31 | """ 32 | import os 33 | 34 | from vsc.utils.run import run_qa, run_qalog, run_qastdout, run_async_to_stdout 35 | from vsc.utils.generaloption import simple_option 36 | 37 | go = simple_option(None) 38 | 39 | SCRIPT_DIR = os.path.join(os.path.dirname(__file__), '..', 'test', 'runtests') 40 | SCRIPT_QA = os.path.join(SCRIPT_DIR, 'qa.py') 41 | 42 | 43 | def test_qa(): 44 | qa_dict = { 45 | 'Simple question:': 'simple answer', 46 | } 47 | ec, output = run_qa([SCRIPT_QA, 'simple'], qa=qa_dict) 48 | return ec, output 49 | 50 | 51 | def test_qalog(): 52 | qa_dict = { 53 | 'Simple question:': 'simple answer', 54 | } 55 | ec, output = run_qalog([SCRIPT_QA, 'simple'], qa=qa_dict) 56 | return ec, output 57 | 58 | 59 | def test_qastdout(): 60 | run_async_to_stdout([SCRIPT_QA, 'simple']) 61 | qa_dict = { 62 | 'Simple question:': 'simple answer', 63 | } 64 | ec, output = run_qastdout([SCRIPT_QA, 'simple'], qa=qa_dict) 65 | return ec, output 66 | 67 | 68 | def test_std_regex(): 69 | qa_dict = { 70 | r'\s(?P