├── .coveragerc ├── .github └── workflows │ ├── build.yml │ └── deploy-pypi.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── Makefile ├── README.md ├── clinto ├── __init__.py ├── ast │ ├── __init__.py │ └── source_parser.py ├── parser.py ├── parsers │ ├── __init__.py │ ├── argparse_.py │ ├── base.py │ ├── compat.py │ ├── constants.py │ └── docopt_.py ├── tests │ ├── __init__.py │ ├── argparse_scripts │ │ ├── choices.py │ │ ├── data_reader.zip │ │ ├── error_script.py │ │ ├── function_argtype.py │ │ ├── mutually_exclusive.py │ │ ├── subparser_script.py │ │ └── zip_app_rel_imports.zip │ ├── docopt_scripts │ │ └── naval_fate.py │ ├── factories.py │ └── test_parsers.py ├── utils.py └── version.py ├── requirements.txt ├── setup.cfg └── setup.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = clinto/tests* 4 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build-and-Test 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | max-parallel: 4 11 | matrix: 12 | # Up to date compatibility matrix 13 | python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] 14 | os: [ubuntu-latest, windows-latest] 15 | 16 | steps: 17 | - uses: actions/checkout@v1 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v1 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | make testenv 26 | - name: Run Tests 27 | run: | 28 | make test 29 | 30 | - uses: codecov/codecov-action@v3 31 | with: 32 | file: ./coverage.xml 33 | flags: unittests 34 | name: codecov-umbrella 35 | fail_ci_if_error: true 36 | -------------------------------------------------------------------------------- /.github/workflows/deploy-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Deploy-To-Pypi 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ${{ matrix.os }} 12 | strategy: 13 | max-parallel: 4 14 | matrix: 15 | python-version: [3.7] 16 | os: [ubuntu-latest] 17 | 18 | steps: 19 | - uses: actions/checkout@v1 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v1 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Setup Packaging Tools 25 | run: | 26 | pip install -U setuptools twine pip 27 | - name: Upload to pypi 28 | env: 29 | PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} 30 | run: | 31 | python setup.py sdist 32 | twine upload -u __token__ -p $PYPI_API_TOKEN dist/* 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # IDE 60 | .idea 61 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 22.10.0 4 | hooks: 5 | - id: black 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Clinto 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of Clinto nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | testenv: 2 | pip install -r requirements.txt 3 | pip install -e . 4 | 5 | test: 6 | pytest --cov=clinto --cov-config=.coveragerc clinto/tests/* 7 | coverage report --omit='clinto/tests*' 8 | coverage xml --omit='clinto/tests*' 9 | 10 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 11 | 12 | clean-build: ## remove build artifacts 13 | rm -fr build/ 14 | rm -fr dist/ 15 | rm -fr .eggs/ 16 | find . -name '*.egg-info' -exec rm -fr {} + 17 | find . -name '*.egg' -exec rm -f {} + 18 | 19 | clean-pyc: ## remove Python file artifacts 20 | find . -name '*.pyc' -exec rm -f {} + 21 | find . -name '*.pyo' -exec rm -f {} + 22 | find . -name '*~' -exec rm -f {} + 23 | find . -name '__pycache__' -exec rm -fr {} + 24 | 25 | clean-test: ## remove test and coverage artifacts 26 | rm -fr .tox/ 27 | rm -f .coverage 28 | rm -fr htmlcov/ 29 | rm -fr .pytest_cache 30 | 31 | release/major release/minor release/patch release/rc: 32 | bumpversion $(@F) 33 | git push 34 | git push --tags 35 | 36 | dist: clean ## builds source and wheel package 37 | python setup.py sdist 38 | python setup.py bdist_wheel 39 | ls -l dist 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # clinto 2 | [![Build Status](https://github.com/wooey/clinto/workflows/Build-and-Test/badge.svg)](https://github.com/wooey/clinto/actions?query=workflow%3ABuild-and-Test) 3 | [![Deploy-To-Pypi](https://github.com/wooey/clinto/workflows/Deploy-To-Pypi/badge.svg)](https://github.com/wooey/clinto/actions?query=workflow%3ADeploy-To-Pypi) 4 | [![codecov](https://codecov.io/gh/wooey/clinto/branch/master/graph/badge.svg)](https://codecov.io/gh/wooey/clinto) 5 | 6 | [![Join the chat at https://gitter.im/wooey/clinto](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/wooey/clinto?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 7 | 8 | This converts an assortment of python command line interfaces into a language agnostic build spec for usage in GUI creation. 9 | 10 | Here's a basic usage: 11 | ``` 12 | from clinto import parser 13 | specs = parser.Parser(script_path='/home/chris/Devel/pythomics/pythomics/scripts/proteinInference.py', script_name='Protein Inference') 14 | specs.get_script_description() 15 | { 16 | 'name': 'Protein Inference Script', 17 | 'path': '/home/chris/Devel/pythomics/pythomics/scripts/proteinInference.py' 18 | 'description': '\nThis script will annotate a tab delimited text file with peptides with\ncorresponding proteins present in an annotation file, and can also\nuse this annotation to include iBAQ measures.\n', 19 | 'inputs': { 20 | 'parser_name': [{ 21 | 'group': 'optional arguments', 22 | 'nodes': [{ 23 | 'choice_limit': None, 24 | 'choices': None, 25 | 'help': 'Threads to run', 26 | 'model': 'IntegerField', 27 | 'name': 'p', 28 | 'param': '-p', 29 | 'required': False, 30 | 'type': 'text', 31 | 'value': 1 32 | }, { 33 | 'choice_limit': '1', 34 | 'choices': None, 35 | 'help': 'The fasta file to match peptides against.', 36 | 'model': 'FileField', 37 | 'name': 'fasta', 38 | 'param': '-f', 39 | 'required': False, 40 | 'type': 'file', 41 | 'upload': True 42 | }], 43 | }, 44 | 'group': 'Protein Grouping Options', 45 | 'nodes': [{ 46 | 'checked': False, 47 | 'choice_limit': 0, 48 | 'choices': None, 49 | 'help': 'Only group proteins with unique peptides', 50 | 'model': 'BooleanField', 51 | 'name': 'unique_only', 52 | 'param': '--unique-only', 53 | 'required': False, 54 | 'type': 'checkbox' 55 | }, { 56 | 'checked': False, 57 | 'choice_limit': 0, 58 | 'choices': None, 59 | 'help': 'Write the position of the peptide matches.', 60 | 'model': 'BooleanField', 61 | 'name': 'position', 62 | 'param': '--position', 63 | 'required': False, 64 | 'type': 'checkbox' 65 | }], 66 | ] 67 | }, 68 | } 69 | ``` 70 | -------------------------------------------------------------------------------- /clinto/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "chris" 2 | -------------------------------------------------------------------------------- /clinto/ast/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'chris' 2 | -------------------------------------------------------------------------------- /clinto/ast/source_parser.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Dec 11, 2013 3 | 4 | @author: Chris 5 | 6 | Collection of functions for extracting argparse related statements from the 7 | client code. 8 | """ 9 | 10 | import ast 11 | import _ast 12 | from itertools import * 13 | 14 | import astor 15 | 16 | 17 | def find_valid_imports(imports): 18 | valid_imports = [] 19 | for import_module in imports: 20 | try: 21 | exec(astor.to_source(import_module)) 22 | except (ImportError, ModuleNotFoundError): 23 | pass 24 | else: 25 | valid_imports.append(import_module) 26 | return valid_imports 27 | 28 | 29 | def parse_source_file(file_name, ignore_bad_imports=False): 30 | """ 31 | Parses the AST of Python file for lines containing 32 | references to the argparse module. 33 | 34 | returns the collection of ast objects found. 35 | 36 | Example client code: 37 | 38 | 1. parser = ArgumentParser(desc="My help Message") 39 | 2. parser.add_argument('filename', help="Name of the file to load") 40 | 3. parser.add_argument('-f', '--format', help='Format of output \nOptions: ['md', 'html'] 41 | 4. args = parser.parse_args() 42 | 43 | Variables: 44 | * nodes Primary syntax tree object 45 | * argparse_assignments The assignment of the ArgumentParser (line 1 in example code) 46 | * add_arg_assignments Calls to add_argument() (lines 2-3 in example code) 47 | * parser_var_name The instance variable of the ArgumentParser (line 1 in example code) 48 | * ast_source The curated collection of all parser related nodes in the client code 49 | """ 50 | with open(file_name, "r") as f: 51 | s = f.read() 52 | 53 | nodes = ast.parse(s) 54 | 55 | module_imports = get_nodes_by_instance_type(nodes, _ast.Import) 56 | specific_imports = get_nodes_by_instance_type(nodes, _ast.ImportFrom) 57 | 58 | if ignore_bad_imports: 59 | module_imports = find_valid_imports(module_imports) 60 | specific_imports = find_valid_imports(specific_imports) 61 | 62 | assignment_objs = get_nodes_by_instance_type(nodes, _ast.Assign) 63 | call_objects = get_nodes_by_instance_type(nodes, _ast.Call) 64 | 65 | argparse_assignments = get_nodes_by_containing_attr( 66 | assignment_objs, "ArgumentParser" 67 | ) 68 | group_arg_assignments = get_nodes_by_containing_attr( 69 | assignment_objs, "add_argument_group" 70 | ) 71 | add_arg_assignments = get_nodes_by_containing_attr(call_objects, "add_argument") 72 | parse_args_assignment = get_nodes_by_containing_attr(call_objects, "parse_args") 73 | # there are cases where we have custom argparsers, such as subclassing ArgumentParser. The above 74 | # will fail on this. However, we can use the methods known to ArgumentParser to do a duck-type like 75 | # approach to finding what is the arg parser 76 | if not argparse_assignments: 77 | aa_references = set( 78 | [i.func.value.id for i in chain(add_arg_assignments, parse_args_assignment)] 79 | ) 80 | argparse_like_objects = [ 81 | getattr(i.value.func, "id", None) 82 | for p_ref in aa_references 83 | for i in get_nodes_by_containing_attr(assignment_objs, p_ref) 84 | ] 85 | argparse_like_objects = filter(None, argparse_like_objects) 86 | argparse_assignments = [ 87 | get_nodes_by_containing_attr(assignment_objs, i) 88 | for i in argparse_like_objects 89 | ] 90 | # for now, we just choose one 91 | try: 92 | argparse_assignments = argparse_assignments[0] 93 | except IndexError: 94 | pass 95 | 96 | # get things that are assigned inside ArgumentParser or its methods 97 | argparse_assigned_variables = get_node_args_and_keywords( 98 | assignment_objs, argparse_assignments, "ArgumentParser" 99 | ) 100 | add_arg_assigned_variables = get_node_args_and_keywords( 101 | assignment_objs, add_arg_assignments, "add_argument" 102 | ) 103 | parse_args_assigned_variables = get_node_args_and_keywords( 104 | assignment_objs, parse_args_assignment, "parse_args" 105 | ) 106 | 107 | ast_argparse_source = chain( 108 | module_imports, 109 | specific_imports, 110 | argparse_assigned_variables, 111 | add_arg_assigned_variables, 112 | parse_args_assigned_variables, 113 | argparse_assignments, 114 | group_arg_assignments, 115 | add_arg_assignments, 116 | ) 117 | return ast_argparse_source 118 | 119 | 120 | def read_client_module(filename): 121 | with open(filename, "r") as f: 122 | return f.readlines() 123 | 124 | 125 | def get_node_args_and_keywords(assigned_objs, assignments, selector=None): 126 | referenced_nodes = set([]) 127 | selector_line = -1 128 | assignment_nodes = [] 129 | for node in assignments: 130 | for i in walk_tree(node): 131 | if i and isinstance(i, (_ast.keyword, _ast.Name)) and "id" in i.__dict__: 132 | if i.id == selector: 133 | selector_line = i.lineno 134 | elif i.lineno == selector_line: 135 | referenced_nodes.add(i.id) 136 | for node in assigned_objs: 137 | for target in node.targets: 138 | if getattr(target, "id", None) in referenced_nodes: 139 | assignment_nodes.append(node) 140 | return assignment_nodes 141 | 142 | 143 | def get_nodes_by_instance_type(nodes, object_type): 144 | return [node for node in walk_tree(nodes) if isinstance(node, object_type)] 145 | 146 | 147 | def get_nodes_by_containing_attr(nodes, attr): 148 | return [node for node in nodes if attr in walk_tree(node)] 149 | 150 | 151 | def walk_tree(node): 152 | try: 153 | d = node.__dict__ 154 | except AttributeError: 155 | d = {} 156 | yield node 157 | for key, value in d.items(): 158 | if isinstance(value, list): 159 | for val in value: 160 | for _ in ast.walk(val): 161 | yield _ 162 | elif issubclass(type(value), ast.AST): 163 | for _ in walk_tree(value): 164 | yield _ 165 | else: 166 | yield value 167 | 168 | 169 | def convert_to_python(ast_source): 170 | """ 171 | Converts the ast objects back into human readable Python code 172 | """ 173 | return map(astor.to_source, ast_source) 174 | -------------------------------------------------------------------------------- /clinto/parser.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import zipfile 4 | 5 | from .parsers import ArgParseParser, DocOptParser 6 | 7 | parsers = [ArgParseParser, DocOptParser] 8 | 9 | 10 | class Parser(object): 11 | def __init__(self, script_path=None, script_name=None, ignore_bad_imports=False): 12 | self.parser = None 13 | self._error = "" 14 | 15 | if zipfile.is_zipfile(script_path): 16 | temp_dir = tempfile.mkdtemp() 17 | with zipfile.ZipFile(script_path) as zip: 18 | zip.extractall(temp_dir) 19 | script_path = os.path.join(temp_dir, "__main__.py") 20 | 21 | with open(script_path, "r") as f: 22 | script_source = f.read() 23 | 24 | parser_obj = [ 25 | pc( 26 | script_path=script_path, 27 | script_source=script_source, 28 | ignore_bad_imports=ignore_bad_imports, 29 | ) 30 | for pc in parsers 31 | ] 32 | parser_obj = sorted(parser_obj, key=lambda x: x.score, reverse=True) 33 | 34 | for po in parser_obj: 35 | if po.is_valid: 36 | # It worked 37 | self.parser = po 38 | break 39 | else: 40 | # No parser found, fetch the error from the highest scoring parser for reporting 41 | self._error = parser_obj[0].error 42 | 43 | def get_script_description(self): 44 | if self.parser: 45 | return self.parser.get_script_description() 46 | 47 | @property 48 | def json(self): 49 | if self.parser: 50 | return self.parser.json 51 | return {} 52 | 53 | @property 54 | def valid(self): 55 | if self.parser: 56 | return self.parser.is_valid 57 | return False 58 | 59 | @property 60 | def error(self): 61 | if self.parser: 62 | return self.parser.error 63 | return self._error 64 | -------------------------------------------------------------------------------- /clinto/parsers/__init__.py: -------------------------------------------------------------------------------- 1 | from .argparse_ import ArgParseParser 2 | from .docopt_ import DocOptParser 3 | -------------------------------------------------------------------------------- /clinto/parsers/argparse_.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import argparse 3 | import json 4 | import imp 5 | import os 6 | import sys 7 | import tempfile 8 | import traceback 9 | import types 10 | from collections import OrderedDict 11 | from itertools import chain 12 | 13 | from ..ast import source_parser 14 | from ..utils import is_upload, expand_iterable 15 | from .base import ( 16 | BaseParser, 17 | parse_args_monkeypatch, 18 | ClintoArgumentParserException, 19 | update_dict_copy, 20 | ) 21 | from .compat import ParserExceptions 22 | from .constants import SPECIFY_EVERY_PARAM 23 | 24 | 25 | # input attributes we try to set: 26 | # checked, name, type, value 27 | # extra information we want to append: 28 | # help, 29 | # required, 30 | # param (for executing the script and knowing if we need - or --), 31 | # upload (boolean providing info on whether it's a file are we uploading or saving) 32 | # choices (for selections) 33 | # choice_limit (for multi selection) 34 | 35 | CHOICE_LIMIT_MAP = {"?": "1", "+": ">=1", "*": ">=0"} 36 | 37 | # We want to map to model fields as well as html input types we encounter in argparse 38 | # keys are known variable types, as defined in __builtins__ 39 | # the model is a Django based model, which can be fitted in the future for other frameworks. 40 | # The type is the HTML input type 41 | # nullcheck is a function to determine if the default value should be checked (for cases like default='' for strings) 42 | # the attr_kwargs is a mapping of the action attributes to its related html input type. It is a dict 43 | # of the form: {'name_for_html_input': { 44 | # and either one or both of: 45 | # 'action_name': 'attribute_name_on_action', 'callback': 'function_to_evaluate_action_and_return_value'} } 46 | 47 | GLOBAL_ATTRS = ["model", "type"] 48 | 49 | 50 | def get_parameter_action(action): 51 | """ 52 | To foster a general schema that can accomodate multiple parsers, the general behavior here is described 53 | rather than the specific language of a given parser. For instance, the 'append' action of an argument 54 | is collapsing each argument given to a single argument. It also returns a set of actions as well, since 55 | presumably some actions can impact multiple parameter options 56 | """ 57 | actions = set() 58 | if isinstance(action, argparse._AppendAction): 59 | actions.add(SPECIFY_EVERY_PARAM) 60 | return actions 61 | 62 | 63 | GLOBAL_ATTR_KWARGS = { 64 | "name": {"action_name": "dest"}, 65 | "value": {"action_name": "default"}, 66 | "required": {"action_name": "required"}, 67 | "help": {"action_name": "help"}, 68 | "param": {"callback": lambda x: x.option_strings[0] if x.option_strings else ""}, 69 | "param_action": {"callback": lambda x: get_parameter_action(x)}, 70 | "choices": {"callback": lambda x: expand_iterable(x.choices)}, 71 | "choice_limit": {"callback": lambda x: CHOICE_LIMIT_MAP.get(x.nargs, x.nargs)}, 72 | } 73 | 74 | TYPE_FIELDS = { 75 | # Python Builtins 76 | bool: { 77 | "model": "BooleanField", 78 | "type": "checkbox", 79 | "nullcheck": lambda x: x.default is None, 80 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 81 | }, 82 | float: { 83 | "model": "FloatField", 84 | "type": "text", 85 | "html5-type": "number", 86 | "nullcheck": lambda x: x.default is None, 87 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 88 | }, 89 | int: { 90 | "model": "IntegerField", 91 | "type": "text", 92 | "nullcheck": lambda x: x.default is None, 93 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 94 | }, 95 | None: { 96 | "model": "CharField", 97 | "type": "text", 98 | "nullcheck": lambda x: x.default is None, 99 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 100 | }, 101 | str: { 102 | "model": "CharField", 103 | "type": "text", 104 | "nullcheck": lambda x: x.default == "" or x.default is None, 105 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 106 | }, 107 | # Argparse specific type field types 108 | argparse.FileType: { 109 | "model": "FileField", 110 | "type": "file", 111 | "nullcheck": lambda x: False, 112 | "attr_kwargs": dict( 113 | GLOBAL_ATTR_KWARGS, 114 | **{ 115 | "value": None, 116 | "required": { 117 | "callback": lambda x: x.required 118 | or x.default in (sys.stdout, sys.stdin) 119 | }, 120 | "upload": {"callback": is_upload}, 121 | } 122 | ), 123 | }, 124 | types.FunctionType: { 125 | "model": "CharField", 126 | "type": "text", 127 | "nullcheck": lambda x: x.default if callable(x.default) else x.default is None, 128 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 129 | }, 130 | } 131 | 132 | 133 | import io 134 | 135 | TYPE_FIELDS.update( 136 | { 137 | io.IOBase: { 138 | "model": "FileField", 139 | "type": "file", 140 | "nullcheck": lambda x: False, 141 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 142 | } 143 | } 144 | ) 145 | 146 | 147 | # There are cases where we can glean additional information about the form structure, e.g. 148 | # a StoreAction with default=True can be different than a StoreTrueAction with default=False 149 | ACTION_CLASS_TO_TYPE_FIELD = { 150 | argparse._StoreAction: update_dict_copy(TYPE_FIELDS, {}), 151 | argparse._StoreConstAction: update_dict_copy(TYPE_FIELDS, {}), 152 | argparse._StoreTrueAction: update_dict_copy( 153 | TYPE_FIELDS, 154 | { 155 | None: { 156 | "model": "BooleanField", 157 | "type": "checkbox", 158 | "nullcheck": lambda x: x.default is None, 159 | "attr_kwargs": update_dict_copy( 160 | GLOBAL_ATTR_KWARGS, 161 | {"checked": {"callback": lambda x: x.default}, "value": None}, 162 | ), 163 | } 164 | }, 165 | ), 166 | argparse._StoreFalseAction: update_dict_copy( 167 | TYPE_FIELDS, 168 | { 169 | None: { 170 | "model": "BooleanField", 171 | "type": "checkbox", 172 | "nullcheck": lambda x: x.default is None, 173 | "attr_kwargs": update_dict_copy( 174 | GLOBAL_ATTR_KWARGS, 175 | {"checked": {"callback": lambda x: x.default}, "value": None}, 176 | ), 177 | } 178 | }, 179 | ), 180 | } 181 | 182 | 183 | class ArgParseNode(object): 184 | """ 185 | This class takes an argument parser entry and assigns it to a Build spec 186 | """ 187 | 188 | def __init__(self, action=None, mutex_group=None): 189 | fields = ACTION_CLASS_TO_TYPE_FIELD.get(type(action), TYPE_FIELDS) 190 | field_type = fields.get(action.type) 191 | if field_type is None: 192 | field_types = [ 193 | i 194 | for i in fields.keys() 195 | if i is not None and issubclass(type(action.type), i) 196 | ] 197 | if len(field_types) > 1: 198 | field_types = [ 199 | i 200 | for i in fields.keys() 201 | if i is not None and isinstance(action.type, i) 202 | ] 203 | if len(field_types) == 1: 204 | field_type = fields[field_types[0]] 205 | if not field_types: 206 | # We cannot ascertain the type, but if it is a callable. Assign it to a charfield by default 207 | if callable(action.type): 208 | field_type = fields[types.FunctionType] 209 | self.node_attrs = dict([(i, field_type[i]) for i in GLOBAL_ATTRS]) 210 | self.node_attrs["mutex_group"] = ( 211 | {"id": mutex_group[0], "title": mutex_group[1]} if mutex_group else {} 212 | ) 213 | null_check = field_type["nullcheck"](action) 214 | for attr, attr_dict in field_type["attr_kwargs"].items(): 215 | if attr_dict is None: 216 | continue 217 | if attr == "value" and null_check: 218 | continue 219 | if "action_name" in attr_dict: 220 | self.node_attrs[attr] = getattr(action, attr_dict["action_name"], None) 221 | elif "callback" in attr_dict: 222 | self.node_attrs[attr] = attr_dict["callback"](action) 223 | 224 | @property 225 | def name(self): 226 | return self.node_attrs.get("name") 227 | 228 | def __str__(self): 229 | return json.dumps(self.node_attrs) 230 | 231 | def to_django(self): 232 | """ 233 | This is a debug function to see what equivalent django models are being generated 234 | """ 235 | exclude = {"name", "model"} 236 | field_module = "models" 237 | django_kwargs = {} 238 | if self.node_attrs["model"] == "CharField": 239 | django_kwargs["max_length"] = 255 240 | django_kwargs["blank"] = not self.node_attrs["required"] 241 | try: 242 | django_kwargs["default"] = self.node_attrs["value"] 243 | except KeyError: 244 | pass 245 | return "{0} = {1}.{2}({3})".format( 246 | self.node_attrs["name"], 247 | field_module, 248 | self.node_attrs["model"], 249 | ", ".join(["{0}={1}".format(i, v) for i, v in django_kwargs.items()]), 250 | ) 251 | 252 | 253 | class ArgParseParser(BaseParser): 254 | def heuristic(self): 255 | return [ 256 | self.script_ext in [".py", ".py3", ".py2"], 257 | "argparse" in self.script_source, 258 | "ArgumentParser" in self.script_source, 259 | ".parse_args" in self.script_source, 260 | ".add_argument" in self.script_source, 261 | ] 262 | 263 | def extract_parser(self): 264 | parsers = [] 265 | 266 | # Try exception-catching first; this should always work 267 | # Store prior to monkeypatch to restore 268 | parse_args_unmonkey = argparse.ArgumentParser.parse_args 269 | argparse.ArgumentParser.parse_args = parse_args_monkeypatch 270 | 271 | errors = {} 272 | 273 | try: 274 | exec( 275 | self.script_source, 276 | { 277 | "argparse": argparse, 278 | "__name__": "__main__", 279 | "__file__": self.script_path, 280 | }, 281 | ) 282 | except ClintoArgumentParserException as e: 283 | # Catch the generated exception, passing the ArgumentParser object 284 | parsers.append(e.parser) 285 | except ParserExceptions: 286 | sys.stderr.write( 287 | "Error while trying exception-catch method on {0}:\n".format( 288 | self.script_path 289 | ) 290 | ) 291 | errors["try-catch"] = "{0}\n".format(traceback.format_exc()) 292 | 293 | argparse.ArgumentParser.parse_args = parse_args_unmonkey 294 | 295 | if not parsers: 296 | try: 297 | module = imp.load_source("__name__", self.script_path) 298 | except Exception: 299 | sys.stderr.write("Error while loading {0}:\n".format(self.script_path)) 300 | errors["source-loader"] = "{0}\n".format(traceback.format_exc()) 301 | sys.stderr.write(self.error) 302 | else: 303 | main_module = ( 304 | module.main.__globals__ if hasattr(module, "main") else globals() 305 | ) 306 | parsers = [ 307 | v 308 | for i, v in chain(main_module.items(), vars(module).items()) 309 | if issubclass(type(v), argparse.ArgumentParser) 310 | ] 311 | if not parsers: 312 | try: 313 | with tempfile.NamedTemporaryFile(delete=False, mode="wb") as f: 314 | ast_source = source_parser.parse_source_file( 315 | self.script_path, ignore_bad_imports=self.ignore_bad_imports 316 | ) 317 | python_code = source_parser.convert_to_python(list(ast_source)) 318 | f.write("\n".join(python_code).encode()) 319 | module = imp.load_source("__main__", f.name) 320 | os.remove(f.name) 321 | except Exception: 322 | sys.stderr.write( 323 | "Error while converting {0} to ast:\n".format(self.script_path) 324 | ) 325 | errors["ast-parser"] = "{0}\n".format(traceback.format_exc()) 326 | sys.stderr.write(self.error) 327 | else: 328 | main_module = ( 329 | module.main.__globals__ if hasattr(module, "main") else globals() 330 | ) 331 | parsers = [ 332 | v 333 | for i, v in chain(main_module.items(), vars(module).items()) 334 | if issubclass(type(v), argparse.ArgumentParser) 335 | ] 336 | if not parsers: 337 | sys.stderr.write( 338 | "Unable to identify ArgParser for {0}:\n".format(self.script_path) 339 | ) 340 | self.error = "Unable to parse script. Errors encountered:\n {}".format( 341 | "\n".join( 342 | "Technique: {0}\nError: {1}".format(i, errors[i]) 343 | for i in sorted(errors) 344 | ) 345 | ) 346 | return 347 | 348 | self.is_valid = True 349 | self.parser = parsers[0] 350 | 351 | def process_parser(self): 352 | self.class_name = os.path.splitext(os.path.basename(self.script_path))[0] 353 | self.script_path = self.script_path 354 | self.script_description = getattr(self.parser, "description", None) 355 | self.script_version = getattr(self.parser, "version", None) 356 | 357 | parsers = [("", self.parser)] 358 | 359 | if self.parser._subparsers is not None: 360 | for action in self.parser._subparsers._actions: 361 | if isinstance(action, argparse._SubParsersAction): 362 | for parser_name, parser in action.choices.items(): 363 | parsers.append((parser_name, parser)) 364 | 365 | self.parsers = OrderedDict() 366 | 367 | for parser_name, parser in parsers: 368 | nodes = OrderedDict() 369 | containers = OrderedDict() 370 | 371 | mutex_groups = {} 372 | mutex_id_map = {} 373 | for mutex_group in parser._mutually_exclusive_groups: 374 | for action in mutex_group._group_actions: 375 | group_id = mutex_id_map.setdefault( 376 | id(mutex_group), len(mutex_id_map) 377 | ) 378 | mutex_groups[id(action)] = (group_id, mutex_group.title) 379 | 380 | for action in parser._actions: 381 | # The action is the subparser 382 | if isinstance(action, argparse._SubParsersAction): 383 | continue 384 | if self.script_version is None and isinstance( 385 | action, argparse._VersionAction 386 | ): 387 | self.script_version = action.version 388 | continue 389 | if action.default == argparse.SUPPRESS: 390 | continue 391 | node = ArgParseNode( 392 | action=action, mutex_group=mutex_groups.get(id(action)) 393 | ) 394 | container = action.container.title 395 | container_node = containers.get(container, None) 396 | if container_node is None: 397 | container_node = [] 398 | containers[container] = container_node 399 | nodes[node.name] = node 400 | container_node.append(node.name) 401 | 402 | self.parsers[parser_name] = {"nodes": nodes, "containers": containers} 403 | 404 | def get_script_description(self): 405 | input_dict = OrderedDict() 406 | parser_schema = { 407 | "name": self.class_name, 408 | "path": self.script_path, 409 | "description": self.script_description, 410 | "version": self.script_version, 411 | "inputs": input_dict, 412 | } 413 | 414 | for parser_name, parser_info in self.parsers.items(): 415 | parser_actions = [] 416 | input_dict[parser_name] = parser_actions 417 | containers, parser_nodes = parser_info["containers"], parser_info["nodes"] 418 | for container_name, container_nodes in containers.items(): 419 | parser_actions.append( 420 | { 421 | "group": container_name, 422 | "nodes": [ 423 | parser_nodes[node].node_attrs for node in container_nodes 424 | ], 425 | } 426 | ) 427 | 428 | return parser_schema 429 | -------------------------------------------------------------------------------- /clinto/parsers/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import copy 3 | import json 4 | import os 5 | import sys 6 | from contextlib import contextmanager 7 | 8 | 9 | def update_dict_copy(a, b): 10 | temp = copy.deepcopy(a) 11 | temp.update(b) 12 | return temp 13 | 14 | 15 | @contextmanager 16 | def inserted_sys_path(path): 17 | if path: 18 | sys.path.insert(0, path) 19 | yield 20 | sys.path.pop(0) 21 | 22 | 23 | class ClintoArgumentParserException(Exception): 24 | def __init__(self, parser, *args, **kwargs): 25 | self.parser = parser 26 | 27 | 28 | def parse_args_monkeypatch(self, *args, **kwargs): 29 | raise ClintoArgumentParserException(self) 30 | 31 | 32 | class BaseParser(object): 33 | def __init__(self, script_path=None, script_source=None, ignore_bad_imports=False): 34 | self.is_valid = False 35 | self.error = "" 36 | self.parser = None 37 | self.ignore_bad_imports = ignore_bad_imports 38 | 39 | self.script_path = script_path 40 | # We need this for heuristic, may as well happen once 41 | if self.script_path: 42 | self.script_ext = os.path.splitext(os.path.basename(self.script_path))[1] 43 | 44 | self.script_source = script_source 45 | 46 | self._heuristic_score = None 47 | 48 | with inserted_sys_path( 49 | os.path.dirname(self.script_path) if self.script_path else None 50 | ): 51 | self.extract_parser() 52 | 53 | if self.parser is None: 54 | return 55 | 56 | self.process_parser() 57 | 58 | @property 59 | def score(self): 60 | """ 61 | Calculate and return a heuristic score for this Parser against the provided 62 | script source and path. This is used to order the ArgumentParsers as "most likely to work" 63 | against a given script/source file. 64 | 65 | Each parser has a calculate_score() function that returns a list of booleans representing 66 | the matches against conditions. This is converted into a % match and used to sort parse engines. 67 | 68 | :return: float 69 | """ 70 | if self._heuristic_score is None: 71 | matches = self.heuristic() 72 | self._heuristic_score = float(sum(matches)) / float(len(matches)) 73 | return self._heuristic_score 74 | 75 | def extract_parser(self): 76 | pass 77 | 78 | def process_parser(self): 79 | pass 80 | 81 | def get_script_description(self): 82 | return {} 83 | 84 | @property 85 | def json(self): 86 | return json.dumps(self.get_script_description()) 87 | -------------------------------------------------------------------------------- /clinto/parsers/compat.py: -------------------------------------------------------------------------------- 1 | try: 2 | import docopt 3 | 4 | DocoptExit = docopt.DocoptExit 5 | except ImportError: 6 | DocoptExit = Exception 7 | 8 | ParserExceptions = (DocoptExit, Exception) 9 | -------------------------------------------------------------------------------- /clinto/parsers/constants.py: -------------------------------------------------------------------------------- 1 | # This indicates a parameter key should be specified for every argument. e.g. --foo 1 --foo 2 instead 2 | # of --foo 1 2 3 | SPECIFY_EVERY_PARAM = 'specify_every_param' -------------------------------------------------------------------------------- /clinto/parsers/docopt_.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import sys 4 | import os 5 | import json 6 | import imp 7 | import inspect 8 | import tempfile 9 | import traceback 10 | from collections import OrderedDict 11 | from itertools import chain 12 | from ..ast import source_parser 13 | from ..utils import is_upload, expand_iterable 14 | from .base import ( 15 | BaseParser, 16 | parse_args_monkeypatch, 17 | ClintoArgumentParserException, 18 | update_dict_copy, 19 | ) 20 | 21 | try: 22 | import docopt 23 | except ImportError: 24 | docopt = None 25 | 26 | 27 | GLOBAL_ATTRS = ["model", "type"] 28 | 29 | 30 | GLOBAL_ATTR_KWARGS = { 31 | "name": {"action_name": "dest"}, 32 | "value": {"action_name": "default"}, 33 | "required": {"action_name": "required"}, 34 | "help": {"action_name": "help"}, 35 | } 36 | 37 | TYPE_FIELDS = { 38 | # Python Builtins 39 | bool: { 40 | "model": "BooleanField", 41 | "type": "checkbox", 42 | "nullcheck": lambda x: x.value is None, 43 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 44 | }, 45 | float: { 46 | "model": "FloatField", 47 | "type": "text", 48 | "html5-type": "number", 49 | "nullcheck": lambda x: x.value is None, 50 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 51 | }, 52 | int: { 53 | "model": "IntegerField", 54 | "type": "text", 55 | "nullcheck": lambda x: x.value is None, 56 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 57 | }, 58 | None: { 59 | "model": "CharField", 60 | "type": "text", 61 | "nullcheck": lambda x: x.value is None, 62 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 63 | }, 64 | str: { 65 | "model": "CharField", 66 | "type": "text", 67 | "nullcheck": lambda x: x.value == "" or x.value is None, 68 | "attr_kwargs": GLOBAL_ATTR_KWARGS, 69 | }, 70 | } 71 | 72 | 73 | class DocOptNode(object): 74 | """ 75 | This class takes an argument parser entry and assigns it to a Build spec 76 | """ 77 | 78 | def __init__(self, name, option=None): 79 | field_type = TYPE_FIELDS.get(option.type) 80 | if field_type is None: 81 | field_types = [ 82 | i 83 | for i in fields.keys() 84 | if i is not None and issubclass(type(option.type), i) 85 | ] 86 | if len(field_types) > 1: 87 | field_types = [ 88 | i 89 | for i in fields.keys() 90 | if i is not None and isinstance(option.type, i) 91 | ] 92 | if len(field_types) == 1: 93 | field_type = fields[field_types[0]] 94 | self.node_attrs = dict([(i, field_type[i]) for i in GLOBAL_ATTRS]) 95 | self.node_attrs["name"] = name 96 | self.node_attrs["param"] = option.long if option.long else option.short 97 | 98 | @property 99 | def name(self): 100 | return self.node_attrs.get("name") 101 | 102 | def __str__(self): 103 | return json.dumps(self.node_attrs) 104 | 105 | 106 | class DocOptParser(BaseParser): 107 | def heuristic(self): 108 | return [ 109 | self.script_ext in [".py", ".py3", ".py2"], 110 | "docopt" in self.script_source, 111 | "__doc__" in self.script_source, 112 | ] 113 | 114 | def extract_parser(self): 115 | if docopt is None: 116 | return False 117 | 118 | try: 119 | module = imp.new_module("__name__") 120 | exec(self.script_source, module.__dict__) 121 | 122 | except Exception: 123 | sys.stderr.write("Error while loading {0}:\n".format(self.script_path)) 124 | self.error = "{0}\n".format(traceback.format_exc()) 125 | sys.stderr.write(self.error) 126 | return 127 | 128 | try: 129 | doc = module.__doc__ 130 | 131 | except AttributeError: 132 | return 133 | 134 | if doc is None: 135 | return 136 | 137 | # We have the documentation string in 'doc' 138 | self.is_valid = True 139 | self.parser = doc 140 | 141 | def process_parser(self): 142 | """ 143 | We can't use the exception catch trick for docopt because the module prevents access to 144 | it's innards __all__ = ['docopt']. Instead call with --help enforced, catch sys.exit and 145 | work up to the calling docopt function to pull out the elements. This is horrible. 146 | 147 | :return: 148 | """ 149 | 150 | try: 151 | # Parse with --help to enforce exit 152 | usage_sections = docopt.docopt(self.parser, ["--help"]) 153 | except SystemExit as e: 154 | parser = inspect.trace()[-2][0].f_locals 155 | 156 | """ 157 | docopt represents all values as strings and doesn't automatically cast, we probably want to do 158 | some testing to see if we can convert the default value (Option.value) to a particular type. 159 | """ 160 | 161 | def guess_type(s): 162 | try: 163 | v = float(s) 164 | v = int(s) 165 | v = s 166 | except ValueError: 167 | pass 168 | 169 | return type(v) 170 | 171 | self.script_groups = ["Arguments"] 172 | self.nodes = OrderedDict() 173 | self.containers = OrderedDict() 174 | self.containers["default"] = [] 175 | 176 | for option in parser["options"]: 177 | if option.long in ["--help", "--version"]: 178 | continue 179 | 180 | option.type = guess_type(option.value) 181 | option_name = option.long.strip("-") 182 | node = DocOptNode(option_name, option=option) 183 | 184 | self.nodes[option_name] = node 185 | self.containers["default"].append(option_name) 186 | 187 | self.class_name = os.path.splitext(os.path.basename(self.script_path))[0] 188 | self.script_path = self.script_path 189 | self.script_description = self.parser 190 | # TODO: Make AST compatible parser for docopt to extract the version provided to docopt function 191 | # self.script_version = parser.get('version') 192 | 193 | def get_script_description(self): 194 | # There are no real subparsers in docopt that we can access via introspection so we use the default 195 | # subparser of '' (no subparser/main parser) 196 | parser_name = "" 197 | parser_inputs = [ 198 | { 199 | "group": container_name, 200 | "nodes": [self.nodes[node].node_attrs for node in nodes], 201 | } 202 | for container_name, nodes in self.containers.items() 203 | ] 204 | return { 205 | "name": self.class_name, 206 | "path": self.script_path, 207 | "description": self.script_description, 208 | "inputs": OrderedDict([(parser_name, parser_inputs)]), 209 | } 210 | -------------------------------------------------------------------------------- /clinto/tests/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'chris' 2 | -------------------------------------------------------------------------------- /clinto/tests/argparse_scripts/choices.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | 4 | parser = argparse.ArgumentParser(description="Something") 5 | parser.add_argument("--version", action="version", version="3") 6 | parser.add_argument("first_pos") 7 | parser.add_argument("second-pos") 8 | parser.add_argument("--one-choice", choices=[0, 1, 2, 3], nargs=1) 9 | parser.add_argument("--two-choices", choices=[0, 1, 2, 3], nargs=2) 10 | parser.add_argument("--at-least-one-choice", choices=[0, 1, 2, 3], nargs="+") 11 | parser.add_argument("--all-choices", choices=[0, 1, 2, 3], nargs="*") 12 | parser.add_argument( 13 | "--need-at-least-one-numbers", type=int, nargs="+", required=True, action="append" 14 | ) 15 | parser.add_argument("--multiple-file-choices", type=argparse.FileType("r"), nargs="*") 16 | parser.add_argument( 17 | "--more-multiple-file-choices", type=argparse.FileType("r"), nargs="*" 18 | ) 19 | 20 | if __name__ == "__main__": 21 | args = parser.parse_args() 22 | sys.stdout.write("{}".format(args)) 23 | -------------------------------------------------------------------------------- /clinto/tests/argparse_scripts/data_reader.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wooey/clinto/866c5820185f08262724af81120e953ee4ee43c3/clinto/tests/argparse_scripts/data_reader.zip -------------------------------------------------------------------------------- /clinto/tests/argparse_scripts/error_script.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import something_i_dont_have 3 | 4 | parser = argparse.ArgumentParser(description="Something") 5 | parser.add_argument("-foo") 6 | 7 | if __name__ == "__main__": 8 | args = parser.parse_args() 9 | sys.stdout.write("{}".format(args)) 10 | -------------------------------------------------------------------------------- /clinto/tests/argparse_scripts/function_argtype.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import re 3 | 4 | def valid_date_type(arg_date): 5 | """ 6 | Custom argparse *date* type for user dates values given from the command line 7 | """ 8 | try: 9 | return re.search('([0-9]{8})', arg_date) 10 | except ValueError: 11 | msg = 'Given Date ({0}) not valid! Expected format, YYYYMMDD!'.format(arg_date_str) 12 | raise argparse.ArgumentTypeError(msg) 13 | 14 | parser = argparse.ArgumentParser(description='Test parsing of a function type argument.') 15 | parser.add_argument( 16 | 'start_date', 17 | type=valid_date_type, 18 | help='Use date in format YYYYMMDD (e.g. 20180131)', 19 | default='20180131' 20 | ) 21 | parser.add_argument( 22 | 'lowercase', 23 | type=str.lower, 24 | help='Lowercase it', 25 | default='ABC', 26 | ) 27 | 28 | 29 | def main(): 30 | args = parser.parse_args() 31 | print(args.start_date) 32 | 33 | 34 | if __name__ == '__main__': 35 | main() -------------------------------------------------------------------------------- /clinto/tests/argparse_scripts/mutually_exclusive.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | 4 | parser = argparse.ArgumentParser(description="Something") 5 | group = parser.add_mutually_exclusive_group() 6 | group.add_argument("--foo", action="store_true") 7 | group.add_argument("--bar", action="store_false") 8 | group2 = parser.add_mutually_exclusive_group() 9 | group2.add_argument("--foo2", action="store_true") 10 | group2.add_argument("--bar2", action="store_false") 11 | 12 | if __name__ == "__main__": 13 | args = parser.parse_args() 14 | sys.stdout.write("{}".format(args)) 15 | -------------------------------------------------------------------------------- /clinto/tests/argparse_scripts/subparser_script.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | parser = argparse.ArgumentParser(description='A useless script with subparsers') 4 | parser.add_argument('--test-arg', type=float, default=2.3, help='a test arg for the main parser') 5 | 6 | subparsers = parser.add_subparsers(help='commands') 7 | 8 | subparser1 = subparsers.add_parser('subparser1', help='Subparser 1') 9 | subparser1.add_argument('--sp1', type=int, default=1, help='sp1') 10 | 11 | if __name__ == '__main__': 12 | args = parser.parse_args() 13 | -------------------------------------------------------------------------------- /clinto/tests/argparse_scripts/zip_app_rel_imports.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wooey/clinto/866c5820185f08262724af81120e953ee4ee43c3/clinto/tests/argparse_scripts/zip_app_rel_imports.zip -------------------------------------------------------------------------------- /clinto/tests/docopt_scripts/naval_fate.py: -------------------------------------------------------------------------------- 1 | # Toy example from https://github.com/docopt/docopt/blob/master/examples/naval_fate.py 2 | """Naval Fate. 3 | Usage: 4 | naval_fate.py ship new ... 5 | naval_fate.py ship move [--speed=] 6 | naval_fate.py ship shoot 7 | naval_fate.py mine (set|remove) [--moored|--drifting] 8 | naval_fate.py -h | --help 9 | naval_fate.py --version 10 | Options: 11 | -h --help Show this screen. 12 | --version Show version. 13 | --speed= Speed in knots [default: 10]. 14 | --moored Moored (anchored) mine. 15 | --drifting Drifting mine. 16 | """ 17 | from docopt import docopt 18 | 19 | 20 | if __name__ == '__main__': 21 | arguments = docopt(__doc__, version='Naval Fate 2.0') 22 | print(arguments) 23 | -------------------------------------------------------------------------------- /clinto/tests/factories.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | param_int = 0 4 | 5 | 6 | class BaseDict(dict): 7 | def __init__(self, **kwargs): 8 | super(BaseDict, self).__init__() 9 | global param_int 10 | base_dict = { 11 | "param": "--test-param{}".format(param_int), 12 | "required": False, 13 | } 14 | param_int = param_int + 1 15 | base_dict.update(**kwargs) 16 | self.update(**base_dict) 17 | 18 | 19 | class ArgParseFactory(object): 20 | DESCRIPTION = "ArgParse Factory" 21 | COMMON = ["param", "required"] 22 | BASEFIELD = BaseDict() 23 | FILEFIELD = BaseDict( 24 | param="--ff-out", type=argparse.FileType("w"), help="Test help" 25 | ) 26 | UPLOADFILEFIELD = BaseDict(type=argparse.FileType("r")) 27 | CHOICEFIELD = BaseDict(choices=["a", "b", "c"]) 28 | RANGECHOICES = BaseDict(choices=range(-10, 20, 2)) 29 | 30 | def __init__(self): 31 | super(ArgParseFactory, self).__init__() 32 | self.parser = argparse.ArgumentParser(description=self.DESCRIPTION) 33 | self.filefield = self.parser.add_argument( 34 | self.FILEFIELD["param"], 35 | help=self.FILEFIELD["help"], 36 | type=self.FILEFIELD["type"], 37 | ) 38 | self.uploadfield = self.parser.add_argument( 39 | self.UPLOADFILEFIELD["param"], type=self.UPLOADFILEFIELD["type"] 40 | ) 41 | self.choicefield = self.parser.add_argument( 42 | self.CHOICEFIELD["param"], choices=self.CHOICEFIELD["choices"] 43 | ) 44 | self.rangefield = self.parser.add_argument( 45 | self.RANGECHOICES["param"], choices=self.RANGECHOICES["choices"] 46 | ) 47 | -------------------------------------------------------------------------------- /clinto/tests/test_parsers.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import unittest 4 | 5 | from . import factories 6 | from clinto.version import PY_MINOR_VERSION, PY36 7 | from clinto.parsers.argparse_ import ArgParseNode, expand_iterable, ArgParseParser 8 | from clinto.parsers.constants import SPECIFY_EVERY_PARAM 9 | from clinto.parser import Parser 10 | 11 | _parser = argparse.ArgumentParser() 12 | OPTIONAL_TITLE = _parser._optionals.title 13 | 14 | 15 | class TestArgParse(unittest.TestCase): 16 | def setUp(self): 17 | self.parser = factories.ArgParseFactory() 18 | 19 | def test_func(fields, defs): 20 | for i in self.parser.COMMON: 21 | assert fields[i] == defs[i], "{}: {} is not {}\n".format( 22 | i, fields[i], defs[i] 23 | ) 24 | 25 | self.base_test = test_func 26 | self.base_dir = os.path.split(__file__)[0] 27 | self.parser_script_dir = "argparse_scripts" 28 | self.script_dir = os.path.join(self.base_dir, self.parser_script_dir) 29 | 30 | def test_subparser(self): 31 | script_path = os.path.join(self.script_dir, "subparser_script.py") 32 | parser = Parser(script_path=script_path) 33 | description = parser.get_script_description() 34 | main_parser = description["inputs"][""] 35 | main_parser_group1 = main_parser[0] 36 | self.assertEqual(main_parser_group1["nodes"][0]["name"], "test_arg") 37 | self.assertEqual(main_parser_group1["group"], OPTIONAL_TITLE) 38 | 39 | subparser1 = description["inputs"]["subparser1"] 40 | subparser_group1 = subparser1[0] 41 | self.assertEqual(subparser_group1["nodes"][0]["name"], "sp1") 42 | 43 | def test_script_version(self): 44 | script_path = os.path.join(self.script_dir, "choices.py") 45 | parser = Parser(script_path=script_path) 46 | description = parser.get_script_description() 47 | self.assertEqual(description["version"], "3") 48 | 49 | def test_file_field(self): 50 | filefield = ArgParseNode(action=self.parser.filefield) 51 | attrs = filefield.node_attrs 52 | self.base_test(attrs, self.parser.FILEFIELD) 53 | assert attrs["upload"] is False 54 | assert attrs["model"] == "FileField" 55 | 56 | def test_upload_file_field(self): 57 | upload = ArgParseNode(action=self.parser.uploadfield) 58 | attrs = upload.node_attrs 59 | self.base_test(attrs, self.parser.UPLOADFILEFIELD) 60 | assert attrs["upload"] is True 61 | assert attrs["model"] == "FileField" 62 | 63 | def test_choice_field(self): 64 | choicefield = ArgParseNode(action=self.parser.choicefield) 65 | attrs = choicefield.node_attrs 66 | self.base_test(attrs, self.parser.CHOICEFIELD) 67 | assert attrs["model"] == "CharField" 68 | # test range 69 | rangefield = ArgParseNode(action=self.parser.rangefield) 70 | assert rangefield.node_attrs["choices"] == expand_iterable( 71 | self.parser.rangefield.choices 72 | ) 73 | 74 | def test_argparse_script(self): 75 | script_path = os.path.join(self.script_dir, "choices.py") 76 | parser = Parser(script_path=script_path) 77 | 78 | script_params = parser.get_script_description() 79 | self.assertEqual(script_params["path"], script_path) 80 | 81 | # Make sure we return parameters in the order the script defined them and in groups 82 | # We do not test this that exhaustively atm since the structure is likely to change when subparsers 83 | # are added 84 | self.assertDictEqual( 85 | script_params["inputs"][""][0], 86 | { 87 | "nodes": [ 88 | { 89 | "param_action": set([]), 90 | "name": "first_pos", 91 | "required": True, 92 | "param": "", 93 | "choices": None, 94 | "choice_limit": None, 95 | "model": "CharField", 96 | "type": "text", 97 | "help": None, 98 | "mutex_group": {}, 99 | }, 100 | { 101 | "param_action": set([]), 102 | "name": "second-pos", 103 | "required": True, 104 | "param": "", 105 | "choices": None, 106 | "choice_limit": None, 107 | "model": "CharField", 108 | "type": "text", 109 | "help": None, 110 | "mutex_group": {}, 111 | }, 112 | ], 113 | "group": "positional arguments", 114 | }, 115 | ) 116 | 117 | def test_argparse_specify_every_param(self): 118 | script_path = os.path.join(self.script_dir, "choices.py") 119 | parser = Parser(script_path=script_path) 120 | 121 | script_params = parser.get_script_description() 122 | self.assertEqual(script_params["path"], script_path) 123 | 124 | append_field = [ 125 | i 126 | for i in script_params["inputs"][""][1]["nodes"] 127 | if i["param"] == "--need-at-least-one-numbers" 128 | ][0] 129 | self.assertIn(SPECIFY_EVERY_PARAM, append_field["param_action"]) 130 | 131 | def test_function_type_script(self): 132 | script_path = os.path.join(self.script_dir, "function_argtype.py") 133 | parser = Parser(script_path=script_path) 134 | 135 | script_params = parser.get_script_description() 136 | self.assertEqual(script_params["path"], script_path) 137 | 138 | self.assertDictEqual( 139 | script_params["inputs"][""][0], 140 | { 141 | "nodes": [ 142 | { 143 | "param_action": set([]), 144 | "name": "start_date", 145 | "required": True, 146 | "param": "", 147 | "choices": None, 148 | "choice_limit": None, 149 | "model": "CharField", 150 | "type": "text", 151 | "help": "Use date in format YYYYMMDD (e.g. 20180131)", 152 | # The default argument 153 | "value": "20180131", 154 | "mutex_group": {}, 155 | }, 156 | { 157 | "param_action": set([]), 158 | "name": "lowercase", 159 | "required": True, 160 | "param": "", 161 | "choices": None, 162 | "value": "ABC", 163 | "choice_limit": None, 164 | "model": "CharField", 165 | "type": "text", 166 | "help": "Lowercase it", 167 | "mutex_group": {}, 168 | }, 169 | ], 170 | "group": "positional arguments", 171 | }, 172 | ) 173 | 174 | def test_error_script(self): 175 | script_path = os.path.join(self.script_dir, "error_script.py") 176 | parser = Parser(script_path=script_path) 177 | 178 | if PY_MINOR_VERSION >= PY36: 179 | self.assertIn("ModuleNotFoundError", parser.error) 180 | else: 181 | self.assertIn("ImportError", parser.error) 182 | self.assertIn("something_i_dont_have", parser.error) 183 | 184 | script_path = os.path.join(self.script_dir, "choices.py") 185 | parser = Parser(script_path=script_path) 186 | self.assertEquals("", parser.error) 187 | 188 | def test_can_exclude_bad_imports(self): 189 | self.maxDiff = None 190 | script_path = os.path.join(self.script_dir, "error_script.py") 191 | parser = Parser(script_path=script_path, ignore_bad_imports=True) 192 | self.assertEquals("", parser.error) 193 | script_params = parser.get_script_description() 194 | self.assertDictEqual( 195 | script_params["inputs"][""][0], 196 | { 197 | "group": OPTIONAL_TITLE, 198 | "nodes": [ 199 | { 200 | "model": "CharField", 201 | "type": "text", 202 | "mutex_group": {}, 203 | "name": "foo", 204 | "required": False, 205 | "help": None, 206 | "param": "-foo", 207 | "param_action": set(), 208 | "choices": None, 209 | "choice_limit": None, 210 | } 211 | ], 212 | }, 213 | ) 214 | 215 | def test_zipapp(self): 216 | script_path = os.path.join(self.script_dir, "data_reader.zip") 217 | parser = Parser(script_path=script_path) 218 | script_params = parser.get_script_description() 219 | self.assertDictEqual( 220 | script_params["inputs"][""][0], 221 | { 222 | "nodes": [ 223 | { 224 | "param_action": set([]), 225 | "name": "n", 226 | "required": False, 227 | "param": "-n", 228 | "choices": None, 229 | "value": -1, 230 | "choice_limit": None, 231 | "model": "IntegerField", 232 | "type": "text", 233 | "help": "The number of rows to read.", 234 | "mutex_group": {}, 235 | } 236 | ], 237 | "group": OPTIONAL_TITLE, 238 | }, 239 | ) 240 | 241 | def test_zipapp_with_relative_imports(self): 242 | script_path = os.path.join(self.script_dir, "zip_app_rel_imports.zip") 243 | parser = Parser(script_path=script_path) 244 | self.assertIsNotNone(parser.get_script_description()) 245 | 246 | def test_mutually_exclusive_groups(self): 247 | script_path = os.path.join(self.script_dir, "mutually_exclusive.py") 248 | parser = Parser(script_path=script_path) 249 | script_params = parser.get_script_description() 250 | self.assertDictEqual( 251 | script_params["inputs"][""][0], 252 | { 253 | "nodes": [ 254 | { 255 | "model": "BooleanField", 256 | "type": "checkbox", 257 | "mutex_group": {"id": 0, "title": None}, 258 | "name": "foo", 259 | "required": False, 260 | "help": None, 261 | "param": "--foo", 262 | "param_action": set(), 263 | "choices": None, 264 | "choice_limit": 0, 265 | "checked": False, 266 | }, 267 | { 268 | "model": "BooleanField", 269 | "type": "checkbox", 270 | "mutex_group": {"id": 0, "title": None}, 271 | "name": "bar", 272 | "required": False, 273 | "help": None, 274 | "param": "--bar", 275 | "param_action": set(), 276 | "choices": None, 277 | "choice_limit": 0, 278 | "checked": True, 279 | }, 280 | { 281 | "model": "BooleanField", 282 | "type": "checkbox", 283 | "mutex_group": {"id": 1, "title": None}, 284 | "name": "foo2", 285 | "required": False, 286 | "help": None, 287 | "param": "--foo2", 288 | "param_action": set(), 289 | "choices": None, 290 | "choice_limit": 0, 291 | "checked": False, 292 | }, 293 | { 294 | "model": "BooleanField", 295 | "type": "checkbox", 296 | "mutex_group": {"id": 1, "title": None}, 297 | "name": "bar2", 298 | "required": False, 299 | "help": None, 300 | "param": "--bar2", 301 | "param_action": set(), 302 | "choices": None, 303 | "choice_limit": 0, 304 | "checked": True, 305 | }, 306 | ], 307 | "group": OPTIONAL_TITLE, 308 | }, 309 | ) 310 | 311 | def test_reports_errors_for_each_loader(self): 312 | script_path = os.path.join(self.script_dir, "error_script.py") 313 | parser = Parser(script_path=script_path) 314 | 315 | for technique in ["ast-parser", "source-loader", "try-catch"]: 316 | self.assertIn(technique, parser.error) 317 | 318 | 319 | class TestDocOpt(unittest.TestCase): 320 | def setUp(self): 321 | self.base_dir = os.path.split(__file__)[0] 322 | self.parser_script_dir = "docopt_scripts" 323 | self.script_dir = os.path.join(self.base_dir, self.parser_script_dir) 324 | 325 | def test_naval_fate(self): 326 | script_path = os.path.join(self.script_dir, "naval_fate.py") 327 | parser = Parser(script_path=script_path) 328 | script_params = parser.get_script_description() 329 | self.assertEqual(script_params["path"], script_path) 330 | self.assertEqual(script_params["name"], "naval_fate") 331 | 332 | # Make sure we return parameters in the order the script defined them and in groups 333 | # We do not test this that exhaustively atm since the structure is likely to change when subparsers 334 | # are added 335 | self.assertDictEqual( 336 | script_params["inputs"][""][0], 337 | { 338 | "nodes": [ 339 | { 340 | "model": "CharField", 341 | "type": "text", 342 | "name": "speed", 343 | "param": "--speed", 344 | }, 345 | { 346 | "model": "BooleanField", 347 | "type": "checkbox", 348 | "name": "moored", 349 | "param": "--moored", 350 | }, 351 | { 352 | "model": "BooleanField", 353 | "type": "checkbox", 354 | "name": "drifting", 355 | "param": "--drifting", 356 | }, 357 | ], 358 | "group": "default", 359 | }, 360 | ) 361 | 362 | 363 | if __name__ == "__main__": 364 | unittest.main() 365 | -------------------------------------------------------------------------------- /clinto/utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | # TODO: Move this stuff to a utils file 4 | def is_upload(action): 5 | """Checks if this should be a user upload 6 | 7 | :param action: 8 | :return: True if this is a file we intend to upload from the user 9 | """ 10 | return "r" in action.type._mode and ( 11 | action.default is None 12 | or getattr(action.default, "name") not in (sys.stderr.name, sys.stdout.name) 13 | ) 14 | 15 | 16 | def expand_iterable(choices): 17 | """ 18 | Expands an iterable into a list. We use this to expand generators/etc. 19 | """ 20 | return [i for i in choices] if hasattr(choices, "__iter__") else None 21 | -------------------------------------------------------------------------------- /clinto/version.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from distutils.version import StrictVersion 4 | 5 | PY_FULL_VERSION = StrictVersion( 6 | "{}.{}.{}".format( 7 | sys.version_info.major, sys.version_info.minor, sys.version_info.micro 8 | ) 9 | ) 10 | PY_MINOR_VERSION = StrictVersion( 11 | "{}.{}".format(sys.version_info.major, sys.version_info.minor) 12 | ) 13 | PY36 = StrictVersion("3.6") 14 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | coveralls 2 | pytest 3 | pytest-cov 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.5.1 3 | commit = True 4 | tag = True 5 | parse = (?P\d+)\.(?P\d+)\.(?P\d+)(rc(?P\d+))? 6 | serialize = 7 | {major}.{minor}.{patch}rc{rc} 8 | {major}.{minor}.{patch} 9 | 10 | [bumpversion:file:setup.py] 11 | search = version="{current_version}" 12 | replace = version="{new_version}" 13 | 14 | [flake8] 15 | ignore = E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E131,E203,E231,E265,E501,E731,F841,W503,W504 16 | exclude = 17 | .git, 18 | .tox, 19 | __pycache__, 20 | build, 21 | dist, 22 | statistics = True 23 | 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup, find_packages 3 | 4 | # allow setup.py to be run from any path 5 | os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) 6 | 7 | setup( 8 | name="clinto", 9 | version="0.5.1", 10 | packages=find_packages(), 11 | scripts=[], 12 | install_requires=["astor"], 13 | include_package_data=True, 14 | description="Clinto", 15 | url="http://www.github.com/wooey/clinto", 16 | author="Chris Mitchell", 17 | author_email="chris.mit7@gmail.com", 18 | classifiers=[ 19 | "Intended Audience :: Developers", 20 | "Operating System :: OS Independent", 21 | "Programming Language :: Python", 22 | ], 23 | ) 24 | --------------------------------------------------------------------------------