├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── README.md ├── cwlgen ├── __init__.py ├── commandlinebinding.py ├── commandlinetool.py ├── common.py ├── import_cwl.py ├── requirements.py ├── utils.py ├── version.py ├── workflow.py └── workflowdeps.py ├── doc ├── Makefile └── source │ ├── _templates │ └── layout.html │ ├── changelogs.rst │ ├── classes.rst │ ├── commandlinetoolclasses.rst │ ├── conf.py │ ├── index.rst │ ├── installation.rst │ ├── references.rst │ ├── user_guide.rst │ └── workflowclasses.rst ├── examples ├── grep.cwl ├── grep.txt ├── grep_example.py ├── readme_example.py └── underdog_lyrics.txt ├── requirements.txt ├── setup.py └── test ├── import_commandlinetool.cwl ├── import_workflow.cwl ├── int_tool.cwl ├── test_export.cwl ├── test_export.py ├── test_full_export.cwl ├── test_unit_cwlgen.py ├── test_unit_import_tool.py ├── test_unit_import_workflow.py ├── test_unit_requirements.py ├── test_unit_typing.py └── test_unit_workflow.py /.gitignore: -------------------------------------------------------------------------------- 1 | test/__pycache__ 2 | cwlgen/__pycache__ 3 | doc/build/ 4 | .ropeproject/ 5 | build/ 6 | cwlgen.egg* 7 | dist/ 8 | *.pyc 9 | .DS_Store 10 | .idea/ 11 | .coverage 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - '2.7' 4 | - '3.6' 5 | install: 6 | - pip install . 7 | - pip install codecov coverage requests_mock nose_parameterized 8 | script: nosetests --with-coverage --cover-package=cwlgen 9 | after_success: 10 | - codecov 11 | deploy: 12 | provider: pypi 13 | user: khillion 14 | password: 15 | secure: FcZL32itgRrdyeOC7mmCmqddE7Vise/UfpyJTcYkePkMAinhnq08dxBVn/AgYKmUg4/FqyM+mlQPQ3vo9EMMEbuUoTu6Pk8i3l2GUY1JuCXsN7BTfr0slC3ofpnWpruw85vzl8rU+YEy8bU7os65TapBKeGOihIx2Y+vBhaN6xdMzXnMGJS2PhXuPajMKafFyBBiaP/hNX9jcCDFBE9dD2FE5OjVMhpJRU2miqQuVgxmux4xkf00cSohPza+c7o3a82banitW3d8p25b1EWK+UxrauQy+FAPCw0WJ37K1Cd4BUjMMszSBS+kfOtmfIOuc5PWVJ+KGHPySRHLf+JSOnyvhhrijfkO7gE7N0608+7nX6DcIqbv47Kr9b/IMzm7nAhAhY3XkF1UV+wziFscEjV9suC1dSlI7J2IEolzu76XrTRCbSv/8vf9rNwCLhNEh0w81XomuUoGE/GW+ho0Evh/t66J+ICy+JagjQp6w2NiA6BqNOGXKAJzDshbmgp6J6WQ4KqfSmmsm8hQynIkpvXpeuLS811MmwEvjiZ+NlwyqT9MmC+91CEHW9hc8iJ6+Ug/yfjj5iYIJ8qxY9ksAZUGM63IYuz53muOwTm4FNrS8jmAqU4Wwzrx3pswkv7ffoDPgXssOIBuK+pzhHl0+HKMNcIj47k+FkqlIQbMDro= 16 | on: 17 | tags: true 18 | distributions: sdist bdist_wheel 19 | repo: common-workflow-language/python-cwlgen 20 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Kenzo-Hugo Hillion 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-cwlgen (deprecated) 2 | 3 | ## Deprecated 4 | 5 | `python-cwlgen` is now deprecated, please use [`cwl-utils >= 0.4`](https://github.com/common-workflow-language/cwl-utils). 6 | 7 | ```sh 8 | from cwl_utils import parser_v1_0 9 | 10 | # You could alias this as cwlgen to simplify the migration 11 | from cwl_utils import parser_v1_0 as cwlgen 12 | ``` 13 | 14 | Migration notes: 15 | 16 | - Method changes 17 | - `get_dict() → save()` 18 | - `parse_cwl(cwlfile)` → `load_document(cwlfile)` 19 | - `parse_dict` → No super clear analogue, but loaded through `_RecordLoader(CommandLineTool)` || `_UnionLoader((CommandLineToolLoader, ...workflow + other loaders)` 20 | 21 | - Field names: 22 | - Uses `camelCase` instead of `snake_case` 23 | - No more special field names, eg: 24 | - `tool_id` | `workflow_id` | `input_id` | etc → `id` 25 | - `StepInput`: `inputs` → `in_` 26 | 27 | - Other notes: 28 | - Classes aren't nested anymore, ie: `cwlgen.InitialWorkDirRequirement.Dirent` → `cwl_utils.parser_v1_0.Dirent`. 29 | - Take care if you're migrating to a newer spec, as some classes might have changed names (notably: `InputParameter` -> `WorkflowInputParameter`) 30 | - Don't forget to catch all references of cwlgen, as missing one (or using mismatch versions of the parser) will cause: 31 | 32 | ```python 33 | raise RepresenterError('cannot represent an object: %s' % (data,)) 34 | ruamel.yaml.representer.RepresenterError: cannot represent an object: 35 | 36 | ``` 37 | 38 | If you have issues with the migration, please see [this thread](https://github.com/common-workflow-language/python-cwlgen/issues/27) or raise an issue on CWLUtils. 39 | 40 | --- 41 | 42 | ## Original README 43 | 44 | [![Build Status](https://travis-ci.org/common-workflow-language/python-cwlgen.svg?branch=master)](https://travis-ci.org/common-workflow-language/python-cwlgen) 45 | [![codecov](https://codecov.io/gh/common-workflow-language/python-cwlgen/branch/master/graph/badge.svg)](https://codecov.io/gh/common-workflow-language/python-cwlgen) 46 | [![Documentation Status](https://readthedocs.org/projects/python-cwlgen/badge/?version=latest)](http://python-cwlgen.readthedocs.io/en/latest/?badge=latest) 47 | [![PyPI version](https://badge.fury.io/py/cwlgen.svg)](https://badge.fury.io/py/cwlgen) 48 | 49 | Python-cwlgen is a python library for the generation of CWL programmatically. 50 | It supports the generation of CommandLineTool, Workflow and DockerRequirement. 51 | The library works for both Python 2.7.12+ and 3.6.0. 52 | 53 | ------------------------ 54 | 55 | 56 | ## Common Workflow Language 57 | 58 | [Common Workflow Language (CWL)](https://www.commonwl.org/v1.0/index.html) is a language to describe workflows. 59 | The [user guide](http://www.commonwl.org/user_guide/01-introduction/index.html) 60 | gives a gentle explanation of what its goals are, but broadly: 61 | 62 | 1. Stop writing bash scripts for long complex jobs. 63 | 2. Take pipelines anywhere (portability). 64 | 3. Enforce reproducibility guidelines. 65 | 66 | This python repository is a python wrapper for _most_ of the classes (work in progress), 67 | allowing you to build the structure of the workflow in Python and have this module generate and export CWL for you. 68 | 69 | **Nb:** This doesn't check the logic of Workflows or CommandLineTools for you. 70 | [CWLTool](https://github.com/common-workflow-language/cwltool) has a `--validate` mode that you can use. 71 | 72 | ## Quick-start guide 73 | 74 | You can install python-cwlgen through pip with the following command: 75 | 76 | ```bash 77 | pip install cwlgen 78 | ``` 79 | 80 | ### How it works? 81 | 82 | This repository contains a number of python classes that mirror the CWL specifications ([Workflow](https://www.commonwl.org/v1.0/Workflow.html)| 83 | [CommandLineTool](https://www.commonwl.org/v1.0/CommandLineTool.html)). In essence, each class's initializer has all 84 | of the properties it expects, which may be another object. The classes include the relevant docstrings to give you 85 | context of classes and their properties. 86 | 87 | The `examples/` folder contains some simple examples, however in essence you simply initialize the class you're 88 | trying to build. An initializer for a class has all of the properties it expects which may be another object. 89 | 90 | 91 | _Creating a CommandLineTool_ 92 | ```python 93 | import cwlgen 94 | 95 | tool_object = cwlgen.CommandLineTool(tool_id="echo-tool", base_command="echo", label=None, doc=None, 96 | cwl_version="v1.0", stdin=None, stderr=None, stdout=None, path=None) 97 | tool_object.inputs.append( 98 | cwlgen.CommandInputParameter("myParamId", param_type="string", label=None, secondary_files=None, param_format=None, 99 | streamable=None, doc=None, input_binding=None, default=None) 100 | ) 101 | 102 | # to get the dictionary representation: 103 | dict_to_export = tool_object.get_dict() 104 | 105 | # to get the string representation (YAML) 106 | yaml_export = tool_object.export_string() 107 | 108 | # print to console 109 | tool_object.export() 110 | 111 | # print to file 112 | tool_object.export("echotool.cwl") 113 | ``` 114 | 115 | ## References 116 | 117 | CWL is developed by an informal, multi-vendor working group consisting of organizations and individuals 118 | aiming to enable scientists to share data analysis workflows. 119 | The [CWL project is on Github](https://github.com/common-workflow-language/common-workflow-language). 120 | 121 | 122 | ## Known issues 123 | 124 | - `SchemaDefRequirement` doesn't parse the `types` subfield into the specific types 125 | (`InputRecordSchema | InputEnumSchema | InputArraySchema`), but leaves them as a simple dictionary. -------------------------------------------------------------------------------- /cwlgen/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Library to handle the manipulation and generation of CWL tool 3 | ''' 4 | 5 | import warnings 6 | 7 | warnings.warn( 8 | "The cwlgen module is deprecated, please use " 9 | "cwl-utils >= 0.4: https://github.com/common-workflow-language/cwl-utils", 10 | DeprecationWarning, 11 | stacklevel=2 12 | ) 13 | 14 | 15 | # Import ------------------------------ 16 | 17 | # General libraries 18 | import logging 19 | 20 | # External libraries 21 | import ruamel.yaml 22 | import six 23 | from .version import __version__ 24 | 25 | from .utils import literal, literal_presenter 26 | 27 | from .import_cwl import parse_cwl, parse_cwl_dict 28 | 29 | logging.basicConfig(level=logging.INFO) 30 | _LOGGER = logging.getLogger(__name__) 31 | 32 | # imports for __init__ 33 | 34 | from .common import * 35 | from .commandlinetool import * 36 | from .workflow import * 37 | from .workflowdeps import * 38 | from .commandlinebinding import CommandLineBinding 39 | from .requirements import * 40 | 41 | -------------------------------------------------------------------------------- /cwlgen/commandlinebinding.py: -------------------------------------------------------------------------------- 1 | from .utils import Serializable 2 | 3 | 4 | class CommandLineBinding(Serializable): 5 | """ 6 | The binding behavior when building the command line depends on the data type of the value. 7 | If there is a mismatch between the type described by the input schema and the effective value, 8 | such as resulting from an expression evaluation, an implementation must use the data type of the effective value. 9 | 10 | Documentation: https://www.commonwl.org/v1.0/CommandLineTool.html#CommandLineBinding 11 | """ 12 | 13 | def __init__(self, load_contents=None, position=None, prefix=None, separate=None, 14 | item_separator=None, value_from=None, shell_quote=None): 15 | """ 16 | :param load_contents: Read up to the fist 64 KiB of text from the file and 17 | place it in the "contents" field of the file object 18 | :type load_contents: BOOLEAN 19 | :param position: The sorting key 20 | :type position: INT 21 | :param prefix: Command line prefix to add before the value 22 | :type prefix: STRING 23 | :param separate: 24 | :type separate: BOOLEAN 25 | :param item_separator: Join the array elements into a single string separated by this item 26 | :type item_separator: STRING 27 | :param value_from: Use this as the value 28 | :type value_from: STRING 29 | :param shell_quote: Value is quoted on the command line 30 | :type shell_quote: BOOLEAN 31 | """ 32 | self.loadContents = load_contents 33 | self.position = position 34 | self.prefix = prefix 35 | self.separate = separate 36 | self.itemSeparator = item_separator 37 | self.valueFrom = value_from 38 | self.shellQuote = shell_quote 39 | -------------------------------------------------------------------------------- /cwlgen/commandlinetool.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | # External libraries 4 | import ruamel.yaml 5 | 6 | from cwlgen.commandlinebinding import CommandLineBinding 7 | from .common import CWL_VERSIONS, DEF_VERSION, CWL_SHEBANG, Namespaces, Parameter 8 | from .requirements import * 9 | from .utils import literal, literal_presenter, Serializable, value_or_default 10 | 11 | logging.basicConfig(level=logging.INFO) 12 | _LOGGER = logging.getLogger(__name__) 13 | 14 | # Function(s) ------------------------------ 15 | 16 | # Class(es) ------------------------------ 17 | 18 | 19 | class CommandOutputBinding(Serializable): 20 | """ 21 | Describes how to generate an output parameter based on the files produced. 22 | """ 23 | 24 | def __init__(self, glob=None, load_contents=None, output_eval=None): 25 | """ 26 | :param glob: Find corresponding file(s) 27 | :type glob: STRING 28 | :param load_contents: For each file matched, read up to the 1st 64 KiB of text and 29 | place it in the contents field 30 | :type load_contents: BOOLEAN 31 | :param output_eval: Evaluate an expression to generate the output value 32 | :type output_eval: STRING 33 | """ 34 | self.glob = glob 35 | self.loadContents = load_contents 36 | self.outputEval = output_eval 37 | 38 | 39 | class CommandInputParameter(Parameter): 40 | """ 41 | An input parameter for a :class:`cwlgen.CommandLineTool`. 42 | """ 43 | 44 | parse_types = {"inputBinding": [CommandLineBinding]} 45 | 46 | def __init__( 47 | self, 48 | param_id, 49 | label=None, 50 | secondary_files=None, 51 | param_format=None, 52 | streamable=None, 53 | doc=None, 54 | input_binding=None, 55 | default=None, 56 | param_type=None, 57 | ): 58 | """ 59 | :param param_id: unique identifier for this parameter 60 | :type param_id: STRING 61 | :param label: short, human-readable label 62 | :type label: STRING 63 | :param secondary_files: If type is a file, describes files that must be 64 | included alongside the primary file(s) 65 | :type secondary_files: STRING 66 | :param param_format: If type is a file, uri to ontology of the format or exact format. 67 | :type param_format: STRING 68 | :param streamable: If type is a file, true indicates that the file is read or written 69 | sequentially without seeking 70 | :type streamable: BOOLEAN 71 | :param doc: documentation 72 | :type doc: STRING 73 | :param input_binding: describes how to handle the input 74 | :type input_binding: :class:`cwlgen.CommandLineBinding` object 75 | :param default: default value 76 | :type default: STRING 77 | :param param_type: type of data assigned to the parameter corresponding to CWLType 78 | :type param_type: STRING 79 | """ 80 | Parameter.__init__( 81 | self, 82 | param_id=param_id, 83 | label=label, 84 | secondary_files=secondary_files, 85 | param_format=param_format, 86 | streamable=streamable, 87 | doc=doc, 88 | param_type=param_type, 89 | ) 90 | self.inputBinding = input_binding 91 | self.default = default 92 | 93 | 94 | class CommandOutputParameter(Parameter): 95 | """ 96 | An output parameter for a :class:`cwlgen.CommandLineTool`. 97 | """ 98 | 99 | parse_types = {"outputBinding": [CommandOutputBinding]} 100 | 101 | def __init__( 102 | self, 103 | param_id, 104 | label=None, 105 | secondary_files=None, 106 | param_format=None, 107 | streamable=None, 108 | doc=None, 109 | output_binding=None, 110 | param_type=None, 111 | ): 112 | """ 113 | :param param_id: unique identifier for this parameter 114 | :type param_id: STRING 115 | :param label: short, human-readable label 116 | :type label: STRING 117 | :param secondary_files: If type is a file, describes files that must be 118 | included alongside the primary file(s) 119 | :type secondary_files: STRING 120 | :param param_format: If type is a file, uri to ontology of the format or exact format 121 | :type param_format: STRING 122 | :param streamable: If type is a file, true indicates that the file is read or written 123 | sequentially without seeking 124 | :type streamable: BOOLEAN 125 | :param doc: documentation 126 | :type doc: STRING 127 | :param output_binding: describes how to handle the output 128 | :type output_binding: :class:`cwlgen.CommandOutputBinding` object 129 | :param param_type: type of data assigned to the parameter corresponding to CWLType 130 | :type param_type: STRING 131 | """ 132 | Parameter.__init__( 133 | self, 134 | param_id, 135 | label, 136 | secondary_files, 137 | param_format, 138 | streamable, 139 | doc, 140 | param_type, 141 | ) 142 | self.outputBinding = output_binding 143 | 144 | 145 | class CommandLineTool(Serializable): 146 | """ 147 | Contain all informations to describe a CWL command line tool. 148 | """ 149 | 150 | __CLASS__ = "CommandLineTool" 151 | 152 | required_fields = ["inputs", "outputs"] 153 | parse_types = {'inputs': [[CommandInputParameter]], "outputs": [[CommandOutputParameter]]} 154 | ignore_fields_on_parse = ["namespaces", "class", "requirements"] 155 | ignore_fields_on_convert = ["namespaces", "class", "metadata", "requirements"] 156 | 157 | def __init__( 158 | self, 159 | tool_id=None, 160 | base_command=None, 161 | label=None, 162 | doc=None, 163 | cwl_version="v1.0", 164 | stdin=None, 165 | stderr=None, 166 | stdout=None, 167 | path=None, 168 | requirements=None, 169 | hints=None, 170 | inputs=None, 171 | outputs=None, 172 | arguments=None, 173 | ): 174 | """ 175 | :param tool_id: Unique identifier for this tool 176 | :type tool_id: str 177 | :param base_command: command line for the tool 178 | :type base_command: str | list[STRING] 179 | :param label: label of this tool 180 | :type label: str 181 | :param doc: documentation for the tool, usually longer than the label 182 | :type doc: str 183 | :param cwl_version: version of the CWL tool 184 | :type cwl_version: str 185 | :param stdin: path to a file whose contents must be piped into stdin 186 | :type stdin: str 187 | :param stderr: capture stderr into the given file 188 | :type stderr: str 189 | :param stdout: capture stdout into the given file 190 | :type stdout: str 191 | 192 | inputs (:class:`cwlgen.CommandInputParameter` objects), 193 | outputs (:class:`cwlgen.CommandOutputParameter` objects), 194 | arguments (:class:`cwlgen.CommandLineBinding` objects), 195 | hints (any | :class:`cwlgen.Requirement` objects) 196 | and requirements (:class:`cwlgen.Requirement` objects) 197 | are stored in lists which are initialized empty. 198 | """ 199 | if cwl_version not in CWL_VERSIONS: 200 | _LOGGER.warning( 201 | "CWL version {} is not recognized as a valid version.".format( 202 | cwl_version 203 | ) 204 | ) 205 | _LOGGER.warning("CWL version is set up to {}.".format(DEF_VERSION)) 206 | cwl_version = DEF_VERSION 207 | self.cwlVersion = cwl_version 208 | self.id = tool_id 209 | self.label = label 210 | self.requirements = value_or_default(requirements, []) # List of objects inheriting from [Requirement] 211 | self.hints = value_or_default(hints, []) # List of objects inheriting from [Requirement] 212 | self.inputs = value_or_default(inputs, []) # List of [CommandInputParameter] objects 213 | self.outputs = value_or_default(outputs, []) # List of [CommandOutputParameter] objects 214 | self.baseCommand = base_command 215 | self.arguments = value_or_default(arguments, []) # List of [CommandLineBinding] objects 216 | self.doc = doc 217 | self.stdin = stdin 218 | self.stderr = stderr 219 | self.stdout = stdout 220 | self.successCodes = [] 221 | self.temporaryFailCodes = [] 222 | self.permanentFailCodes = [] 223 | self._path = path 224 | self.namespaces = Namespaces() 225 | self.metadata = {} 226 | 227 | def get_dict(self): 228 | 229 | d = super(CommandLineTool, self).get_dict() 230 | 231 | d["class"] = self.__CLASS__ 232 | 233 | if self.metadata: 234 | for key, value in self.metadata.__dict__.items(): 235 | d["s:" + key] = value 236 | # - Add Namespaces 237 | d[self.namespaces.name] = {} 238 | for k, v in self.namespaces.__dict__.items(): 239 | if "$" not in v: 240 | d[self.namespaces.name][k] = v 241 | 242 | if "inputs" not in d: 243 | # Tool can have no inputs but still needs to be bound 244 | d["inputs"] = {} 245 | if "outputs" not in d: 246 | d["outputs"] = {} 247 | 248 | if self.requirements: 249 | d["requirements"] = {r.get_class(): r.get_dict() for r in self.requirements} 250 | if self.hints: 251 | d["hints"] = {r.get_class(): r.get_dict() for r in self.hints} 252 | 253 | return d 254 | 255 | @classmethod 256 | def parse_dict(cls, d): 257 | clt = super(CommandLineTool, cls).parse_dict(d) 258 | 259 | reqs = d.get("requirements") 260 | if reqs: 261 | if isinstance(reqs, list): 262 | clt.requirements = [Requirement.parse_dict(r) for r in reqs] 263 | elif isinstance(reqs, dict): 264 | # splat operator here would be so nice {**r, "class": c} 265 | clt.requirements = [] 266 | for c, r in reqs.items(): 267 | rdict = {'class': c} 268 | rdict.update(r) 269 | clt.requirements.append(Requirement.parse_dict(rdict)) 270 | 271 | hnts = d.get("hints") 272 | if hnts: 273 | if isinstance(hnts, list): 274 | clt.hints = [Requirement.parse_dict(r) for r in hnts] 275 | elif isinstance(hnts, dict): 276 | # splat operator here would be so nice {**r, "class": c} 277 | clt.hints = [] 278 | for c, r in hnts.items(): 279 | rdict = {'class': c} 280 | rdict.update(r) 281 | clt.hints.append(Requirement.parse_dict(rdict)) 282 | 283 | return clt 284 | 285 | def export_string(self): 286 | ruamel.yaml.add_representer(literal, literal_presenter) 287 | cwl_tool = self.get_dict() 288 | return ruamel.yaml.dump(cwl_tool, default_flow_style=False) 289 | 290 | def export(self, outfile=None): 291 | """ 292 | Export the tool in CWL either on STDOUT or in outfile. 293 | """ 294 | rep = self.export_string() 295 | 296 | # Write CWL file in YAML 297 | if outfile is None: 298 | six.print_(CWL_SHEBANG, "\n", sep="") 299 | six.print_(rep) 300 | else: 301 | out_write = open(outfile, "w") 302 | out_write.write(CWL_SHEBANG + "\n\n") 303 | out_write.write(rep) 304 | out_write.close() 305 | -------------------------------------------------------------------------------- /cwlgen/common.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .utils import Serializable 4 | 5 | logging.basicConfig(level=logging.INFO) 6 | _LOGGER = logging.getLogger(__name__) 7 | 8 | # Constant(s) ------------------------------ 9 | 10 | CWL_SHEBANG = "#!/usr/bin/env cwl-runner" 11 | CWL_VERSIONS = ['draft-2', 'draft-3.dev1', 'draft-3.dev2', 'draft-3.dev3', 12 | 'draft-3.dev4', 'draft-3.dev5', 'draft-3', 'draft-4.dev1', 13 | 'draft-4.dev2', 'draft-4.dev3', 'v1.0.dev4', 'v1.0'] 14 | DEF_VERSION = 'v1.0' 15 | 16 | 17 | # Function(s) ------------------------------ 18 | 19 | def parse_type(param_type, requires_type=False): 20 | """ 21 | Parses the parameter type as one of the required types: 22 | :param requires_type: 23 | :: https://www.commonwl.org/v1.0/CommandLineTool.html#CommandInputParameter 24 | 25 | :param param_type: a CWL type that is _validated_ 26 | :type param_type: CWLType | CommandInputRecordSchema | CommandInputEnumSchema | CommandInputArraySchema | string | 27 | array 28 | :return: CWLType | CommandInputRecordSchema | CommandInputEnumSchema | CommandInputArraySchema | string | 29 | array 30 | """ 31 | 32 | if requires_type is False and param_type is None: 33 | return None 34 | 35 | if isinstance(param_type, str) and len(param_type) > 0: 36 | # Must be CWLType 37 | optional = param_type[-1] == "?" 38 | if optional: 39 | _LOGGER.debug("Detected {param_type} to be optional".format(param_type=param_type)) 40 | cwltype = param_type[:-1] 41 | else: 42 | cwltype = param_type 43 | # cwltype = param_type[:-1] if optional else param_type 44 | 45 | # check for arrays 46 | if len(cwltype) > 2 and cwltype[-2:] == "[]": 47 | array_type = CommandInputArraySchema(items=cwltype[:-2]) 48 | # How to make arrays optional input: https://www.biostars.org/p/233562/#234089 49 | return [CwlTypes.DEF_TYPE, array_type] if optional else array_type 50 | 51 | if cwltype not in CwlTypes.TYPES: 52 | _LOGGER.warning("The type '{param_type}' is not a valid CWLType, expected one of: {types}" 53 | .format(param_type=param_type, types=", ".join(str(x) for x in CwlTypes.TYPES))) 54 | _LOGGER.warning("type is set to {}.".format(CwlTypes.DEF_TYPE)) 55 | return CwlTypes.DEF_TYPE 56 | return param_type 57 | 58 | elif isinstance(param_type, list): 59 | return [parse_type(p) for p in param_type] 60 | 61 | elif isinstance(param_type, CommandInputArraySchema) \ 62 | or isinstance(param_type, CommandInputRecordSchema) \ 63 | or isinstance(param_type, CommandInputEnumSchema): 64 | return param_type # validate if required 65 | 66 | if requires_type is True: 67 | raise Exception("'parse_type' was required but failed to parse '{ptype}', exiting") 68 | 69 | _LOGGER.warning("Unable to detect type of param '{param_type}'".format(param_type=param_type)) 70 | return CwlTypes.DEF_TYPE 71 | 72 | 73 | def get_type_dict(param_type): 74 | """ 75 | Generic method to the get dict for any of the valid param_type types, 76 | ie: CWLType | InputRecordSchema | InputEnumSchema | InputArraySchema | string 77 | | array 78 | :param param_type: 79 | :type param_type: CWLType | InputRecordSchema | InputEnumSchema | InputArraySchema | string | 80 | array 81 | :return: str | dict 82 | """ 83 | if isinstance(param_type, str): 84 | return param_type 85 | elif isinstance(param_type, list): 86 | return [get_type_dict(p) for p in param_type] 87 | elif isinstance(param_type, dict): 88 | return param_type 89 | elif getattr(param_type, 'get_dict', None) and callable(getattr(param_type, 'get_dict', None)): 90 | return param_type.get_dict() 91 | else: 92 | raise Exception("Could not convert '{param_type}' to dictionary as it was unrecognised" 93 | .format(param_type=type(param_type))) 94 | 95 | 96 | # Class(es) ------------------------------ 97 | 98 | class CwlTypes: 99 | DEF_TYPE = "null" 100 | 101 | NULL = "null" 102 | BOOLEAN = "boolean" 103 | INT = "int" 104 | LONG = "long" 105 | FLOAT = "float" 106 | DOUBLE = "double" 107 | STRING = "string" 108 | FILE = "File" 109 | DIRECTORY = "Directory" 110 | STDOUT = "stdout" 111 | STDERR = "stderr" 112 | ARRAY = "array" 113 | 114 | NON_NULL_TYPES = [BOOLEAN, INT, LONG, FLOAT, DOUBLE, STRING, FILE, DIRECTORY, STDOUT, STDERR] 115 | TYPES = [NULL, None, BOOLEAN, INT, LONG, FLOAT, DOUBLE, STRING, FILE, DIRECTORY, STDOUT, STDERR] 116 | 117 | 118 | # functions 119 | 120 | class Namespaces(Serializable): 121 | """ 122 | Define different namespace for the description. 123 | """ 124 | 125 | def __init__(self): 126 | """ 127 | """ 128 | self.name = "$namespaces" 129 | self.s = "http://schema.org/" 130 | 131 | 132 | class Metadata(Serializable): 133 | """ 134 | Represent metadata described by http://schema.org. 135 | """ 136 | 137 | def __init__(self, **kwargs): 138 | for key, value in kwargs.items(): 139 | setattr(self, key, value) 140 | 141 | 142 | class Parameter(Serializable): 143 | ''' 144 | Based class for parameters (common field of Input and Output) for CommandLineTool and Workflow 145 | ''' 146 | 147 | def __init__(self, param_id, label=None, secondary_files=None, param_format=None, 148 | streamable=None, doc=None, param_type=None, requires_type=False): 149 | ''' 150 | :param param_id: unique identifier for this parameter 151 | :type param_id: STRING 152 | :param label: short, human-readable label 153 | :type label: STRING 154 | :param secondary_files: If type is a file, describes files that must be 155 | included alongside the primary file(s) 156 | :type secondary_files: STRING 157 | :param param_format: If type is a file, uri to ontology of the format or exact format 158 | :type param_format: STRING 159 | :param streamable: If type is a file, true indicates that the file is read or written 160 | sequentially without seeking 161 | :type streamable: BOOLEAN 162 | :param doc: documentation 163 | :type doc: STRING 164 | :param param_type: type of data assigned to the parameter 165 | :type param_type: STRING corresponding to CWLType 166 | ''' 167 | 168 | self.id = param_id 169 | self.label = label 170 | self.secondaryFiles = secondary_files 171 | self.format = param_format 172 | self.streamable = streamable 173 | self.doc = doc 174 | self.type = parse_type(param_type, requires_type) 175 | 176 | @classmethod 177 | def parse_with_id(cls, d, identifier): 178 | d["id"] = identifier 179 | return super(Parameter, cls).parse_dict(d) 180 | 181 | # @classmethod 182 | # def parse_dict(cls, d): 183 | # self = super(Parameter, cls).parse_dict(d) 184 | # secs = d.get("secondaryFiles") 185 | # self.secondaryFiles = [] 186 | # if secs: 187 | # self.secondaryFiles = secs if isinstance(secs, list) else [secs] 188 | # return self 189 | 190 | 191 | class CommandInputArraySchema(Serializable): 192 | ''' 193 | Based on the parameter set out in the CWL spec: 194 | https://www.commonwl.org/v1.0/CommandLineTool.html#CommandInputArraySchema 195 | ''' 196 | 197 | def __init__(self, items=None, label=None, input_binding=None): 198 | ''' 199 | :param items: Defines the type of the array elements. 200 | :type: `CWLType | CommandInputRecordSchema | CommandInputEnumSchema | CommandInputArraySchema | string | array` 201 | :param label: A short, human-readable label of this object. 202 | :type label: STRING 203 | :param input_binding: 204 | :type input_binding: CommandLineBinding 205 | ''' 206 | self.type = CwlTypes.ARRAY 207 | self.items = parse_type(items, requires_type=True) 208 | self.label = label 209 | self.inputBinding = input_binding 210 | 211 | 212 | class CommandInputRecordSchema(Serializable): 213 | """ 214 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#CommandInputRecordSchema 215 | """ 216 | 217 | def __init__(self, label=None, name=None): 218 | """ 219 | :param fields: Defines the fields of the record. 220 | :type fields: array 221 | :param label: A short, human-readable label of this object. 222 | :param name: NF (Name of the InputRecord) 223 | """ 224 | self.fields = [] 225 | self.label = label 226 | self.name = name 227 | self.type = "record" 228 | 229 | class CommandInputRecordField(Serializable): 230 | """ 231 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#CommandInputRecordField 232 | """ 233 | 234 | def __init__(self, name, input_type, doc=None, input_binding=None, label=None): 235 | """ 236 | :param name: 237 | :param input_type: 238 | :type input_type: CWLType | InputRecordSchema | InputEnumSchema | InputArraySchema | string | 239 | array 240 | :param doc: A documentation string for this field 241 | :param input_binding: 242 | :type input_binding: CommandLineBinding 243 | :param label: 244 | """ 245 | self.name = name 246 | self.type = parse_type(input_type, requires_type=True) 247 | self.doc = doc 248 | self.inputBinding = input_binding 249 | self.label = label 250 | 251 | def get_dict(self): 252 | d = super(type(self), self).get_dict() 253 | d["type"] = get_type_dict(self.type) 254 | return d 255 | 256 | 257 | class CommandInputEnumSchema(Serializable): 258 | """ 259 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#CommandInputEnumSchema 260 | """ 261 | 262 | def __init__(self, symbols, label=None, name=None, input_binding=None): 263 | """ 264 | :param symbols: Defines the set of valid symbols. 265 | :type symbols: List[str] 266 | :param label: A short, human-readable label of this object. 267 | :type label: str 268 | :param name: 269 | :type name: str 270 | :param input_binding: 271 | :type input_binding: CommandLineBinding 272 | """ 273 | self.type = "enum" 274 | self.symbols = symbols 275 | self.label = label 276 | self.name = name 277 | self.inputBinding = input_binding 278 | -------------------------------------------------------------------------------- /cwlgen/import_cwl.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Library to handle the manipulation and generation of CWL tool 3 | ''' 4 | 5 | # Import ------------------------------ 6 | 7 | # General libraries 8 | import os 9 | import six 10 | import logging 11 | 12 | # External libraries 13 | import ruamel.yaml as ryaml 14 | import cwlgen 15 | 16 | logging.basicConfig(level=logging.INFO) 17 | _LOGGER = logging.getLogger(__name__) 18 | 19 | # Class(es) ------------------------------ 20 | 21 | 22 | def parse_cwl(cwl_path): 23 | """ 24 | Method that parses a CWL file and will a 25 | :class:`cwlgen.Workflow` or :class:`cwlgen.CommandLineTool`. 26 | Note: this will not import additional files. 27 | 28 | :param cwl_path: PATH to the CWL file 29 | :type cwl_path: str 30 | :return: :class:`cwlgen.Workflow` | :class:`cwlgen.CommandLineTool` 31 | """ 32 | 33 | with open(cwl_path) as yaml_file: 34 | cwl_dict = ryaml.load(yaml_file, Loader=ryaml.Loader) 35 | return parse_cwl_dict(cwl_dict) 36 | 37 | 38 | def parse_cwl_string(cwlstr): 39 | cwl_dict = ryaml.load(cwlstr, Loader=ryaml.Loader) 40 | return parse_cwl_dict(cwl_dict) 41 | 42 | 43 | def parse_cwl_dict(cwl_dict): 44 | """ 45 | Method that parses a dictionary and will return a 46 | :class:`cwlgen.Workflow` or :class:`cwlgen.CommandLineTool`. 47 | 48 | :param cwl_dict: The dictionary to pass, must contain a 'class' field. 49 | :type cwl_dict: :class:`dict` 50 | :return: :class:`cwlgen.Workflow` | :class:`cwlgen.CommandLineTool` 51 | """ 52 | cl = cwl_dict.get("class") 53 | 54 | if cl == "CommandLineTool": 55 | return cwlgen.CommandLineTool.parse_dict(cwl_dict) 56 | elif cl == "Workflow": 57 | return cwlgen.Workflow.parse_dict(cwl_dict) 58 | 59 | raise NotImplementedError("The CWL class '" + str(cl) + "' was not a recognised CWL class") 60 | -------------------------------------------------------------------------------- /cwlgen/requirements.py: -------------------------------------------------------------------------------- 1 | import six 2 | from cwlgen.commandlinebinding import CommandLineBinding 3 | 4 | from .common import parse_type, get_type_dict 5 | from .utils import Serializable 6 | 7 | 8 | class Requirement(Serializable): 9 | 10 | ignore_fields_on_parse = ["class"] 11 | 12 | ''' 13 | Requirement that must be met in order to execute the process. 14 | ''' 15 | def __init__(self, req_class): 16 | ''' 17 | :param req_class: requirement class 18 | :type req_class: STRING 19 | ''' 20 | # class is protected keyword 21 | self._req_class = req_class 22 | 23 | def get_class(self): 24 | return self._req_class 25 | 26 | def __hash__(self): 27 | return hash(self.get_class()) 28 | 29 | @classmethod 30 | def parse_dict(cls, d): 31 | 32 | c = d["class"] 33 | requirements = [ 34 | InlineJavascriptRequirement, SchemaDefRequirement, SoftwareRequirement, InitialWorkDirRequirement, 35 | SubworkflowFeatureRequirement, ScatterFeatureRequirement, MultipleInputFeatureRequirement, 36 | StepInputExpressionRequirement, DockerRequirement, EnvVarRequirement, ShellCommandRequirement, 37 | ResourceRequirement 38 | ] 39 | 40 | for Req in requirements: 41 | if Req.__name__ == c: 42 | return Req.parse_dict_generic(Req, d) 43 | 44 | return None 45 | 46 | 47 | class InlineJavascriptRequirement(Requirement): 48 | """ 49 | Indicates that the workflow platform must support inline Javascript expressions. 50 | If this requirement is not present, the workflow platform must not perform expression interpolatation. 51 | 52 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#InlineJavascriptRequirement 53 | """ 54 | 55 | def __init__(self, expression_lib=None): 56 | ''' 57 | :param expression_lib: List of Strings 58 | :type expression_lib: list[STRING] 59 | ''' 60 | Requirement.__init__(self, 'InlineJavascriptRequirement') 61 | self.expressionLib = [expression_lib] if isinstance(expression_lib, six.string_types) else expression_lib 62 | 63 | 64 | class SchemaDefRequirement(Requirement): 65 | """ 66 | This field consists of an array of type definitions which must be used when interpreting the inputs and 67 | outputs fields. When a type field contain a IRI, the implementation must check if the type is defined 68 | in schemaDefs and use that definition. If the type is not found in schemaDefs, it is an error. 69 | The entries in schemaDefs must be processed in the order listed such that later schema definitions 70 | may refer to earlier schema definitions. 71 | 72 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#SchemaDefRequirement 73 | """ 74 | 75 | def __init__(self, types): 76 | """ 77 | :param types: The list of type definitions. 78 | :type types: list[InputRecordSchema | InputEnumSchema | InputArraySchema] 79 | """ 80 | Requirement.__init__(self, "SchemaDefRequirement") 81 | self.types = types 82 | 83 | class InputRecordSchema(Serializable): 84 | """ 85 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#InputRecordSchema 86 | """ 87 | def __init__(self, label=None, name=None): 88 | """ 89 | :param fields: Defines the fields of the record. 90 | :type fields: array 91 | :param label: A short, human-readable label of this object. 92 | :param name: NF (Name of the InputRecord) 93 | """ 94 | self.fields = [] 95 | self.label = label 96 | self.name = name 97 | self.type = "record" 98 | 99 | def parse_dict(cls, d): 100 | if d["type"] != "record": 101 | return None 102 | return cls.parse_dict_generic(cls, d) 103 | 104 | class InputRecordField(Serializable): 105 | """ 106 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#InputRecordField 107 | """ 108 | def __init__(self, name, type, doc=None, input_binding=None, label=None): 109 | """ 110 | :param name: 111 | :param input_type: 112 | :type input_type: CWLType | InputRecordSchema | InputEnumSchema | InputArraySchema | string | 113 | array 114 | :param doc: A documentation string for this field 115 | :param input_binding: 116 | :type input_binding: CommandLineBinding 117 | :param label: 118 | """ 119 | self.name = name 120 | self.type = parse_type(type, requires_type=True) 121 | self.doc = doc 122 | self.inputBinding = input_binding 123 | self.label = label 124 | 125 | def get_dict(self): 126 | d = super(self, self).get_dict() 127 | d["type"] = get_type_dict(self.type) 128 | return d 129 | 130 | # def parse_dict(cls, d): 131 | # d["input_type"] = "string" 132 | # ret = super(SchemaDefRequirement.InputRecordSchema.InputRecordField, cls).parse_dict(d) 133 | # ret.type = 134 | 135 | 136 | # ignore_fields_on_parse = "type" 137 | parse_types = { 138 | "fields": [InputRecordField], 139 | "inputBinding": [CommandLineBinding] 140 | } 141 | 142 | class InputEnumSchema(Serializable): 143 | """ 144 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#InputEnumSchema 145 | """ 146 | def __init__(self, symbols, label=None, name=None, input_binding=None): 147 | """ 148 | :param symbols: Defines the set of valid symbols. 149 | :type symbols: list[STRING] 150 | :param label: A short, human-readable label of this object. 151 | :type label: STRING 152 | :param name: 153 | :type name: STRING 154 | :param input_binding: 155 | :type input_binding: CommandLineBinding 156 | """ 157 | self.type = "enum" 158 | self.symbols = symbols 159 | self.label = label 160 | self.name = name 161 | self.inputBinding = input_binding 162 | 163 | def parse_dict(cls, d): 164 | if d["type"] != "enum": 165 | return None 166 | return cls.parse_dict_generic(cls, d) 167 | 168 | parse_types = { 169 | "inputBinding": CommandLineBinding 170 | } 171 | 172 | class InputArraySchema(Serializable): 173 | """ 174 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#InputArraySchema 175 | """ 176 | 177 | def __init__(self, items, label=None, input_binding=None): 178 | """ 179 | :param items: Defines the type of the array elements. 180 | :type items: CWLType | InputRecordSchema | InputEnumSchema | InputArraySchema | string | 181 | array 182 | :param label: A short, human-readable label of this object. 183 | :type label: STRING 184 | :param input_binding: 185 | :type input_binding: CommandLineBinding 186 | """ 187 | self.type = "array" 188 | self.items = items 189 | self.label = label 190 | self.inputBinding = input_binding 191 | 192 | def parse_dict(cls, d): 193 | if d["type"] != "record": 194 | return None 195 | return cls.parse_dict_generic(cls, d) 196 | 197 | parse_types = { 198 | # Defined below because we need to have declared the class before we can use it 199 | } 200 | 201 | parse_types = { 202 | # "types": [InputRecordSchema, InputEnumSchema, InputArraySchema], 203 | } 204 | 205 | 206 | # declare this here, because inside the InputArraySchema we haven't fully defined the schemas 207 | # SchemaDefRequirement.InputArraySchema.parse_types = { 208 | # "items": [SchemaDefRequirement.InputRecordSchema, SchemaDefRequirement.InputEnumSchema, 209 | # SchemaDefRequirement.InputArraySchema, str] 210 | # } 211 | 212 | 213 | class SoftwareRequirement(Requirement): 214 | """ 215 | A list of software packages that should be configured in the environment of the defined process. 216 | 217 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#SoftwareRequirement 218 | """ 219 | def __init__(self, packages=None): 220 | Requirement.__init__(self, "SoftwareRequirement") 221 | self.packages = packages or [] # list[SoftwarePackage] 222 | 223 | class SoftwarePackage(Serializable): 224 | """ 225 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#SoftwarePackage 226 | """ 227 | def __init__(self, package, version=None, specs=None): 228 | """ 229 | :param package: The name of the software to be made available. If the name is common, inconsistent, 230 | or otherwise ambiguous it should be combined with one or more identifiers in the specs field 231 | :param version: The (optional) versions of the software that are known to be compatible. 232 | :param specs: One or more IRIs identifying resources for installing or enabling the software in 'package' 233 | """ 234 | self.package = package 235 | self.version = version 236 | self.specs = specs 237 | 238 | parse_types = { 239 | "packages": [SoftwarePackage] 240 | } 241 | 242 | 243 | class InitialWorkDirRequirement(Requirement): 244 | """ 245 | Define a list of files and subdirectories that must be created by the workflow 246 | platform in the designated output directory prior to executing the command line tool. 247 | 248 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#InitialWorkDirRequirement 249 | """ 250 | def __init__(self, listing): 251 | """ 252 | :param listing: The list of files or subdirectories that must be placed in the 253 | designated output directory prior to executing the command line tool. 254 | :type listing: array | string | Expression 255 | """ 256 | Requirement.__init__(self, "InitialWorkDirRequirement") 257 | self.listing = listing 258 | 259 | def get_dict(self): 260 | base = super(InitialWorkDirRequirement, self).get_dict() 261 | 262 | if isinstance(self.listing, str): 263 | base["listing"] = self.listing 264 | elif isinstance(self.listing, list): 265 | if len(self.listing) == 0: 266 | raise Exception("InitialWorkDirRequirement.listing must have at least one element") 267 | base["listing"] = [r if isinstance(r, str) else r.get_dict() for r in self.listing] 268 | else: 269 | raise Exception("Couldn't recognise type of '{list_type}', expected: array | string | Expression".format(list_type=type(self.listing))) 271 | 272 | return base 273 | 274 | class Dirent(Serializable): 275 | """ 276 | Define a file or subdirectory that must be placed in the designated output directory 277 | prior to executing the command line tool. May be the result of executing an expression, 278 | such as building a configuration file from a template. 279 | 280 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#Dirent 281 | """ 282 | def __init__(self, entry, entryname=None, writable=None): 283 | self.entry = entry 284 | self.entryname = entryname 285 | self.writable = writable 286 | 287 | parse_types = { 288 | "listing": [Dirent, str] 289 | } 290 | 291 | 292 | class SubworkflowFeatureRequirement(Requirement): 293 | """ 294 | Indicates that the workflow platform must support nested workflows in the run field of WorkflowStep. 295 | 296 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#SubworkflowFeatureRequirement 297 | """ 298 | 299 | def __init__(self): 300 | Requirement.__init__(self, 'SubworkflowFeatureRequirement') 301 | 302 | 303 | class ScatterFeatureRequirement(Requirement): 304 | """ 305 | Indicates that the workflow platform must support the scatter and scatterMethod fields of WorkflowStep. 306 | 307 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#ScatterFeatureRequirement 308 | """ 309 | 310 | def __init__(self): 311 | Requirement.__init__(self, 'ScatterFeatureRequirement') 312 | 313 | 314 | class MultipleInputFeatureRequirement(Requirement): 315 | """ 316 | Indicates that the workflow platform must support multiple 317 | inbound data links listed in the source field of WorkflowStepInput. 318 | 319 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#MultipleInputFeatureRequirement 320 | """ 321 | 322 | def __init__(self): 323 | Requirement.__init__(self, 'MultipleInputFeatureRequirement') 324 | 325 | 326 | class StepInputExpressionRequirement(Requirement): 327 | """ 328 | Indicate that the workflow platform must support the valueFrom field of WorkflowStepInput. 329 | 330 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#StepInputExpressionRequirement 331 | """ 332 | 333 | def __init__(self): 334 | Requirement.__init__(self, 'StepInputExpressionRequirement') 335 | 336 | 337 | class DockerRequirement(Requirement): 338 | """ 339 | Indicates that a workflow component should be run in a Docker container, 340 | and specifies how to fetch or build the image. 341 | 342 | Documentation: https://www.commonwl.org/v1.0/CommandLineTool.html#DockerRequirement 343 | """ 344 | 345 | def __init__(self, docker_pull=None, docker_load=None, docker_file=None, 346 | docker_import=None, docker_image_id=None, docker_output_dir=None): 347 | """ 348 | :param docker_pull: image to retrive with docker pull 349 | :type docker_pull: STRING 350 | :param docker_load: HTTP URL from which to download Docker image 351 | :type docker_load: STRING 352 | :param docker_file: supply the contents of a Dockerfile 353 | :type docker_file: STRING 354 | :param docker_import: HTTP URL to download and gunzip a Docker images 355 | :type docker_import: STRING 356 | :param docker_image_id: Image id for docker run 357 | :type docker_image_id: STRING 358 | :param docker_output_dir: designated output dir inside the Docker container 359 | :type docker_output_dir: STRING 360 | """ 361 | Requirement.__init__(self, 'DockerRequirement') 362 | self.dockerPull = docker_pull 363 | self.dockerLoad = docker_load 364 | self.dockerFile = docker_file 365 | self.dockerImport = docker_import 366 | self.dockerImageId = docker_image_id 367 | self.dockerOutputDirectory = docker_output_dir 368 | 369 | 370 | class EnvVarRequirement(Requirement): 371 | """ 372 | Define a list of environment variables which will be set in the execution environment of the tool. 373 | See EnvironmentDef for details. 374 | 375 | Documentation: https://www.commonwl.org/v1.0/CommandLineTool.html#EnvVarRequirement 376 | """ 377 | def __init__(self, env_def): 378 | """ 379 | :param env_def: The list of environment variables. 380 | :type env_def: list[EnvironmentDef] 381 | """ 382 | Requirement.__init__(self, 'EnvVarRequirement') 383 | self.envDef = env_def 384 | 385 | class EnvironmentDef(Serializable): 386 | """ 387 | Define an environment variable that will be set in the runtime environment 388 | by the workflow platform when executing the command line tool. 389 | May be the result of executing an expression, such as getting a parameter from input. 390 | 391 | Documentation: https://www.commonwl.org/v1.0/CommandLineTool.html#EnvironmentDef 392 | """ 393 | def __init__(self, env_name, env_value): 394 | """ 395 | :param env_name: The environment variable name 396 | :type env_name: STRING 397 | :param env_value: The environment variable value 398 | :type env_value: STRING 399 | """ 400 | self.envName = env_name 401 | self.envValue = env_value 402 | 403 | 404 | class ShellCommandRequirement(Requirement): 405 | """ 406 | Modify the behavior of CommandLineTool to generate a single string containing a shell command line. 407 | 408 | Documentation: https://www.commonwl.org/v1.0/CommandLineTool.html#ShellCommandRequirement 409 | """ 410 | 411 | def __init__(self): 412 | Requirement.__init__(self, 'ShellCommandRequirement') 413 | 414 | 415 | class ResourceRequirement(Requirement): 416 | """ 417 | Specify basic hardware resource requirements. 418 | 419 | Documentation: https://www.commonwl.org/v1.0/CommandLineTool.html#ResourceRequirement 420 | """ 421 | 422 | def __init__(self, cores_min=None, cores_max=None, ram_min=None, ram_max=None, tmpdir_min=None, tmpdir_max=None, 423 | outdir_min=None, outdir_max=None): 424 | """ 425 | :param cores_min: Minimum reserved number of CPU cores 426 | :type cores_min: string | float | None 427 | :param cores_max: Maximum reserved number of CPU cores 428 | :type cores_max: string | float | None 429 | :param ram_min: Minimum reserved RAM in mebibytes (2**20) 430 | :type ram_min: string | float | None 431 | :param ram_max: Maximum reserved RAM in mebibytes (2**20) 432 | :type ram_max: string | float | None 433 | :param tmpdir_min: Minimum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) 434 | :type tmpdir_min: string | float | None 435 | :param tmpdir_max: Maximum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) 436 | :type tmpdir_max: string | float | None 437 | :param outdir_min: Minimum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) 438 | :type outdir_min: string | float | None 439 | :param outdir_max: Maximum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) 440 | :type outdir_max: string | float | None 441 | """ 442 | Requirement.__init__(self, 'ResourceRequirement') 443 | 444 | self.coresMin = cores_min 445 | self.coresMax = cores_max 446 | self.ramMin = ram_min 447 | self.ramMax = ram_max 448 | self.tmpdirMin = tmpdir_min 449 | self.tmpdirMax = tmpdir_max 450 | self.outdirMin = outdir_min 451 | self.outdirMax = outdir_max 452 | -------------------------------------------------------------------------------- /cwlgen/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Set of util functions and classes 3 | """ 4 | import inspect 5 | 6 | class literal(str): pass 7 | 8 | _unparseable_types = [str, int, float, bool] 9 | 10 | 11 | def literal_presenter(dumper, data): 12 | return dumper.represent_scalar('tag:yaml.org,2002:str', data, style="|") 13 | 14 | 15 | class Serializable(object): 16 | """ 17 | The Serializable class contains logic to automatically serialize a class based on 18 | its attributes. This behaviour can be overridden via the ``get_dict`` method on its 19 | subclasses with a call to super. Fields can be ignored by the base converter through 20 | the ``ignore_field_on_convert`` static attribute on your subclass. 21 | 22 | The parsing behaviour (beta) is similar, however it will attempt to set all attributes 23 | from the dictionary onto a newly initialised class. If your initialiser has required 24 | arguments, this converter will do its best to pull the id out of the dictionary to provide 25 | to your initializer (or pull it from the { $id: value } dictionary). Typing hints can be 26 | provided by the ``parse_types`` static attribute, and required attributes can be tagged 27 | the ``required_fields`` attribute. 28 | """ 29 | 30 | 31 | """ 32 | This is a special field, with format: {fieldName: str, [Serializable]} 33 | If the field name is present in the dict, then it will call the parse_dict(cls, d) 34 | method on that type. It should return None if it can't parse that dictionary. This 35 | means the type will need to override the ``parse_dict`` method. 36 | """ 37 | parse_types = {} # type: {str, [type]} 38 | ignore_fields_on_parse = [] 39 | ignore_fields_on_convert = [] 40 | required_fields = [] # type: str 41 | 42 | @staticmethod 43 | def serialize(obj): 44 | if isinstance(obj, str) or isinstance(obj, int) or isinstance(obj, float) or isinstance(obj, bool): 45 | return obj 46 | if isinstance(obj, list): 47 | return [Serializable.serialize(x) for x in obj] 48 | if isinstance(obj, dict): 49 | return {k: Serializable.serialize(v) for k, v in obj.items() if v is not None} 50 | if callable(getattr(obj, "get_dict", None)): 51 | return obj.get_dict() 52 | if obj is None: 53 | return None # some types allow None as value, such as default so we should explicitly allow it 54 | raise Exception("Can't serialize '{unsupported_type}'".format(unsupported_type=type(obj))) 55 | 56 | @staticmethod 57 | def should_exclude_object(value): 58 | return value is None or ((isinstance(value, list) or isinstance(value, dict)) and len(value) == 0) 59 | 60 | def get_dict(self): 61 | d = {} 62 | ignore_attributes = set() 63 | req_fields = set(self.required_fields or []) 64 | 65 | if hasattr(self, "ignore_attributes") and self.ignore_attributes: 66 | ignore_attributes = set(self.ignore_attributes) 67 | 68 | if self.ignore_fields_on_convert: 69 | ignore_attributes = ignore_attributes.union(self.ignore_fields_on_convert) 70 | 71 | for k, v in vars(self).items(): 72 | is_required = k in req_fields 73 | should_skip = ( 74 | self.should_exclude_object(v) 75 | or k.startswith("_") 76 | or k in ignore_attributes 77 | or k == "ignore_attributes" 78 | ) 79 | if not is_required and should_skip: 80 | continue 81 | s = self.serialize(v) 82 | if self.should_exclude_object(s): 83 | continue 84 | d[k] = s 85 | return d 86 | 87 | @classmethod 88 | def parse_dict(cls, d): 89 | return cls.parse_dict_generic(cls, d) 90 | 91 | @staticmethod 92 | def parse_dict_generic(T, d, parse_types=None, required_fields=None, ignore_fields_on_parse=None): 93 | 94 | if parse_types is None and hasattr(T, "parse_types"): 95 | parse_types = T.parse_types 96 | if required_fields is None and hasattr(T, "required_fields"): 97 | required_fields = T.required_fields 98 | if ignore_fields_on_parse is None and hasattr(T, "ignore_fields_on_parse"): 99 | ignore_fields_on_parse = T.ignore_fields_on_parse 100 | 101 | pts = parse_types 102 | req = {r: False for r in required_fields} 103 | ignore = set(ignore_fields_on_parse) 104 | 105 | # may not be able to just initialise blank class 106 | # but we can use inspect to get required params and init using **kwargs 107 | try: 108 | required_init_kwargs = T.get_required_input_params_for_cls(T, d) 109 | self = T(**required_init_kwargs) 110 | except Exception as e: 111 | return None 112 | 113 | for k, v in d.items(): 114 | if k in ignore: continue 115 | val = T.try_parse(v, pts.get(k)) 116 | if val is None: continue 117 | if not hasattr(self, k): 118 | raise KeyError("Key '%s' does not exist on type '%s'" % (k, type(self))) 119 | self.__setattr__(k, val) 120 | req[k] = True 121 | 122 | if not all(req.values()): 123 | # There was a required field that wasn't mapped 124 | req_fields = ", ".join(r for r in req if not req[r]) 125 | clsname = T.__name__ 126 | 127 | raise Exception("The fields %s were not found when parsing type '%s'" % (req_fields, clsname)) 128 | 129 | return self 130 | 131 | @classmethod 132 | def parse_with_id(cls, d, identifier): 133 | if not isinstance(d, dict): 134 | raise Exception("parse_with_id will require override to handle default object of type '%s'" % type(d)) 135 | d["id"] = identifier 136 | return d 137 | 138 | @staticmethod 139 | def get_required_input_params_for_cls(cls, valuesdict): 140 | try: 141 | argspec = inspect.getfullargspec(cls.__init__) 142 | except: 143 | # we're in Python 2 144 | argspec = inspect.getargspec(cls.__init__) 145 | 146 | args, defaults = argspec.args, argspec.defaults 147 | required_param_keys = set(args[1:-len(defaults)]) if defaults is not None and len(defaults) > 0 else args[1:] 148 | 149 | inspect_ignore_keys = {"self", "args", "kwargs"} 150 | # Params can't shadow the built in 'id', so we'll put in a little hack 151 | # to guess the required param name that ends in 152 | 153 | id_field_names = [k for k in required_param_keys if k == "id" or k.endswith("_id")] 154 | id_field_name = None 155 | id_field_value = valuesdict.get("id") 156 | 157 | if len(id_field_names) == 1: 158 | id_field_name = id_field_names[0] 159 | inspect_ignore_keys.add(id_field_name) 160 | elif len(id_field_names) > 1: 161 | print("Warning, can't determine if there are multiple id fieldnames") 162 | 163 | required_init_kwargs = {k: valuesdict[k] for k in required_param_keys if (k not in inspect_ignore_keys)} 164 | if id_field_name: 165 | required_init_kwargs[id_field_name] = id_field_value 166 | 167 | return required_init_kwargs 168 | 169 | @staticmethod 170 | def try_parse(value, types): 171 | if types is None: return value 172 | if isinstance(value, (dict, list)) and len(value) == 0: return [] 173 | 174 | # If it's an array, we should call try_parse (recursively) 175 | 176 | if isinstance(value, list): 177 | retval = [Serializable.try_parse(t, types) for t in value] 178 | invalid_values = get_indices_of_element_in_list([False if v is None else True for v in retval], False) 179 | if invalid_values: 180 | invalid_valuesstr = ','.join(str(i) for i in invalid_values) 181 | invalid_itemstr = ", ".join([str(value[i]) for i in invalid_values]) 182 | raise Exception("Couldn't parse items at indices " + invalid_valuesstr + ", corresponding to: " + invalid_itemstr) 183 | return retval 184 | 185 | for T in types: 186 | retval = Serializable.try_parse_type(value, T) 187 | if retval: 188 | return retval 189 | 190 | return 191 | 192 | 193 | @staticmethod 194 | def try_parse_type(value, T): 195 | # We're all good, don't need to do anything 196 | if not isinstance(T, list) and isinstance(value, T): return value 197 | 198 | # if T is a primitive (str, bool, int, float), just return the T representation of retval 199 | elif T in _unparseable_types: 200 | try: 201 | if isinstance(value, list): 202 | return [T(v) for v in value] 203 | return T(value) 204 | except: 205 | return None 206 | 207 | # the type is [T] which is our our indicator that T will be a (list | dictionary) (with key 'id') 208 | elif isinstance(T, list): 209 | T = T[0] 210 | 211 | if T in _unparseable_types: 212 | try: 213 | if isinstance(value, list): 214 | return [T(v) for v in value] 215 | return T(value) 216 | except: 217 | return None 218 | elif isinstance(value, list): 219 | return [T.parse_dict(vv) for vv in value] 220 | elif isinstance(value, dict): 221 | # We'll need to map the 'id' back in 222 | retval = [] 223 | for nested_key in value: 224 | dd = value[nested_key] 225 | retval.append(T.parse_with_id(dd, nested_key)) 226 | return retval 227 | else: 228 | raise Exception("Don't recognise type '%s', expected dictionary or list" % type(value)) 229 | 230 | # T is the retval, or an array of the values (because some params are allowed to be both 231 | return T.parse_dict_generic(T, value) if not isinstance(value, list) else [T.parse_dict_generic(T, vv) for vv in value] 232 | 233 | 234 | def get_indices_of_element_in_list(searchable, element): 235 | indices = [] 236 | for i in range(len(searchable)): 237 | if element == searchable[i]: 238 | indices.append(i) 239 | return indices 240 | 241 | 242 | def value_or_default(value, default): 243 | return value if value is not None else default 244 | -------------------------------------------------------------------------------- /cwlgen/version.py: -------------------------------------------------------------------------------- 1 | version_info = (0, 4, 2) 2 | __version__ = ".".join(str(c) for c in version_info) 3 | -------------------------------------------------------------------------------- /cwlgen/workflow.py: -------------------------------------------------------------------------------- 1 | # Import ------------------------------ 2 | 3 | # General libraries 4 | import logging 5 | 6 | # External libraries 7 | import ruamel.yaml 8 | import six 9 | 10 | # Internal libraries 11 | 12 | from .requirements import Requirement 13 | from .utils import literal, literal_presenter, Serializable, value_or_default 14 | from .common import Parameter, CWL_SHEBANG 15 | from .workflowdeps import InputParameter, WorkflowOutputParameter, WorkflowStep 16 | 17 | 18 | # Logging setup 19 | 20 | logging.basicConfig(level=logging.INFO) 21 | _LOGGER = logging.getLogger(__name__) 22 | 23 | 24 | # Function(s) ------------------------------ 25 | 26 | 27 | # Class(es) ------------------------------ 28 | 29 | class Workflow(Serializable): 30 | """ 31 | A workflow describes a set of steps and the dependencies between those steps. 32 | When a step produces output that will be consumed by a second step, 33 | the first step is a dependency of the second step. 34 | 35 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#Workflow 36 | """ 37 | __CLASS__ = 'Workflow' 38 | required_fields = ["inputs", "outputs", "steps"] 39 | ignore_fields_on_parse = ["class", "requirements"] 40 | ignore_fields_on_convert = ["inputs", "outputs", "requirements"] 41 | parse_types = { 42 | "inputs": [[InputParameter]], 43 | "outputs": [[WorkflowOutputParameter]], 44 | "steps": [[WorkflowStep]], 45 | } 46 | 47 | def __init__(self, workflow_id=None, label=None, doc=None, cwl_version='v1.0', inputs=None, outputs=None, steps=None, requirements=None, hints=None): 48 | """ 49 | :param workflow_id: The unique identifier for this process object. 50 | :type workflow_id: STRING 51 | :param label: A short, human-readable label of this process object. 52 | :type label: STRING 53 | :param doc: A long, human-readable description of this process object. 54 | :type doc: STRING 55 | :param cwl_version: CWL document version. Always required at the document root. Default: 'v1.0' 56 | :type cwl_version: CWLVersion 57 | """ 58 | self.id = workflow_id 59 | self.label = label 60 | self.doc = doc 61 | self.cwlVersion = cwl_version 62 | 63 | self.inputs = value_or_default(inputs, []) # list[InputParameter] 64 | self.outputs = value_or_default(outputs, []) # list[WorkflowOutputParameter] 65 | self.steps = value_or_default(steps, []) # list[WorkflowStep] 66 | self.requirements = value_or_default(requirements, []) # list[Requirement] 67 | self.hints = value_or_default(hints, []) # list[Requirement] 68 | self._path = None 69 | 70 | def get_dict(self): 71 | cwl_workflow = super(Workflow, self).get_dict() 72 | 73 | cwl_workflow['class'] = self.__CLASS__ 74 | 75 | # steps, inputs, outputs are required properties, so it should fail if we can't place it 76 | cwl_workflow['steps'] = {step.id: step.get_dict() for step in self.steps} 77 | cwl_workflow['inputs'] = {i.id: i.get_dict() for i in self.inputs} 78 | cwl_workflow['outputs'] = {o.id: o.get_dict() for o in self.outputs} 79 | 80 | if self.requirements: 81 | cwl_workflow['requirements'] = {r.get_class(): r.get_dict() for r in self.requirements} 82 | if self.hints: 83 | cwl_workflow["hints"] = {r.get_class(): r.get_dict() for r in self.hints} 84 | 85 | return cwl_workflow 86 | 87 | @classmethod 88 | def parse_dict(cls, d): 89 | wf = super(Workflow, cls).parse_dict(d) 90 | 91 | reqs = d.get("requirements") 92 | if reqs: 93 | if isinstance(reqs, list): 94 | wf.requirements = [Requirement.parse_dict(r) for r in reqs] 95 | elif isinstance(reqs, dict): 96 | # splat operator here would be so nice {**r, "class": c} 97 | wf.requirements = [] 98 | for c, r in reqs.items(): 99 | rdict = {'class': c} 100 | rdict.update(r) 101 | wf.requirements.append(Requirement.parse_dict(rdict)) 102 | 103 | hnts = d.get("hints") 104 | if hnts: 105 | if isinstance(hnts, list): 106 | wf.hints = [Requirement.parse_dict(r) for r in hnts] 107 | elif isinstance(hnts, dict): 108 | # splat operator here would be so nice {**r, "class": c} 109 | wf.hints = [] 110 | for c, r in hnts.items(): 111 | rdict = {'class': c} 112 | rdict.update(r) 113 | wf.hints.append(Requirement.parse_dict(rdict)) 114 | 115 | return wf 116 | 117 | def export_string(self): 118 | ruamel.yaml.add_representer(literal, literal_presenter) 119 | cwl_tool = self.get_dict() 120 | return ruamel.yaml.dump(cwl_tool, default_flow_style=False) 121 | 122 | def export(self, outfile=None): 123 | """ 124 | Export the workflow in CWL either on STDOUT or in outfile. 125 | """ 126 | rep = self.export_string() 127 | 128 | # Write CWL file in YAML 129 | if outfile is None: 130 | six.print_(CWL_SHEBANG, "\n", sep='') 131 | six.print_(rep) 132 | else: 133 | out_write = open(outfile, 'w') 134 | out_write.write(CWL_SHEBANG + '\n\n') 135 | out_write.write(rep) 136 | out_write.close() 137 | 138 | 139 | 140 | ############################ 141 | # Workflow construction classes 142 | 143 | # class File: 144 | # """ 145 | # An abstract file reference used for generating workflows 146 | # """ 147 | # def __init__(self, path): 148 | # self.path = path 149 | # 150 | # 151 | # class Variable: 152 | # """ 153 | # An output variable from a workflow step 154 | # """ 155 | # def __init__(self, workflow, step, name): 156 | # self.step = step 157 | # self.name = name 158 | # self.workflow = workflow 159 | # 160 | # def path(self): 161 | # return "%s/%s" % (self.step, self.name) 162 | # 163 | # def store(self): 164 | # self.workflow.outputs.append( 165 | # WorkflowOutputParameter(self.path().replace("/", "_"), 166 | # outputSource=self.path(), 167 | # param_type="File")) 168 | # return 169 | 170 | 171 | # class StepRun: 172 | # """ 173 | # Result of adding a step into a workflow 174 | # """ 175 | # def __init__(self, workflow, id, tool, params): 176 | # self.tool = tool 177 | # self.workflow = workflow 178 | # self.id = id 179 | # 180 | # step = WorkflowStep(id=id, run=tool._path) 181 | # workflow.steps.append(step) 182 | # 183 | # for i, j in params.items(): 184 | # if isinstance(j, six.string_types): 185 | # step.inputs.append(WorkflowStepInput(i, default=j)) 186 | # elif isinstance(j, Variable): 187 | # step.inputs.append(WorkflowStepInput(i, src=j.path())) 188 | # elif isinstance(j, InputParameter): 189 | # self.workflow.inputs.append(j), 190 | # step.inputs.append(WorkflowStepInput(j.id, src=j.id)) 191 | # elif isinstance(j, File): 192 | # # This is just used as a stub, the 'path' inside the file doesn't do anything 193 | # self.workflow.inputs.append(InputParameter(i, param_type="File")) 194 | # step.inputs.append(WorkflowStepInput(i, src=i)) 195 | # for o in tool.outputs: 196 | # step.outputs.append(o.id) 197 | # 198 | # def store_all(self): 199 | # for i in self.tool.outputs: 200 | # Variable(self.workflow, self.id, i.id).store() 201 | # 202 | # def __getitem__(self, key): 203 | # for i in self.tool.outputs: 204 | # if i.id == key: 205 | # return Variable(self.workflow, self.id, key) 206 | # raise KeyError 207 | -------------------------------------------------------------------------------- /cwlgen/workflowdeps.py: -------------------------------------------------------------------------------- 1 | # Import ------------------------------ 2 | 3 | # General libraries 4 | import logging 5 | 6 | # External libraries 7 | import ruamel.yaml 8 | import six 9 | 10 | from .version import __version__ 11 | 12 | # Internal libraries 13 | 14 | from .utils import literal, literal_presenter, Serializable 15 | from .common import Parameter, CWL_SHEBANG 16 | 17 | logging.basicConfig(level=logging.INFO) 18 | _LOGGER = logging.getLogger(__name__) 19 | 20 | SCATTER_METHODS = ["dotproduct", "nested_crossproduct", "flat_crossproduct"] 21 | LINK_MERGE_METHODS = ["merge_nested", "merge_flattened"] 22 | 23 | # Function(s) ------------------------------ 24 | 25 | 26 | def parse_scatter_method(scatter_method, required=False): 27 | if scatter_method is None and not required: 28 | return None 29 | elif scatter_method not in SCATTER_METHODS: 30 | if required: 31 | raise Exception("The scatter method '{method}' is not a valid ScatterMethod and requires one of: {expected}" 32 | .format(method=scatter_method, expected=" ,".join(SCATTER_METHODS))) 33 | elif scatter_method is not None: 34 | _LOGGER.info("The scatter method '{method}' is not a valid ScatterMethod, expected one of: {expected}" 35 | .format(method=scatter_method, expected=" ,".join(SCATTER_METHODS))) 36 | return None 37 | return scatter_method 38 | 39 | 40 | def parse_link_merge_method(link_merge, required=False): 41 | if link_merge is None and not required: 42 | return None 43 | elif link_merge not in LINK_MERGE_METHODS: 44 | if required: 45 | raise Exception("The link merge method '{method}' is not a valid LinkMergeMethod and requires one of:" 46 | " {expected}. ".format(method=link_merge, expected=" ,".join(LINK_MERGE_METHODS))) 47 | elif link_merge is not None: 48 | _LOGGER.info("The link merge method '{method}' is not a valid LinkMergeMethod, expected one of:" 49 | " {expected}. This value will be null which CWL defaults to 'merge_nested'" 50 | .format(method=link_merge, expected=" ,".join(LINK_MERGE_METHODS))) 51 | return None 52 | return link_merge 53 | 54 | 55 | # Class(es) ------------------------------ 56 | class InputParameter(Parameter): 57 | """ 58 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#InputParameter 59 | """ 60 | def __init__(self, param_id, label=None, secondary_files=None, param_format=None, 61 | streamable=None, doc=None, input_binding=None, default=None, param_type=None): 62 | """ 63 | :param param_id: unique identifier for this parameter 64 | :type param_id: STRING 65 | :param label: short, human-readable label 66 | :type label: STRING 67 | :param secondary_files: If type is a file, describes files that must be 68 | included alongside the primary file(s) 69 | :type secondary_files: STRING 70 | :param param_format: If type is a file, uri to ontology of the format or exact format 71 | :type param_format: STRING 72 | :param streamable: If type is a file, true indicates that the file is read or written 73 | sequentially without seeking 74 | :type streamable: BOOLEAN 75 | :param doc: documentation 76 | :type doc: STRING 77 | :param param_type: type of data assigned to the parameter 78 | :type param_type: STRING corresponding to CWLType 79 | :param input_binding: 80 | :type input_binding: CommandLineBinding 81 | 82 | """ 83 | Parameter.__init__(self, param_id=param_id, label=label, 84 | secondary_files=secondary_files, param_format=param_format, 85 | streamable=streamable, doc=doc, param_type=param_type, requires_type=False) 86 | self.inputBinding = input_binding 87 | self.default = default 88 | 89 | @classmethod 90 | def parse_with_id(cls, d, identifier): 91 | if isinstance(d, str): 92 | d = {"type": d} 93 | d["id"] = identifier 94 | return super(InputParameter, cls).parse_dict(d) 95 | 96 | 97 | class WorkflowStepInput(Serializable): 98 | """ 99 | The input of a workflow step connects an upstream parameter (from the workflow inputs, or the outputs of 100 | other workflows steps) with the input parameters of the underlying step. 101 | 102 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#WorkflowStepInput 103 | """ 104 | def __init__(self, input_id, source=None, link_merge=None, default=None, value_from=None): 105 | """ 106 | :param input_id: A unique identifier for this workflow input parameter. 107 | :type input_id: STRING 108 | :param source: Specifies one or more workflow parameters that will provide input to the underlying step parameter. 109 | :type source: STRING | list[STRING] 110 | :param link_merge: The method to use to merge multiple inbound links into a single array. 111 | If not specified, the default method is "merge_nested". 112 | :type link_merge: LinkMergeMethod 113 | :param default: The default value for this parameter to use if either there is no source field, 114 | or the value produced by the source is null 115 | :type default: Any | DictRepresentible 116 | :param value_from: If valueFrom is a constant string value, use this as the value for this input parameter. 117 | If valueFrom is a parameter reference or expression, 118 | it must be evaluated to yield the actual value to be assiged to the input field. 119 | :type value_from: STRING 120 | """ 121 | 122 | self.id = input_id 123 | self.source = source 124 | self.linkMerge = parse_link_merge_method(link_merge) 125 | self.default = default 126 | self.valueFrom = value_from 127 | 128 | 129 | @classmethod 130 | def parse_with_id(cls, d, identifier): 131 | if isinstance(d, str): 132 | d = {"source": d} 133 | d["id"] = identifier 134 | return super(WorkflowStepInput, cls).parse_dict(d) 135 | 136 | 137 | class WorkflowStepOutput(Serializable): 138 | """ 139 | Associate an output parameter of the underlying process with a workflow parameter. 140 | The workflow parameter (given in the id field) be may be used as a source to connect with 141 | input parameters of other workflow steps, or with an output parameter of the process. 142 | 143 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#WorkflowStepOutput 144 | """ 145 | def __init__(self, output_id): 146 | """ 147 | :param output_id: A unique identifier for this workflow output parameter. This is the identifier to use in 148 | the source field of WorkflowStepInput to connect the output value to downstream parameters. 149 | :type output_id: STRING 150 | """ 151 | self.id = output_id 152 | 153 | def get_dict(self): 154 | return self.id 155 | 156 | 157 | class WorkflowStep(Serializable): 158 | """ 159 | A workflow step is an executable element of a workflow. It specifies the underlying process implementation 160 | (such as CommandLineTool or another Workflow) in the run field and connects the input and output parameters 161 | of the underlying process to workflow parameters. 162 | 163 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#WorkflowStep 164 | """ 165 | 166 | # dict['in'] gets converted to dict['inputs'] as 'in' is a reserved keyword 167 | parse_types = { 168 | "inputs": [[WorkflowStepInput]], 169 | "out": [str, [WorkflowStepOutput]] 170 | } 171 | 172 | def __init__(self, step_id, run, label=None, doc=None, scatter=None, scatter_method=None): 173 | """ 174 | :param step_id: The unique identifier for this workflow step. 175 | :type step_id: STRING 176 | :param run: Specifies the process to run. 177 | :type run: STRING | CommandLineTool | ExpressionTool | Workflow 178 | :param label: A short, human-readable label of this process object. 179 | :type label: STRING 180 | :param doc: A long, human-readable description of this process object. 181 | :type doc: STRING | list[STRING] 182 | :param scatter: Field to scatter on, see: https://www.commonwl.org/v1.0/Workflow.html#WorkflowStep 183 | :type scatter: STRING | list[STRING] 184 | :param scatter_method: Required if scatter is an array of more than one element. 185 | :type scatter_method: STRING | list[STRING] in [dotproduct, nested_crossproduct, flat_crossproduct] 186 | """ 187 | self.id = step_id 188 | self.run = run 189 | self.label = label 190 | self.doc = doc 191 | self.scatter = scatter 192 | self.scatterMethod = parse_scatter_method(scatter_method) 193 | 194 | # in is a reserved keywork 195 | self.inputs = [] 196 | self.out = [] 197 | self.requirements = [] 198 | self.hints = [] 199 | 200 | self.ignore_attributes = ["id", "inputs"] 201 | 202 | def get_dict(self): 203 | d = super(WorkflowStep, self).get_dict() 204 | d['in'] = {i.id: self.serialize(i) for i in self.inputs} 205 | return d 206 | 207 | @classmethod 208 | def parse_with_id(cls, d, identifier): 209 | d["id"] = identifier 210 | return cls.parse_dict(d) 211 | 212 | @classmethod 213 | def parse_dict(cls, d): 214 | # We just need to map in -> inputs instead 215 | d["inputs"] = d.pop("in", []) 216 | return super(WorkflowStep, cls).parse_dict(d) 217 | 218 | 219 | class WorkflowOutputParameter(Parameter): 220 | """ 221 | Describe an output parameter of a workflow. The parameter must be connected to one or more parameters 222 | defined in the workflow that will provide the value of the output parameter. 223 | 224 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#WorkflowOutputParameter 225 | """ 226 | def __init__(self, param_id, output_source=None, label=None, secondary_files=None, param_format=None, 227 | streamable=None, doc=None, param_type=None, output_binding=None, linkMerge=None): 228 | """ 229 | Documentation: https://www.commonwl.org/v1.0/Workflow.html#WorkflowOutputParameter 230 | :param param_id: The unique identifier for this parameter object. 231 | :type param_id: STRING 232 | :param output_source: Specifies one or more workflow parameters that supply the value of to the output parameter 233 | :type output_source: STRING | list[STRING] 234 | :param label: A short, human-readable label of this object. 235 | :type label: STRING 236 | :param secondary_files: Provides a pattern or expression specifying files or directories that must be 237 | included alongside the primary file. 238 | :type secondary_files: STRING \ list[STRING] 239 | :param param_format: This is the file format that will be assigned to the output paramete 240 | :type param_format: STRING 241 | :param streamable: A value of true indicates that the file is read or written sequentially without seeking 242 | :type streamable: BOOLEAN 243 | :param doc: A documentation string for this type, or an array of strings which should be concatenated. 244 | :type doc: STRING | list[STRING] 245 | :param param_type: Specify valid types of data that may be assigned to this parameter. 246 | :type param_type: CWLType | OutputRecordSchema | OutputEnumSchema | OutputArraySchema | string | Array 247 | :param output_binding: Describes how to handle the outputs of a process. 248 | :type output_binding: CommandOutputBinding 249 | :param linkMerge: 250 | :type linkMerge: STRING 251 | """ 252 | Parameter.__init__(self, param_id=param_id, label=label, 253 | secondary_files=secondary_files, param_format=param_format, 254 | streamable=streamable, doc=doc, param_type=param_type, requires_type=False) 255 | self.outputSource = output_source 256 | self.outputBinding = output_binding # CommandOutputBinding 257 | self.linkMerge = linkMerge 258 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/IntegronFinder.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/IntegronFinder.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/IntegronFinder" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/IntegronFinder" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /doc/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends '!layout.html' %} 2 | 3 | {% block footer %} 4 | 5 | 6 | Fork me on GitHub 10 | 11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /doc/source/changelogs.rst: -------------------------------------------------------------------------------- 1 | .. python-cwlgen - Python library for manipulation and generation of CWL tools. 2 | 3 | .. _changelogs: 4 | 5 | ********** 6 | Changelogs 7 | ********** 8 | 9 | Summary of developments of Python-cwlgen library. 10 | 11 | v0.3 12 | ==== 13 | 14 | v0.3.0 15 | ------ 16 | 17 | This update brings more completeness to the v1.0 spec. 18 | 19 | * Large increase in the number of supported classes 20 | * New translation mechanism 21 | * More docstrings 22 | 23 | v0.2 24 | ==== 25 | 26 | v0.2.3 27 | ------ 28 | 29 | * Bug fix: fix dumping in out_file 30 | * Handle multiline for doc in CWL tool 31 | 32 | v0.2.2 33 | ------ 34 | 35 | * Add namespaces and possibility to add metadata described by schema.org 36 | 37 | v0.2.1 38 | ------ 39 | 40 | * Change order of attribute for CommandLineTool 41 | * remove id field of input and output that appeared in v0.2.0 42 | 43 | v0.2.0 44 | ------ 45 | 46 | * Add import feature for what is covered so far by the library 47 | * Change attribute names of object to correspond exactly to CWL Tool fields 48 | 49 | v0.1 50 | ==== 51 | 52 | v0.1.1 53 | ------ 54 | 55 | This is the first release of Python-cwlgen: 56 | 57 | * Basic model of CWL Tool 58 | * export feature to STDOUT or output file 59 | -------------------------------------------------------------------------------- /doc/source/classes.rst: -------------------------------------------------------------------------------- 1 | .. python-cwlgen - Python library for manipulation and generation of CWL tools. 2 | 3 | *********** 4 | API classes 5 | *********** 6 | 7 | Workflow and CommandLineTool 8 | ============================ 9 | 10 | See the links below to the `CommandLineTool` and `Workflow` classes: 11 | 12 | - :class:`cwlgen.CommandLineTool` 13 | - :class:`cwlgen.Workflow` 14 | 15 | Requirements 16 | ============ 17 | 18 | Requirement 19 | """"""""""" 20 | 21 | This is the (abstract) base requirement class. 22 | 23 | .. autoclass:: cwlgen.Requirement 24 | :members: 25 | :private-members: 26 | :special-members: 27 | :exclude-members: __weakref__ 28 | 29 | InlineJavascriptRequirement 30 | """"""""""""""""""""""""""" 31 | 32 | .. autoclass:: cwlgen.InlineJavascriptRequirement 33 | :members: 34 | :private-members: 35 | :special-members: 36 | :exclude-members: __weakref__ 37 | 38 | SchemaDefRequirement 39 | """""""""""""""""""" 40 | 41 | See the Schema section Below: 42 | 43 | - :class:`cwlgen.SchemaDefRequirement` 44 | 45 | 46 | SubworkflowFeatureRequirement 47 | """"""""""""""""""""""""""""" 48 | 49 | .. autoclass:: cwlgen.SubworkflowFeatureRequirement 50 | :members: 51 | :private-members: 52 | :special-members: 53 | :exclude-members: __weakref__ 54 | 55 | ScatterFeatureRequirement 56 | """""""""""""""""""""""""" 57 | 58 | .. autoclass:: cwlgen.ScatterFeatureRequirement 59 | :members: 60 | :private-members: 61 | :special-members: 62 | :exclude-members: __weakref__ 63 | 64 | MultipleInputFeatureRequirement 65 | """"""""""""""""""""""""""""""" 66 | 67 | .. autoclass:: cwlgen.MultipleInputFeatureRequirement 68 | :members: 69 | :private-members: 70 | :special-members: 71 | :exclude-members: __weakref__ 72 | 73 | StepInputExpressionRequirement 74 | """"""""""""""""""""""""""""""" 75 | 76 | .. autoclass:: cwlgen.StepInputExpressionRequirement 77 | :members: 78 | :private-members: 79 | :special-members: 80 | :exclude-members: __weakref__ 81 | 82 | DockerRequirement 83 | """"""""""""""""" 84 | 85 | .. autoclass:: cwlgen.DockerRequirement 86 | :members: 87 | :private-members: 88 | :special-members: 89 | :exclude-members: __weakref__ 90 | 91 | SoftwareRequirement 92 | """"""""""""""""""" 93 | 94 | .. autoclass:: cwlgen.SoftwareRequirement 95 | :members: 96 | :private-members: 97 | :special-members: 98 | :exclude-members: SoftwarePackage,__weakref__ 99 | 100 | .. autoclass:: cwlgen.SoftwareRequirement.SoftwarePackage 101 | :members: 102 | :private-members: 103 | :special-members: 104 | :exclude-members: __weakref__ 105 | 106 | 107 | InitialWorkDirRequirement 108 | """""""""""""""""""""""""" 109 | 110 | .. autoclass:: cwlgen.InitialWorkDirRequirement 111 | :members: 112 | :private-members: 113 | :special-members: 114 | :exclude-members: Dirent,__weakref__ 115 | 116 | .. autoclass:: cwlgen.InitialWorkDirRequirement.Dirent 117 | :members: 118 | :private-members: 119 | :special-members: 120 | :exclude-members: __weakref__ 121 | 122 | 123 | EnvVarRequirement 124 | """"""""""""""""""" 125 | 126 | .. autoclass:: cwlgen.EnvVarRequirement 127 | :members: 128 | :private-members: 129 | :special-members: 130 | :exclude-members: EnvironmentDef,__weakref__ 131 | 132 | .. autoclass:: cwlgen.EnvVarRequirement.EnvironmentDef 133 | :members: 134 | :private-members: 135 | :special-members: 136 | :exclude-members: __weakref__ 137 | 138 | ShellCommandRequirement 139 | """"""""""""""""""""""" 140 | 141 | .. autoclass:: cwlgen.ShellCommandRequirement 142 | :members: 143 | :private-members: 144 | :special-members: 145 | :exclude-members: __weakref__ 146 | 147 | ResourceRequirement 148 | """"""""""""""""""" 149 | 150 | .. autoclass:: cwlgen.ResourceRequirement 151 | :members: 152 | :private-members: 153 | :special-members: 154 | :exclude-members: __weakref__ 155 | 156 | 157 | Schema 158 | ====== 159 | 160 | .. autoclass:: cwlgen.SchemaDefRequirement 161 | :members: 162 | :private-members: 163 | :special-members: 164 | :exclude-members: InputRecordSchema,InputEnumSchema,InputArraySchema,__weakref__ 165 | 166 | 167 | Workflow Input Schema 168 | """"""""""""""""""""" 169 | 170 | .. autoclass:: cwlgen.SchemaDefRequirement.InputRecordSchema 171 | :members: 172 | :private-members: 173 | :special-members: 174 | :exclude-members: __weakref__ 175 | 176 | .. autoclass:: cwlgen.SchemaDefRequirement.InputEnumSchema 177 | :members: 178 | :private-members: 179 | :special-members: 180 | :exclude-members: __weakref__ 181 | 182 | .. autoclass:: cwlgen.SchemaDefRequirement.InputArraySchema 183 | :members: 184 | :private-members: 185 | :special-members: 186 | :exclude-members: __weakref__ 187 | 188 | - :class:`CommandLineBinding` 189 | 190 | When listed under inputBinding in the input schema, the term "value" 191 | refers to the the corresponding value in the input object. For binding objects listed in 192 | CommandLineTool.arguments, the term "value" refers to the effective value after evaluating valueFrom. 193 | 194 | 195 | Import CWL 196 | ========== 197 | 198 | As of release v0.3.0 the existing importing CWL has been replaced by an 199 | automated deserialization. Each function that inherits from the :class:`Serializable` 200 | class will have a ``parse_dict`` method. 201 | 202 | If you're adding a class and want to provide a hint on how to parse a particular 203 | field, you can add a static ``parse_types`` dictionary onto your class with the 204 | fieldname and a list of types that you want to try and parse as. If your input 205 | can be a list (eg: ``T[]``), or a dictionary with the identifier as the key 206 | (eg: ``{ $identifier: T }``, you can let your type be ``[T]`` in the ``parse_types`` 207 | dict. It will automatically inject this identifier in the constructor. 208 | See the ``Serializable.parse_dict`` class for more information. 209 | 210 | .. code-block:: python 211 | 212 | class Workflow: 213 | parse_types = { 214 | # Parse inputs as : [InputParameter] or { id: InputParameter } 215 | "inputs": [[InputParameter]], 216 | 217 | # will attempt to parse extraParam as a string, then SecondaryType, 218 | # then (TertiaryType[] || { $identifier: TertiaryType } 219 | "extraParam": [str, SecondaryType, [TertiaryType]] 220 | } 221 | 222 | .. autofunction:: cwlgen.parse_cwl 223 | 224 | .. autofunction:: cwlgen.parse_cwl_dict 225 | -------------------------------------------------------------------------------- /doc/source/commandlinetoolclasses.rst: -------------------------------------------------------------------------------- 1 | .. python-cwlgen - Python library for manipulation and generation of CWL tools. 2 | 3 | *************************** 4 | CommandLineTool API classes 5 | *************************** 6 | 7 | CommandLineTool 8 | =============== 9 | 10 | .. autoclass:: cwlgen.CommandLineTool 11 | :members: 12 | :private-members: 13 | :special-members: 14 | :exclude-members: __weakref__ 15 | 16 | 17 | Input and outputs 18 | ================= 19 | 20 | CommandInputParameter 21 | """"""""""""""""""""" 22 | 23 | .. autoclass:: cwlgen.CommandInputParameter 24 | :members: 25 | :private-members: 26 | :special-members: 27 | :exclude-members: __weakref__ 28 | 29 | CommandOutputParameter 30 | """""""""""""""""""""" 31 | 32 | .. autoclass:: cwlgen.CommandOutputParameter 33 | :members: 34 | :private-members: 35 | :special-members: 36 | :exclude-members: __weakref__ 37 | 38 | CommandLineBinding 39 | """""""""""""""""" 40 | 41 | .. autoclass:: cwlgen.CommandLineBinding 42 | :members: 43 | :private-members: 44 | :special-members: 45 | :exclude-members: __weakref__ 46 | 47 | CommandOutputBinding 48 | """""""""""""""""""" 49 | 50 | .. autoclass:: cwlgen.CommandOutputBinding 51 | :members: 52 | :private-members: 53 | :special-members: 54 | :exclude-members: __weakref__ 55 | 56 | Special Types 57 | ============== 58 | 59 | .. autoclass:: cwlgen.CommandInputRecordSchema 60 | :members: 61 | :private-members: 62 | :special-members: 63 | :exclude-members: CommandInputRecordField,__weakref__ 64 | 65 | .. autoclass:: cwlgen.CommandInputRecordSchema.CommandInputRecordField 66 | :members: 67 | :private-members: 68 | :special-members: 69 | :exclude-members: __weakref__ 70 | 71 | .. autoclass:: cwlgen.CommandInputArraySchema 72 | :members: 73 | :private-members: 74 | :special-members: 75 | :exclude-members: __weakref__ 76 | 77 | .. autoclass:: cwlgen.CommandInputEnumSchema 78 | :members: 79 | :private-members: 80 | :special-members: 81 | :exclude-members: __weakref__ -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Python-cwlgen documentation build configuration file 4 | # 5 | # This file is execfile()d with the current directory set to its 6 | # containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys 15 | import os 16 | import shlex 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.viewcode', 34 | ] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix(es) of source filenames. 40 | # You can specify multiple suffix as a list of string: 41 | # source_suffix = ['.rst', '.md'] 42 | source_suffix = '.rst' 43 | 44 | # The encoding of source files. 45 | #source_encoding = 'utf-8-sig' 46 | 47 | # The master toctree document. 48 | master_doc = 'index' 49 | 50 | # General information about the project. 51 | project = u'python-cwlgen' 52 | copyright = u'2016-2017 Kenzo-Hugo Hillion, Hervé Ménager' 53 | author = u'2016-2017 Kenzo-Hugo Hillion, Hervé Ménager' 54 | 55 | # The version info for the project you're documenting, acts as replacement for 56 | # |version| and |release|, also used in various other places throughout the 57 | # built documents. 58 | # 59 | # The short X.Y version. 60 | version = '0.1' 61 | # The full version, including alpha/beta/rc tags. 62 | release = '0.1' 63 | 64 | # The language for content autogenerated by Sphinx. Refer to documentation 65 | # for a list of supported languages. 66 | # 67 | # This is also used if you do content translation via gettext catalogs. 68 | # Usually you set "language" from the command line for these cases. 69 | language = None 70 | 71 | # There are two options for replacing |today|: either, you set today to some 72 | # non-false value, then it is used: 73 | #today = '' 74 | # Else, today_fmt is used as the format for a strftime call. 75 | #today_fmt = '%B %d, %Y' 76 | 77 | # List of patterns, relative to source directory, that match files and 78 | # directories to ignore when looking for source files. 79 | exclude_patterns = [] 80 | 81 | # The reST default role (used for this markup: `text`) to use for all 82 | # documents. 83 | #default_role = None 84 | 85 | # If true, '()' will be appended to :func: etc. cross-reference text. 86 | #add_function_parentheses = True 87 | 88 | # If true, the current module name will be prepended to all description 89 | # unit titles (such as .. function::). 90 | #add_module_names = True 91 | 92 | # If true, sectionauthor and moduleauthor directives will be shown in the 93 | # output. They are ignored by default. 94 | #show_authors = False 95 | 96 | # The name of the Pygments (syntax highlighting) style to use. 97 | pygments_style = 'sphinx' 98 | 99 | # A list of ignored prefixes for module index sorting. 100 | #modindex_common_prefix = [] 101 | 102 | # If true, keep warnings as "system message" paragraphs in the built documents. 103 | #keep_warnings = False 104 | 105 | # If true, `todo` and `todoList` produce output, else they produce nothing. 106 | todo_include_todos = False 107 | 108 | 109 | # -- Options for HTML output ---------------------------------------------- 110 | 111 | # The theme to use for HTML and HTML Help pages. See the documentation for 112 | # a list of builtin themes. 113 | html_theme = 'sphinx_rtd_theme' 114 | # list: alabaster, basic, classic, sphinxdoc, scrolls, agogo, traditional, 115 | # nature, haiku, pyramid, bizstyle 116 | 117 | 118 | 119 | # Theme options are theme-specific and customize the look and feel of a theme 120 | # further. For a list of options available for each theme, see the 121 | # documentation. 122 | #html_theme_options = {} 123 | 124 | # Add any paths that contain custom themes here, relative to this directory. 125 | #html_theme_path = [] 126 | 127 | # The name for this set of Sphinx documents. If None, it defaults to 128 | # " v documentation". 129 | #html_title = None 130 | 131 | # A shorter title for the navigation bar. Default is the same as html_title. 132 | #html_short_title = None 133 | 134 | # The name of an image file (relative to this directory) to place at the top 135 | # of the sidebar. 136 | #html_logo = None 137 | 138 | # The name of an image file (within the static path) to use as favicon of the 139 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 140 | # pixels large. 141 | #html_favicon = None 142 | 143 | # Add any paths that contain custom static files (such as style sheets) here, 144 | # relative to this directory. They are copied after the builtin static files, 145 | # so a file named "default.css" will overwrite the builtin "default.css". 146 | html_static_path = ['_static'] 147 | 148 | # Add any extra paths that contain custom files (such as robots.txt or 149 | # .htaccess) here, relative to this directory. These files are copied 150 | # directly to the root of the documentation. 151 | #html_extra_path = [] 152 | 153 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 154 | # using the given strftime format. 155 | #html_last_updated_fmt = '%b %d, %Y' 156 | 157 | # If true, SmartyPants will be used to convert quotes and dashes to 158 | # typographically correct entities. 159 | html_use_smartypants = False 160 | 161 | # Custom sidebar templates, maps document names to template names. 162 | #html_sidebars = {} 163 | 164 | # Additional templates that should be rendered to pages, maps page names to 165 | # template names. 166 | #html_additional_pages = {} 167 | 168 | # If false, no module index is generated. 169 | #html_domain_indices = True 170 | 171 | # If false, no index is generated. 172 | #html_use_index = True 173 | 174 | # If true, the index is split into individual pages for each letter. 175 | #html_split_index = False 176 | 177 | # If true, links to the reST source_map are added to the pages. 178 | #html_show_sourcelink = True 179 | 180 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 181 | #html_show_sphinx = True 182 | 183 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 184 | #html_show_copyright = True 185 | 186 | # If true, an OpenSearch description file will be output, and all pages will 187 | # contain a tag referring to it. The value of this option must be the 188 | # base URL from which the finished HTML is served. 189 | #html_use_opensearch = '' 190 | 191 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 192 | #html_file_suffix = None 193 | 194 | # Language to be used for generating the HTML full-text search index. 195 | # Sphinx supports the following languages: 196 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 197 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 198 | #html_search_language = 'en' 199 | 200 | # A dictionary with options for the search language support, empty by default. 201 | # Now only 'ja' uses this config value 202 | #html_search_options = {'type': 'default'} 203 | 204 | # The name of a javascript file (relative to the configuration directory) that 205 | # implements a search results scorer. If empty, the default will be used. 206 | #html_search_scorer = 'scorer.js' 207 | 208 | # Output file base name for HTML help builder. 209 | htmlhelp_basename = 'python-cwlgendoc' 210 | 211 | # -- Options for LaTeX output --------------------------------------------- 212 | 213 | latex_elements = { 214 | 215 | 'classoptions': ',openany,oneside', 216 | 'babel' : '\\usepackage[english]{babel}', 217 | 218 | # The paper size ('letterpaper' or 'a4paper'). 219 | #'papersize': 'letterpaper', 220 | 221 | # The font size ('10pt', '11pt' or '12pt'). 222 | 'pointsize': '12pt', 223 | 224 | # Additional stuff for the LaTeX preamble. 225 | #'preamble': '', 226 | 227 | # Latex figure (float) alignment 228 | #'figure_align': 'htbp', 229 | } 230 | 231 | # Grouping the document tree into LaTeX files. List of tuples 232 | # (source start file, target name, title, 233 | # author, documentclass [howto, manual, or own class]). 234 | latex_documents = [ 235 | (master_doc, 'python-cwlgen.tex', u'python-cwlgen Documentation', 236 | u'Kenzo-Hugo Hillion, Hervé Ménager', 'manual'), 237 | ] 238 | 239 | # The name of an image file (relative to this directory) to place at the top of 240 | # the title page. 241 | #latex_logo = None 242 | 243 | # For "manual" documents, if this is true, then toplevel headings are parts, 244 | # not chapters. 245 | #latex_use_parts = False 246 | 247 | # If true, show page references after internal links. 248 | #latex_show_pagerefs = False 249 | 250 | # If true, show URL addresses after external links. 251 | #latex_show_urls = False 252 | 253 | # Documents to append as an appendix to all manuals. 254 | #latex_appendices = [] 255 | 256 | # If false, no module index is generated. 257 | #latex_domain_indices = True 258 | 259 | 260 | # -- Options for manual page output --------------------------------------- 261 | 262 | # One entry per manual page. List of tuples 263 | # (source start file, name, description, authors, manual section). 264 | man_pages = [ 265 | (master_doc, 'python-cwlgen', u'python-cwlgen Documentation', 266 | [author], 1) 267 | ] 268 | 269 | # If true, show URL addresses after external links. 270 | #man_show_urls = False 271 | 272 | 273 | # -- Options for Texinfo output ------------------------------------------- 274 | 275 | # Grouping the document tree into Texinfo files. List of tuples 276 | # (source start file, target name, title, author, 277 | # dir menu entry, description, category) 278 | texinfo_documents = [ 279 | (master_doc, 'python-cwlgen', u'python-cwlgen Documentation', 280 | author, 'python-cwlgen', 'Library for manipulation and generation of CWL tool.', 281 | 'Miscellaneous'), 282 | ] 283 | 284 | # Documents to append as an appendix to all manuals. 285 | #texinfo_appendices = [] 286 | 287 | # If false, no module index is generated. 288 | #texinfo_domain_indices = True 289 | 290 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 291 | #texinfo_show_urls = 'footnote' 292 | 293 | # If true, do not generate a @detailmenu in the "Top" node's menu. 294 | #texinfo_no_detailmenu = False 295 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. python-cwlgen - Python library for manipulation and generation of CWL tools. 2 | 3 | 4 | Python-CWLGen (**Deprecated**) 5 | ========================================= 6 | 7 | .. warning:: 8 | 9 | ``python-cwlgen`` is now deprecated, please use `cwl-utils >= 0.4 `_. 10 | 11 | Example migration: 12 | 13 | .. code-block:: bash 14 | 15 | from cwl_utils import parser_v1_0 16 | 17 | # You could alias this as cwlgen to simplify the migration 18 | from cwl_utils import parser_v1_0 as cwlgen 19 | 20 | 21 | Migration notes: 22 | +++++++++++++++++++++ 23 | 24 | - Method changes 25 | 26 | - ``get_dict() → save()`` 27 | - ``parse_cwl(cwlfile)`` → ``load_document(cwlfile)`` 28 | - ``parse_dict`` → No super clear analogue, but loaded through ``_RecordLoader(CommandLineTool)`` || ``_UnionLoader((CommandLineToolLoader, ...workflow + other loaders)`` 29 | 30 | - Field names: 31 | 32 | - Uses ``camelCase`` instead of ``snake_case`` 33 | - No more special field names, eg: 34 | - ``tool_id`` | ``workflow_id`` | ``input_id`` | etc → ``id`` 35 | - ``StepInput``: ``inputs`` → ``in_`` 36 | 37 | - Other notes: 38 | 39 | - Classes aren't nested anymore, ie: ``cwlgen.InitialWorkDirRequirement.Dirent`` → ``cwlutils``. 40 | - Take care if you're migrating to a newer spec, as some classes might have changed names (notably: ``InputParameter`` -> ``WorkflowInputParameter``) 41 | - Don't forget to catch all references of cwlgen, as missing one (or using mismatch versions of the parser) will cause: 42 | 43 | .. code-block:: python 44 | 45 | raise RepresenterError('cannot represent an object: %s' % (data,)) 46 | ruamel.yaml.representer.RepresenterError: cannot represent an object: 47 | 48 | 49 | If you have issues with the migration, please see `this thread `_ or raise an issue on CWLUtils. 50 | 51 | 52 | .. image:: https://travis-ci.org/common-workflow-language/python-cwlgen.svg?branch=master&style=flat 53 | :target: https://travis-ci.org/common-workflow-language/python-cwlgen 54 | :alt: Travis Build Status 55 | 56 | .. image:: https://readthedocs.org/projects/python-cwlgen/badge/?version=latest 57 | :target: https://python-cwlgen.readthedocs.io/en/latest/?badge=latest) 58 | :alt: Documentation 59 | 60 | .. image:: https://badge.fury.io/py/cwlgen.svg 61 | :target: https://pypi.org/project/cwlgen/ 62 | :alt: Pypi module 63 | 64 | .. image:: https://codecov.io/gh/common-workflow-language/python-cwlgen/branch/master/graph/badge.svg 65 | :target: https://codecov.io/gh/common-workflow-language/python-cwlgen 66 | :alt: Code Coverage 67 | 68 | 69 | Python-cwlgen is a python library for the programmatic generation of CWL v1.0. 70 | It supports the generation of CommandLineTool and Workflows. 71 | 72 | The library works for both Python 2.7.12+ and 3.6.0+. 73 | 74 | Quick-start 75 | =========== 76 | 77 | You can install Python-CWLGen through pip with the following command: 78 | 79 | .. code-block:: bash 80 | 81 | pip install cwlgen 82 | 83 | The classes very closely (if not exactly) mirror the CWL v1.0 specification. You can find 84 | more about their parameters in the following specifications: 85 | 86 | - :class:`cwlgen.CommandLineTool` 87 | - :class:`cwlgen.Workflow` 88 | 89 | 90 | Python-cwlgen 91 | ============= 92 | .. toctree:: 93 | :maxdepth: 2 94 | 95 | installation 96 | user_guide 97 | references 98 | 99 | Python-cwlgen API documentation 100 | =============================== 101 | .. toctree:: 102 | :maxdepth: 1 103 | 104 | classes 105 | commandlinetoolclasses 106 | workflowclasses 107 | changelogs 108 | 109 | .. 110 | Indices and tables 111 | ================== 112 | 113 | * :ref:`genindex` 114 | * :ref:`modindex` 115 | * :ref:`search` 116 | -------------------------------------------------------------------------------- /doc/source/installation.rst: -------------------------------------------------------------------------------- 1 | .. python-cwlgen - Python library for manipulation and generation of CWL tools. 2 | 3 | .. _install: 4 | 5 | ************ 6 | Installation 7 | ************ 8 | 9 | .. Note:: 10 | We highly recommend the use of a virtual environment with Python 3.6.0 11 | using `virtualenv`_ or `conda`_. 12 | 13 | .. _virtualenv: https://virtualenv.pypa.io/en/latest/ 14 | .. _conda: http://docs.readthedocs.io/en/latest/conda.html 15 | 16 | .. _dependencies: 17 | 18 | python-cwlgen dependencies 19 | ========================== 20 | 21 | python-cwlgen has been primarily tested using Python3 and uses the following libraries: 22 | 23 | - ``ruamel.yaml`` (between 0.12.4 and 0.15.87) 24 | - ``six`` (1.10.0) 25 | 26 | The project has been designed to work with Python 2.7+ and has accompanying tests, however 27 | please raise an issue if you have incompatibility issues. 28 | 29 | 30 | .. _installation: 31 | 32 | Installation procedure 33 | ====================== 34 | 35 | Pip 36 | --- 37 | 38 | You can use pip to install the latest version from pypi: 39 | 40 | .. code-block:: bash 41 | 42 | pip install cwlgen 43 | 44 | Manually 45 | -------- 46 | 47 | Clone the repository and install cwlgen with the following command: 48 | 49 | .. code-block:: bash 50 | 51 | git clone https://github.com/common-workflow-language/python-cwlgen.git 52 | cd python-cwlgen 53 | pip install . 54 | 55 | .. _uninstallation: 56 | 57 | Uninstallation procedure 58 | ========================= 59 | 60 | Pip 61 | --- 62 | 63 | You can remove python-cwlgen with the following command: 64 | 65 | .. code-block:: bash 66 | 67 | pip uninstall cwlgen 68 | 69 | .. Note:: 70 | This will not uninstall dependencies. To do so you can make use of `pip-autoremove`_. 71 | 72 | .. _pip-autoremove: https://github.com/invl/pip-autoremove 73 | -------------------------------------------------------------------------------- /doc/source/references.rst: -------------------------------------------------------------------------------- 1 | .. python-cwlgen - Python library for manipulation and generation of CWL tools. 2 | 3 | .. _references: 4 | 5 | ********** 6 | References 7 | ********** 8 | 9 | Common Workflow Language 10 | ======================== 11 | 12 | CWL is developed by an informal, multi-vendor working group consisting of 13 | organizations and individuals aiming to enable scientists to share data analysis 14 | workflows. The CWL project is on `Github`_. 15 | 16 | .. _`Github`: https://github.com/common-workflow-language/common-workflow-language 17 | -------------------------------------------------------------------------------- /doc/source/user_guide.rst: -------------------------------------------------------------------------------- 1 | .. python-cwlgen - Python library for manipulation and generation of CWL tools. 2 | 3 | .. _user_guide: 4 | 5 | ********** 6 | User Guide 7 | ********** 8 | 9 | This user guide assumes you have at least some basic knowledge about CWL. 10 | 11 | .. Note:: 12 | Here is a `CWL user guide`_ for an introduction to tool and workflows wrappers. 13 | 14 | .. _`CWL user guide`: https://www.commonwl.org/user_guide/ 15 | 16 | The aim is to help you through the different steps to build your CWL tool with 17 | python-cwlgen. 18 | 19 | .. Note:: 20 | If you find a bug, have any questions or suggestions, please `submit an issue on Github`_. 21 | 22 | .. _`submit an issue on Github`: https://github.com/common-workflow-language/python-cwlgen/issues/new 23 | 24 | Basic example 25 | ------------- 26 | Through this little tutorial, we will go step by step through the example you can `find on Github`_. It aims to wrap the `grep` command. 27 | 28 | .. _`find on Github`: https://github.com/common-workflow-language/python-cwlgen/blob/master/examples/example.py 29 | 30 | Initialize your tool 31 | """""""""""""""""""" 32 | 33 | You need to initialize a `CommandLineTool` object 34 | 35 | .. code-block:: python 36 | 37 | import cwlgen 38 | cwl_tool = cwlgen.CommandLineTool(tool_id='grep', 39 | label='print lines matching a pattern', 40 | base_command='grep') 41 | 42 | Now that you have your object, you can attach the different elements of a tool description. 43 | 44 | Add Inputs 45 | """""""""" 46 | 47 | Now we need to add inputs to our tool. We are going to only wrap a simple version of the `grep` command with a input file and a pattern. 48 | 49 | First the input file: 50 | 51 | .. code-block:: python 52 | 53 | file_binding = cwlgen.CommandLineBinding(position=2) 54 | input_file = cwlgen.CommandInputParameter('input_file', 55 | param_type='File', 56 | input_binding=file_binding, 57 | doc='input file from which you want to look for the pattern') 58 | cwl_tool.inputs.append(input_file) 59 | 60 | And finally the pattern: 61 | 62 | .. code-block:: python 63 | 64 | pattern_binding = cwlgen.CommandLineBinding(position=1) 65 | pattern = cwlgen.CommandInputParameter('pattern', 66 | param_type='string', 67 | input_binding=pattern_binding, 68 | doc='pattern to find in the input file') 69 | cwl_tool.inputs.append(pattern) 70 | 71 | .. Note:: 72 | You can specify more information concerning your inputs: `Input documentation`_ 73 | 74 | .. _`Input documentation`: http://python-cwlgen.readthedocs.io/en/latest/classes.html#input-and-outputs 75 | 76 | This is it for the inputs, now let's add some outputs and the description will be ready to be tested. 77 | 78 | Add an Output 79 | """"""""""""" 80 | 81 | The only output which is retrieved in our example is a File with the line containing the pattern. Here is how to add this output: 82 | 83 | .. code-block:: python 84 | 85 | output = cwlgen.CommandOutputParameter('output', 86 | param_type='stdout', 87 | doc='lines found with the pattern') 88 | cwl_tool.outputs.append(output) 89 | # Now specify a name for your output file 90 | cwl_tool.stdout = "grep.txt" 91 | 92 | Add Documentation and Metadata 93 | """""""""""""""""""""""""""""" 94 | 95 | You can ask bunch of information and metadata concerning your tool. 96 | For instance you can add some documentation: 97 | 98 | .. code-block:: python 99 | 100 | cwl_tool.doc = "grep searches for a pattern in a file." 101 | 102 | For the metadata: 103 | 104 | .. code-block:: python 105 | 106 | metadata = {'name': 'grep', 107 | 'about' : 'grep searches for a pattern in a file.'} 108 | cwl_tool.metadata = cwlgen.Metadata(**metadata) 109 | 110 | Write your tool 111 | """"""""""""""" 112 | 113 | Finally, you can export your tool description with the `export()` method. 114 | 115 | .. code-block:: python 116 | 117 | cwl_tool.export() # On STDOUT 118 | cwl_tool.export(outfile="grep.cwl") # As a file (grep.cwl) 119 | 120 | You can then try your tool description (using `cwltool`_ for instance): 121 | 122 | .. _`cwltool`: https://github.com/common-workflow-language/cwltool/ 123 | 124 | .. code-block:: bash 125 | 126 | cwltool grep.cwl --input_file underdog_lyrics.txt --pattern lost -------------------------------------------------------------------------------- /doc/source/workflowclasses.rst: -------------------------------------------------------------------------------- 1 | .. python-cwlgen - Python library for manipulation and generation of CWL tools. 2 | 3 | .. _classes: 4 | 5 | ******************** 6 | Workflow API classes 7 | ******************** 8 | 9 | .. _CWLGen - Workflow: 10 | 11 | 12 | CWL Workflow 13 | ============ 14 | 15 | Workflow 16 | """""""" 17 | 18 | .. autoclass:: cwlgen.workflow.Workflow 19 | :members: 20 | :private-members: 21 | :special-members: 22 | :exclude-members: __weakref__ 23 | 24 | 25 | Inputs and Outputs 26 | ================== 27 | 28 | InputParameter 29 | """""""""""""" 30 | 31 | .. autoclass:: cwlgen.workflow.InputParameter 32 | :members: 33 | :private-members: 34 | :special-members: 35 | :exclude-members: __weakref__ 36 | 37 | WorkflowStep 38 | """""""""""" 39 | 40 | .. autoclass:: cwlgen.workflow.WorkflowStep 41 | :members: 42 | :private-members: 43 | :special-members: 44 | :exclude-members: __weakref__ -------------------------------------------------------------------------------- /examples/grep.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | 3 | $namespaces: 4 | s: http://schema.org/ 5 | baseCommand: grep 6 | class: CommandLineTool 7 | cwlVersion: v1.0 8 | doc: grep searches for a pattern in a file. 9 | id: grep 10 | inputs: 11 | - doc: input file from which you want to look for the pattern 12 | id: input_file 13 | inputBinding: 14 | position: 2 15 | type: File 16 | - doc: pattern to find in the input file 17 | id: pattern 18 | inputBinding: 19 | position: 1 20 | type: string 21 | label: print lines matching a pattern 22 | metadata: 23 | about: grep searches for a pattern in a file. 24 | name: grep 25 | outputs: 26 | - doc: lines found with the pattern 27 | id: output 28 | type: stdout 29 | s:about: grep searches for a pattern in a file. 30 | s:name: grep 31 | stdout: grep.txt 32 | -------------------------------------------------------------------------------- /examples/grep.txt: -------------------------------------------------------------------------------- 1 | Feels like I'm lost in a moment 2 | It feels like I'm lost in a moment 3 | -------------------------------------------------------------------------------- /examples/grep_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ## Author(s): Kenzo-Hugo Hillion 4 | ## Contact(s): kehillio@pasteur.fr 5 | ## Python version: 3.6.0 6 | ## Creation : 12-30-2016 7 | 8 | ''' 9 | Example of usage of the cwlgen library 10 | ''' 11 | 12 | ########### Import ########### 13 | 14 | import cwlgen 15 | 16 | if __name__ == "__main__": 17 | 18 | # Create a tool 19 | cwl_tool = cwlgen.CommandLineTool(tool_id='grep', 20 | label='print lines matching a pattern', 21 | base_command='grep') 22 | 23 | # Add 2 inputs (input file and pattern) 24 | file_binding = cwlgen.CommandLineBinding(position=2) 25 | input_file = cwlgen.CommandInputParameter('input_file', 26 | param_type='File', 27 | input_binding=file_binding, 28 | doc='input file from which you want to look for the pattern') 29 | cwl_tool.inputs.append(input_file) 30 | pattern_binding = cwlgen.CommandLineBinding(position=1) 31 | pattern = cwlgen.CommandInputParameter('pattern', 32 | param_type='string', 33 | input_binding=pattern_binding, 34 | doc='pattern to find in the input file') 35 | cwl_tool.inputs.append(pattern) 36 | 37 | # Add 1 output 38 | output = cwlgen.CommandOutputParameter('output', 39 | param_type='stdout', 40 | doc='lines found with the pattern') 41 | cwl_tool.outputs.append(output) 42 | cwl_tool.stdout = "grep.txt" 43 | 44 | # Add documentation 45 | cwl_tool.doc = "grep searches for a pattern in a file." 46 | 47 | # Add Metadata 48 | metadata = {'name': 'grep', 49 | 'about': 'grep searches for a pattern in a file.'} 50 | cwl_tool.metadata = cwlgen.Metadata(**metadata) 51 | cwl_tool.metadata = cwlgen.Metadata(**metadata) 52 | 53 | # Write in an output file 54 | cwl_tool.export() 55 | # cwl_tool.export("grep.cwl") 56 | -------------------------------------------------------------------------------- /examples/readme_example.py: -------------------------------------------------------------------------------- 1 | import cwlgen 2 | 3 | tool_object = cwlgen.CommandLineTool(tool_id="echo-tool", base_command="echo", label=None, doc=None, 4 | cwl_version="v1.0", stdin=None, stderr=None, stdout=None, path=None) 5 | tool_object.inputs.append( 6 | cwlgen.CommandInputParameter("myParamId", label=None, secondary_files=None, param_format=None, 7 | streamable=None, doc=None, input_binding=None, default=None, param_type="string") 8 | ) 9 | 10 | # to get the dictionary representation: 11 | dict_to_export = tool_object.get_dict() 12 | 13 | # to get the string representation (YAML) 14 | yaml_export = tool_object.export_string() 15 | 16 | # print to console 17 | tool_object.export() 18 | 19 | tool_object.export("echotool.cwl") 20 | -------------------------------------------------------------------------------- /examples/underdog_lyrics.txt: -------------------------------------------------------------------------------- 1 | Kill me if you dare 2 | Hold my head up everywhere 3 | Keep myself riding down this train 4 | I'm the underdog 5 | Live my life like a lullaby 6 | I keep myself riding on this train 7 | I keep myself riding on this train 8 | Love in technicolor, sprayed out on walls 9 | Well I've been pounding at the pavement 10 | 'Til there's nothing at all 11 | I got my cloak and dagger 12 | In a bar room brawl 13 | See the local loves a fighter 14 | Loves a winner to fall 15 | Feels like I'm lost in a moment 16 | But I'm always losing to win 17 | And I can't get away from the moment 18 | Seems like it's time to begin 19 | Kill me if you dare 20 | Hold my head up everywhere 21 | Keep myself riding on this train 22 | I'm the underdog 23 | Live my life on a lullaby 24 | I keep myself riding on this train 25 | I keep myself riding on this train 26 | It don't matter 27 | I won't do what you say 28 | No money and no power 29 | I won't go your way 30 | I can't take for the people 31 | They don't matter at all 32 | Well I've been waiting in the fuckin' shadows 33 | 'Til the day that you fall 34 | It feels like I'm lost in a moment 35 | But I'm always losing to win 36 | Can't get away from the moment 37 | Seems like it's time to begin 38 | Kill me if you dare 39 | Hold my head up everywhere 40 | Keep myself riding on this train 41 | But I'm the underdog 42 | Live my life on a lullaby 43 | I keep myself riding on this train 44 | So tell me if you're down 45 | Throw your weapons to the ground 46 | I keep myself riding on this train 47 | Paper on the wire 48 | Sold your soul for another one 49 | I keep myself riding on this train 50 | (Okay Leicester) 51 | I keep myself riding on this train 52 | 53 | Songwriters: Sergio Pizzorno 54 | Underdog lyrics © Sony/ATV Music Publishing LLC 55 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | six 2 | ruamel.yaml>=0.12.4,<0.16.6 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | exec(open("cwlgen/version.py").read()) 4 | 5 | setup( 6 | name="cwlgen", 7 | version=__version__, 8 | description="Generation of CWL programmatically. Available types: Workflow, CommandLineTool and Requirements", 9 | author="Kenzo-Hugo Hillion and Herve Menager", 10 | author_email="kehillio@pasteur.fr", 11 | long_description=open("./README.md").read(), 12 | long_description_content_type="text/markdown", 13 | license="MIT", 14 | keywords=["cwl"], 15 | install_requires=["ruamel.yaml >= 0.12.4, <= 0.16.5"], 16 | packages=["cwlgen"], 17 | classifiers=[ 18 | "Development Status :: 4 - Beta", 19 | "Topic :: Scientific/Engineering :: Bio-Informatics", 20 | "Intended Audience :: Developers", 21 | "Intended Audience :: Science/Research", 22 | "Environment :: Console", 23 | ], 24 | ) 25 | -------------------------------------------------------------------------------- /test/import_commandlinetool.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | 3 | cwlVersion: v1.0 4 | id: tool_id 5 | label: a_label 6 | baseCommand: share 7 | class: CommandLineTool 8 | doc: "super_doc" 9 | stdin: "in" 10 | stderr: "err" 11 | stdout: "out" 12 | inputs: 13 | INPUT1: 14 | label: label_in 15 | secondaryFiles: 'sec_file_in' 16 | format: format_1930 17 | streamable: True 18 | default: 'def_in' 19 | doc: 'documentation_in' 20 | inputBinding: 21 | loadContents: True 22 | position: 0 23 | prefix: --input 24 | separate: True 25 | itemSeparator: ';' 26 | valueFrom: here 27 | shellQuote: True 28 | type: File 29 | outputs: 30 | OUTPUT1: 31 | label: label_out 32 | secondaryFiles: 'sec_file_out' 33 | format: format_1930 34 | streamable: True 35 | doc: 'documentation_out' 36 | outputBinding: 37 | glob: "find" 38 | loadContents: True 39 | outputEval: "eval" 40 | type: File 41 | -------------------------------------------------------------------------------- /test/import_workflow.cwl: -------------------------------------------------------------------------------- 1 | class: Workflow 2 | cwlVersion: v1.0 3 | doc: This is a documentation string 4 | id: 1stWorkflow 5 | inputs: 6 | tarball: 7 | id: tarball 8 | type: File 9 | default: def_in 10 | doc: documentation_in 11 | format: format_1930 12 | label: label_in 13 | secondaryFiles: sec_file_in 14 | streamable: true 15 | name_of_file_to_extract: 16 | id: name_of_file_to_extract 17 | type: string 18 | outputs: 19 | compiled_class: 20 | id: compiled_class 21 | outputSource: compile/classfile 22 | type: File 23 | steps: 24 | compile: 25 | in: 26 | src: 27 | id: src 28 | source: untar/extracted_file 29 | out: 30 | - classfile 31 | run: arguments.cwl 32 | untar: 33 | in: 34 | extractfile: 35 | id: extractfile 36 | source: name_of_file_to_extract 37 | tarfile: 38 | id: tarfile 39 | source: tarball 40 | out: 41 | - extracted_file 42 | run: tar-param.cwl -------------------------------------------------------------------------------- /test/int_tool.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | 3 | cwlVersion: v1.0 4 | id: tool_id 5 | label: a_label 6 | baseCommand: cat 7 | class: CommandLineTool 8 | doc: "super_doc" 9 | stdin: "in" 10 | stderr: "err" 11 | stdout: "out" 12 | inputs: 13 | INTEGER: 14 | doc: 'documentation_in' 15 | inputBinding: 16 | loadContents: True 17 | position: 0 18 | prefix: --input 19 | separate: True 20 | itemSeparator: ';' 21 | valueFrom: here 22 | shellQuote: True 23 | type: int 24 | outputs: 25 | OUTPUT1: 26 | label: label_out 27 | doc: 'documentation_out' 28 | outputBinding: 29 | glob: "find" 30 | loadContents: True 31 | outputEval: "eval" 32 | type: File 33 | -------------------------------------------------------------------------------- /test/test_export.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | 3 | cwlVersion: v1.0 4 | id: an_id 5 | label: a description with spaces. 6 | baseCommand: a_command 7 | class: CommandLineTool 8 | -------------------------------------------------------------------------------- /test/test_export.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class TestExport(unittest.TestCase): 5 | 6 | def test_workflow_export(self): 7 | import cwlgen 8 | w = cwlgen.Workflow("identifier") 9 | expected = """\ 10 | class: Workflow 11 | cwlVersion: v1.0 12 | id: identifier 13 | inputs: {} 14 | outputs: {} 15 | steps: {} 16 | """ 17 | self.assertEqual(expected, w.export_string()) 18 | 19 | def test_commandlinetool_export(self): 20 | import cwlgen 21 | 22 | c = cwlgen.CommandLineTool("identifier", "echo") 23 | expected = """\ 24 | baseCommand: echo 25 | class: CommandLineTool 26 | cwlVersion: v1.0 27 | id: identifier 28 | inputs: {} 29 | outputs: {} 30 | """ 31 | self.assertEqual(expected, c.export_string()) 32 | -------------------------------------------------------------------------------- /test/test_full_export.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | 3 | cwlVersion: v1.0 4 | id: an_id 5 | label: a description with spaces. 6 | baseCommand: a_command 7 | class: CommandLineTool 8 | doc: documentation 9 | inputs: 10 | an_in_id: 11 | type: File 12 | outputs: 13 | an_out_id: 14 | type: File 15 | -------------------------------------------------------------------------------- /test/test_unit_cwlgen.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Unit tests for cwlgen library 5 | ''' 6 | 7 | # Import ------------------------------ 8 | 9 | # General libraries 10 | import os 11 | import filecmp 12 | import unittest 13 | 14 | # External libraries 15 | import cwlgen 16 | 17 | 18 | # Class(es) ------------------------------ 19 | 20 | class TestCommandLineTool(unittest.TestCase): 21 | 22 | def setUp(self): 23 | self.cwl = cwlgen.CommandLineTool(tool_id='an_id', label='a description ' + \ 24 | 'with spaces.', base_command='a_command') 25 | 26 | self.array_basecommand = cwlgen.CommandLineTool(tool_id="base-command-array", base_command=["base", "command"]) 27 | 28 | def test_init(self): 29 | self.assertEqual(self.cwl.id, 'an_id') 30 | self.assertEqual(self.cwl.label, 'a description with spaces.') 31 | self.assertEqual(self.cwl.baseCommand, 'a_command') 32 | self.assertListEqual(self.cwl.inputs, []) 33 | self.assertListEqual(self.cwl.outputs, []) 34 | self.assertIsNone(self.cwl.doc) 35 | 36 | def test_array_basecommand(self): 37 | dict_test = self.array_basecommand.get_dict() 38 | self.assertIn('baseCommand', dict_test) 39 | 40 | def test_codes(self): 41 | code_tool = cwlgen.CommandLineTool() 42 | code_tool.permanentFailCodes.extend([400, 401, 500]) 43 | code_tool.temporaryFailCodes.append(408) 44 | code_tool.successCodes = [] # empty (falsy) 45 | 46 | dict_test = code_tool.get_dict() 47 | self.assertIn('permanentFailCodes', dict_test) 48 | self.assertEqual(len(dict_test['permanentFailCodes']), 3) 49 | self.assertIn('temporaryFailCodes', dict_test) 50 | self.assertEqual(len(dict_test['temporaryFailCodes']), 1) 51 | self.assertNotIn('successCodes', dict_test) 52 | 53 | """ 54 | def test_export(self): 55 | tmp_file = 'test_export.tmp' 56 | expected_file = os.path.dirname(__file__) + '/test_export.cwl' 57 | self.cwl.export(tmp_file) 58 | try: 59 | self.assertTrue(filecmp.cmp(expected_file, tmp_file)) 60 | finally: 61 | os.remove(tmp_file) 62 | 63 | def test_full_export(self): 64 | tmp_file = 'test_full_export.tmp' 65 | expected_file = os.path.dirname(__file__) + '/test_full_export.cwl' 66 | self.cwl.doc = "documentation" 67 | an_input = cwlgen.CommandInputParameter('an_in_id', param_type='File') 68 | self.cwl.inputs.append(an_input) 69 | an_output = cwlgen.CommandOutputParameter('an_out_id', param_type='File') 70 | self.cwl.outputs.append(an_output) 71 | self.cwl.export(tmp_file) 72 | try: 73 | self.assertTrue(filecmp.cmp(expected_file, tmp_file)) 74 | finally: 75 | os.remove(tmp_file) 76 | """ 77 | 78 | 79 | class TestWorkflow(unittest.TestCase): 80 | 81 | def setUp(self): 82 | self.workflow = cwlgen.Workflow("test-workflow", label="testing-workflow", doc="A method for testing " 83 | "the features of a workflow.") 84 | self.workflow.inputs.append(cwlgen.InputParameter("inputParam1", "input-parameter")) 85 | self.workflow.steps.append(cwlgen.WorkflowStep("step1", run="mytool.cwl")) 86 | self.workflow.outputs.append(cwlgen.WorkflowOutputParameter("outputFile", output_source="step/output")) 87 | 88 | def test_init(self): 89 | self.assertEqual(self.workflow.id, "test-workflow") 90 | self.assertEqual(self.workflow.label, "testing-workflow") 91 | self.assertEqual(self.workflow.doc, "A method for testing the features of a workflow.") 92 | 93 | def test_added_inputs(self): 94 | self.assertEqual(len(self.workflow.inputs), 1) 95 | self.assertEqual(self.workflow.inputs[0].id, "inputParam1") 96 | 97 | def test_added_outputs(self): 98 | self.assertEqual(len(self.workflow.outputs), 1) 99 | self.assertEqual(self.workflow.outputs[0].id, "outputFile") 100 | 101 | def test_added_steps(self): 102 | self.assertEqual(len(self.workflow.steps), 1) 103 | step = self.workflow.steps[0] 104 | self.assertEqual(step.id, "step1") 105 | self.assertEqual(step.run, "mytool.cwl") 106 | 107 | def test_serialization(self): 108 | d = self.workflow.get_dict() 109 | self.assertIsInstance(d, dict) 110 | 111 | 112 | class TestWorkflowStepInput(unittest.TestCase): 113 | def test_optional_source(self): 114 | wsi = cwlgen.WorkflowStepInput("test", None).get_dict() 115 | self.assertNotIn("source", wsi) 116 | 117 | def test_required_source(self): 118 | source = "step_id/source" 119 | wsi = cwlgen.WorkflowStepInput("required-input", source=source).get_dict() 120 | self.assertIn("source", wsi) 121 | self.assertEqual(wsi["source"], source) 122 | 123 | 124 | class TestSubworkflow(unittest.TestCase): 125 | def setUp(self): 126 | self.subworkflow = cwlgen.Workflow("subworkflow", doc="This is the subworkflow") 127 | self.workflow = cwlgen.Workflow("workflow", doc="this will embed this subworkflow") 128 | 129 | self.workflow.steps.append(cwlgen.WorkflowStep("subworkflow-step", run=self.subworkflow)) 130 | 131 | def test_subworkflow(self): 132 | sw = self.workflow.steps[0].run 133 | self.assertIsInstance(sw, cwlgen.Workflow) 134 | self.assertEqual(sw.id, self.subworkflow.id) 135 | 136 | def test_serialization(self): 137 | d = self.workflow.get_dict() 138 | self.assertIn("steps", d) 139 | steps = d["steps"] # this should be a dictionary, so we look at 'key in dict' 140 | step_id = self.workflow.steps[0].id 141 | self.assertIn(step_id, steps) 142 | sw_step = steps[step_id] 143 | self.assertIsInstance(sw_step, dict) 144 | self.assertIn("run", sw_step) 145 | sw = sw_step["run"] 146 | self.assertIn("class", sw) 147 | sw_class = sw["class"] 148 | self.assertEqual(sw_class, "Workflow") 149 | 150 | 151 | class TestParameter(unittest.TestCase): 152 | 153 | def setUp(self): 154 | self.param = cwlgen.Parameter('an_id', param_type='File', label='a_label', \ 155 | doc='a_doc', param_format='a_format', \ 156 | streamable=True, secondary_files='sec_files') 157 | self.nonfile_param = cwlgen.Parameter('non-file', param_type="string", label="a string", 158 | streamable=True, secondary_files=[".txt"], doc="documentation here") 159 | 160 | self.array_param = cwlgen.Parameter('an array', param_type='string[]', label="an array of strings") 161 | 162 | def test_init(self): 163 | self.assertEqual(self.param.id, 'an_id') 164 | self.assertEqual(self.param.type, 'File') 165 | self.assertEqual(self.param.doc, 'a_doc') 166 | self.assertEqual(self.param.format, 'a_format') 167 | self.assertEqual(self.param.label, 'a_label') 168 | self.assertEqual(self.param.secondaryFiles, 'sec_files') 169 | self.assertTrue(self.param.streamable) 170 | 171 | def test_get_dict(self): 172 | dict_test = self.param.get_dict() 173 | self.assertEqual(dict_test['type'], 'File') 174 | self.assertEqual(dict_test['doc'], 'a_doc') 175 | self.assertEqual(dict_test['format'], 'a_format') 176 | self.assertEqual(dict_test['label'], 'a_label') 177 | self.assertEqual(dict_test['secondaryFiles'], 'sec_files') 178 | self.assertTrue(dict_test['streamable']) 179 | 180 | # def test_nonfile_get_dict(self): 181 | # dict_test = self.nonfile_param.get_dict() 182 | # self.assertEqual(dict_test['type'], 'string') 183 | # self.assertEqual(dict_test['doc'], self.nonfile_param.doc) 184 | # self.assertNotIn('secondaryFiles', dict_test) 185 | # self.assertNotIn('streamable', dict_test) 186 | # self.assertNotIn('format', dict_test) 187 | 188 | def test_array(self): 189 | dict_test = self.array_param.get_dict() 190 | td = dict_test['type'] 191 | self.assertIsInstance(td, dict) 192 | self.assertEqual(td['type'], 'array') 193 | self.assertEqual(td['items'], self.array_param.type.items) 194 | 195 | 196 | class TestCommandInputParameter(unittest.TestCase): 197 | 198 | def setUp(self): 199 | self.frst_inp = cwlgen.CommandInputParameter('frst_id', param_type='File', \ 200 | label='a_label', \ 201 | default='def_value') 202 | binding = cwlgen.CommandLineBinding(position=2, prefix='--prefix') 203 | self.scnd_inp = cwlgen.CommandInputParameter('scnd_id', param_type='File', \ 204 | input_binding=binding) 205 | 206 | def test_init(self): 207 | # Test first input 208 | self.assertEqual(self.frst_inp.id, 'frst_id') 209 | self.assertEqual(self.frst_inp.type, 'File') 210 | self.assertEqual(self.frst_inp.label, 'a_label') 211 | self.assertEqual(self.frst_inp.default, 'def_value') 212 | # Test second input 213 | self.assertEqual(self.scnd_inp.id, 'scnd_id') 214 | self.assertEqual(self.scnd_inp.type, 'File') 215 | self.assertEqual(self.scnd_inp.inputBinding.position, 2) 216 | self.assertEqual(self.scnd_inp.inputBinding.prefix, '--prefix') 217 | 218 | def test_get_dict(self): 219 | # Test first input 220 | dict_frst = self.frst_inp.get_dict() 221 | self.assertEqual(dict_frst['type'], 'File') 222 | self.assertEqual(dict_frst['default'], 'def_value') 223 | self.assertEqual(dict_frst['label'], 'a_label') 224 | # Test second input 225 | dict_scnd = self.scnd_inp.get_dict() 226 | self.assertEqual(dict_scnd['type'], 'File') 227 | self.assertEqual(dict_scnd['inputBinding']['prefix'], '--prefix') 228 | self.assertEqual(dict_scnd['inputBinding']['position'], 2) 229 | 230 | def test_default(self): 231 | df = "DefaultParam" 232 | t = cwlgen.CommandInputParameter("has-default", default=df) 233 | d = t.get_dict() 234 | self.assertIn("default", d) 235 | self.assertEqual(t.default, d["default"]) 236 | 237 | 238 | class TestStep(unittest.TestCase): 239 | 240 | def test_ignore_id(self): 241 | step = cwlgen.WorkflowStep("identifier", "run") 242 | d = step.get_dict() 243 | self.assertNotIn("id", d) 244 | self.assertNotIn("ignore_attributes", d) 245 | 246 | def test_include_id(self): 247 | step = cwlgen.WorkflowStep("identifier", "run") 248 | step.ignore_attributes = None 249 | d = step.get_dict() 250 | self.assertIn("id", d) 251 | self.assertNotIn("ignore_attributes", d) 252 | 253 | 254 | class TestCommandOutputParameter(unittest.TestCase): 255 | 256 | def setUp(self): 257 | self.outp = cwlgen.CommandOutputParameter('an_out_id', param_type='File') 258 | 259 | def test_init(self): 260 | self.assertEqual(self.outp.id, 'an_out_id') 261 | self.assertEqual(self.outp.type, 'File') 262 | 263 | def test_get_dict(self): 264 | dict_test = self.outp.get_dict() 265 | self.assertEqual(dict_test['type'], 'File') 266 | 267 | def test_empty_outputbinding(self): 268 | s = cwlgen.CommandOutputParameter("empty", output_binding=cwlgen.CommandOutputBinding()).get_dict() 269 | self.assertNotIn('outputBinding', s) 270 | self.assertEqual(s, {'id': 'empty'}) 271 | 272 | def test_nonempty_outputbinding(self): 273 | s = cwlgen.CommandOutputParameter( 274 | "nonempty", 275 | output_binding=cwlgen.CommandOutputBinding(glob="$(inputs.test)")).get_dict() 276 | self.assertEqual(s, {'id': 'nonempty', 'outputBinding': {'glob': '$(inputs.test)'}}) 277 | 278 | 279 | class TestCommandLineBinding(unittest.TestCase): 280 | 281 | def setUp(self): 282 | self.line_binding = cwlgen.CommandLineBinding(load_contents=True, position=1, prefix='--prefix', separate=True, 283 | item_separator='-', shell_quote=True, value_from='text.txt') 284 | self.false_line_binding = cwlgen.CommandLineBinding(load_contents=False, position=0, prefix='', separate=False, 285 | item_separator='', shell_quote=False, value_from="") 286 | 287 | def test_init(self): 288 | self.assertTrue(self.line_binding.loadContents) 289 | self.assertEqual(self.line_binding.position, 1) 290 | self.assertEqual(self.line_binding.prefix, '--prefix') 291 | self.assertTrue(self.line_binding.separate) 292 | self.assertEqual(self.line_binding.itemSeparator, '-') 293 | self.assertTrue(self.line_binding.shellQuote) 294 | self.assertEqual(self.line_binding.valueFrom, 'text.txt') 295 | 296 | def test_get_dict(self): 297 | dict_test = self.line_binding.get_dict() 298 | self.assertEqual(dict_test['position'], 1) 299 | self.assertEqual(dict_test['prefix'], '--prefix') 300 | self.assertTrue(dict_test['separate']) 301 | self.assertEqual(dict_test['itemSeparator'], '-') 302 | self.assertTrue(dict_test['shellQuote']) 303 | self.assertEqual(dict_test['valueFrom'], 'text.txt') 304 | 305 | def test_false_dict(self): 306 | d = self.false_line_binding.get_dict() 307 | self.assertIn("loadContents", d) 308 | self.assertIn("position", d) 309 | self.assertIn("prefix", d) 310 | self.assertIn("separate", d) 311 | self.assertIn("itemSeparator", d) 312 | self.assertIn("shellQuote", d) 313 | self.assertIn("valueFrom", d) 314 | 315 | 316 | class TestCommandOutputBinding(unittest.TestCase): 317 | 318 | def setUp(self): 319 | self.out_binding = cwlgen.CommandOutputBinding(glob='file.txt', load_contents=True, \ 320 | output_eval='eval') 321 | 322 | def test_init(self): 323 | self.assertEqual(self.out_binding.glob, 'file.txt') 324 | self.assertTrue(self.out_binding.loadContents) 325 | self.assertEqual(self.out_binding.outputEval, 'eval') 326 | 327 | def test_get_dict(self): 328 | dict_test = self.out_binding.get_dict() 329 | self.assertEqual(dict_test['glob'], 'file.txt') 330 | self.assertTrue(dict_test['loadContents']) 331 | self.assertEqual(dict_test['outputEval'], 'eval') 332 | 333 | ########### Main ########### 334 | 335 | # if __name__ == "__main__": 336 | # unittest.main() 337 | -------------------------------------------------------------------------------- /test/test_unit_import_tool.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Unit tests for import feature of cwlgen library 5 | ''' 6 | 7 | # Import ------------------------------ 8 | 9 | # General libraries 10 | from os import path 11 | import filecmp 12 | import unittest 13 | import ruamel.yaml as ryaml 14 | 15 | # External libraries 16 | import cwlgen.requirements as Requirements 17 | from cwlgen.import_cwl import parse_cwl, parse_cwl_string 18 | 19 | # Class(es) ------------------------------ 20 | 21 | test_dir = path.dirname(path.abspath(__file__)) 22 | 23 | 24 | class TestImport(unittest.TestCase): 25 | 26 | # def setUp(self): 27 | # ctp = CWLToolParser() 28 | # self.tool = ctp.import_cwl(test_dir + '/import_cwl.cwl') 29 | 30 | @classmethod 31 | def setUpClass(cls): 32 | cls.path = test_dir + '/import_commandlinetool.cwl' 33 | cls.tool = parse_cwl(cls.path) 34 | 35 | 36 | class TestImportCWL(TestImport): 37 | 38 | def test_load_id(self): 39 | self.assertEqual(self.tool.id, 'tool_id') 40 | 41 | def test_load_baseCommand(self): 42 | self.assertEqual(self.tool.baseCommand, 'share') 43 | 44 | def test_load_label(self): 45 | self.assertEqual(self.tool.label, 'a_label') 46 | 47 | def test_load_doc(self): 48 | self.assertEqual(self.tool.doc, 'super_doc') 49 | 50 | def test_load_cwlVersion(self): 51 | self.assertEqual(self.tool.cwlVersion, 'v1.0') 52 | 53 | def test_load_stdin(self): 54 | self.assertEqual(self.tool.stdin, 'in') 55 | 56 | def test_load_stderr(self): 57 | self.assertEqual(self.tool.stderr, 'err') 58 | 59 | def test_load_stdout(self): 60 | self.assertEqual(self.tool.stdout, 'out') 61 | 62 | 63 | class TestInputsParser(TestImport): 64 | 65 | def test_init_input(self): 66 | self.assertEqual(self.tool.inputs[0].id, 'INPUT1') 67 | 68 | def test_load_label(self): 69 | self.assertEqual(self.tool.inputs[0].label, 'label_in') 70 | 71 | def test_load_secondaryFiles(self): 72 | self.assertEqual(self.tool.inputs[0].secondaryFiles, 'sec_file_in') 73 | 74 | def test_load_format(self): 75 | self.assertEqual(self.tool.inputs[0].format, 'format_1930') 76 | 77 | def test_load_streamable(self): 78 | self.assertTrue(self.tool.inputs[0].streamable) 79 | 80 | def test_load_doc(self): 81 | self.assertEqual(self.tool.inputs[0].doc, 'documentation_in') 82 | 83 | def test_load_default(self): 84 | self.assertEqual(self.tool.inputs[0].default, 'def_in') 85 | 86 | def test_load_type(self): 87 | self.assertEqual(self.tool.inputs[0].type, 'File') 88 | 89 | 90 | class TestInputBindingParser(TestImport): 91 | 92 | def test_load_loadContents(self): 93 | self.assertTrue(self.tool.inputs[0].inputBinding.loadContents) 94 | 95 | def test_load_position(self): 96 | self.assertEqual(self.tool.inputs[0].inputBinding.position, 0) 97 | 98 | def test_load_prefix(self): 99 | self.assertEqual(self.tool.inputs[0].inputBinding.prefix, '--input') 100 | 101 | def test_load_separate(self): 102 | self.assertTrue(self.tool.inputs[0].inputBinding.separate) 103 | 104 | def test_load_itemSeparator(self): 105 | self.assertEqual(self.tool.inputs[0].inputBinding.itemSeparator, ';') 106 | 107 | def test_load_valueFrom(self): 108 | self.assertEqual(self.tool.inputs[0].inputBinding.valueFrom, 'here') 109 | 110 | def test_load_shellQuote(self): 111 | self.assertTrue(self.tool.inputs[0].inputBinding.shellQuote) 112 | 113 | 114 | class TestOutputsParser(TestImport): 115 | 116 | def test_init_input(self): 117 | self.assertEqual(self.tool.outputs[0].id, 'OUTPUT1') 118 | 119 | def test_load_label(self): 120 | self.assertEqual(self.tool.outputs[0].label, 'label_out') 121 | 122 | def test_load_secondaryFiles(self): 123 | self.assertEqual(self.tool.outputs[0].secondaryFiles, 'sec_file_out') 124 | 125 | def test_load_format(self): 126 | self.assertEqual(self.tool.outputs[0].format, 'format_1930') 127 | 128 | def test_load_streamable(self): 129 | self.assertTrue(self.tool.outputs[0].streamable) 130 | 131 | def test_load_doc(self): 132 | self.assertEqual(self.tool.outputs[0].doc, 'documentation_out') 133 | 134 | def test_load_type(self): 135 | self.assertEqual(self.tool.outputs[0].type, 'File') 136 | 137 | 138 | class TestOutputBindingParser(TestImport): 139 | 140 | def test_load_glob(self): 141 | self.assertEqual(self.tool.outputs[0].outputBinding.glob, 'find') 142 | 143 | def test_load_loadContents(self): 144 | self.assertTrue(self.tool.outputs[0].outputBinding.loadContents) 145 | 146 | def test_load_outputEval(self): 147 | self.assertEqual(self.tool.outputs[0].outputBinding.outputEval, "eval") 148 | 149 | 150 | class TestIntegrationImportWorkflow(unittest.TestCase): 151 | def test_issue_25_requirements(self): 152 | wfstr = """\ 153 | class: CommandLineTool 154 | cwlVersion: v1.0 155 | 156 | inputs: {} 157 | outputs: {} 158 | 159 | requirements: 160 | - class: MultipleInputFeatureRequirement 161 | - class: StepInputExpressionRequirement 162 | - class: SubworkflowFeatureRequirement""" 163 | wf = parse_cwl_string(wfstr) 164 | self.assertEqual(3, len(wf.requirements)) 165 | self.assertIsInstance(wf.requirements[0], Requirements.MultipleInputFeatureRequirement) 166 | self.assertIsInstance(wf.requirements[1], Requirements.StepInputExpressionRequirement) 167 | self.assertIsInstance(wf.requirements[2], Requirements.SubworkflowFeatureRequirement) 168 | 169 | wfd = wf.get_dict() 170 | expected = { 171 | 'MultipleInputFeatureRequirement': {}, 172 | 'StepInputExpressionRequirement': {}, 173 | 'SubworkflowFeatureRequirement': {} 174 | } 175 | self.assertDictEqual(expected, wfd["requirements"]) 176 | 177 | def test_issue_25_requirements_dict(self): 178 | wfstr = """\ 179 | class: CommandLineTool 180 | cwlVersion: v1.0 181 | 182 | inputs: {} 183 | outputs: {} 184 | 185 | requirements: 186 | MultipleInputFeatureRequirement: {} 187 | StepInputExpressionRequirement: {} 188 | SubworkflowFeatureRequirement: {}""" 189 | wf = parse_cwl_string(wfstr) 190 | self.assertEqual(3, len(wf.requirements)) 191 | 192 | wfd = wf.get_dict() 193 | expected = { 194 | 'MultipleInputFeatureRequirement': {}, 195 | 'StepInputExpressionRequirement': {}, 196 | 'SubworkflowFeatureRequirement': {} 197 | } 198 | self.assertDictEqual(expected, wfd["requirements"]) 199 | -------------------------------------------------------------------------------- /test/test_unit_import_workflow.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Unit tests for import feature of cwlgen library 5 | ''' 6 | 7 | # Import ------------------------------ 8 | 9 | import unittest 10 | # General libraries 11 | from os import path 12 | 13 | import ruamel.yaml as ryaml 14 | 15 | # External libraries 16 | import cwlgen.requirements as Requirements 17 | from cwlgen.import_cwl import parse_cwl_dict, parse_cwl_string 18 | 19 | # Class(es) ------------------------------ 20 | 21 | # Use this to ensure location to import_workflow is correct when directly running the file. 22 | test_dir = path.dirname(path.abspath(__file__)) 23 | 24 | 25 | class TestImport(unittest.TestCase): 26 | 27 | # def setUp(self): 28 | # ctp = CWLToolParser() 29 | # self.wf = ctp.import_cwl(test_dir + '/import_cwl.cwl') 30 | 31 | @classmethod 32 | def setUpClass(cls): 33 | cls.path = test_dir + '/import_workflow.cwl' 34 | with open(cls.path) as mf: 35 | cls.wf_dict = ryaml.load(mf, Loader=ryaml.Loader) 36 | cls.wf = parse_cwl_dict(cls.wf_dict) 37 | 38 | 39 | class TestImportCWL(TestImport): 40 | 41 | # def test_export_loaded(self): 42 | # self.maxDiff = None 43 | # print(self.wf_dict) 44 | # print(self.wf.get_dict()) 45 | # self.assertDictEqual(self.wf_dict, self.wf.get_dict()) 46 | 47 | def test_load_id(self): 48 | self.assertEqual(self.wf.id, '1stWorkflow') 49 | 50 | def test_load_class(self): 51 | self.assertEqual(self.wf.cwlVersion, 'v1.0') 52 | 53 | def test_load_doc(self): 54 | self.assertEqual(self.wf.doc, 'This is a documentation string') 55 | 56 | 57 | class TestInputsParser(TestImport): 58 | 59 | def test_init_input(self): 60 | self.assertEqual(self.wf.inputs[0].id, 'tarball') 61 | 62 | def test_load_label(self): 63 | self.assertEqual(self.wf.inputs[0].label, 'label_in') 64 | 65 | def test_load_secondaryFiles(self): 66 | self.assertEqual(self.wf.inputs[0].secondaryFiles, 'sec_file_in') 67 | 68 | def test_load_format(self): 69 | self.assertEqual(self.wf.inputs[0].format, 'format_1930') 70 | 71 | def test_load_streamable(self): 72 | self.assertTrue(self.wf.inputs[0].streamable) 73 | 74 | def test_load_doc(self): 75 | self.assertEqual(self.wf.inputs[0].doc, 'documentation_in') 76 | 77 | def test_load_default(self): 78 | self.assertEqual(self.wf.inputs[0].default, 'def_in') 79 | 80 | def test_load_type(self): 81 | self.assertEqual(self.wf.inputs[0].type, 'File') 82 | 83 | 84 | class TestOutputsParser(TestImport): 85 | 86 | def test_init_input(self): 87 | self.assertEqual(self.wf.outputs[0].id, 'compiled_class') 88 | 89 | def test_load_label(self): 90 | self.assertEqual(self.wf.outputs[0].outputSource, 'compile/classfile') 91 | 92 | def test_load_secondaryFiles(self): 93 | self.assertEqual(self.wf.outputs[0].type, 'File') 94 | 95 | 96 | class TestStepParser(TestImport): 97 | 98 | def test_load_id(self): 99 | self.assertEqual(self.wf.steps[0].id, 'compile') 100 | 101 | def test_load_out(self): 102 | self.assertListEqual(self.wf.steps[0].out, ['classfile']) 103 | 104 | 105 | class TestStepInputParser(TestImport): 106 | 107 | def test_load_id(self): 108 | self.assertEqual(self.wf.steps[0].inputs[0].id, "src") 109 | 110 | def test_load_source(self): 111 | self.assertEqual(self.wf.steps[0].inputs[0].source, "untar/extracted_file") 112 | 113 | 114 | class TestIntegrationImportWorkflow(unittest.TestCase): 115 | def test_issue_25_requirements(self): 116 | wfstr = """\ 117 | class: Workflow 118 | cwlVersion: v1.0 119 | 120 | label: joint calling workflow 121 | doc: Perform joint calling on multiple sets aligned reads from the same family. 122 | 123 | inputs: {} 124 | steps: {} 125 | outputs: {} 126 | 127 | requirements: 128 | - class: MultipleInputFeatureRequirement 129 | - class: StepInputExpressionRequirement 130 | - class: SubworkflowFeatureRequirement""" 131 | wf = parse_cwl_string(wfstr) 132 | self.assertEqual(3, len(wf.requirements)) 133 | self.assertIsInstance(wf.requirements[0], Requirements.MultipleInputFeatureRequirement) 134 | self.assertIsInstance(wf.requirements[1], Requirements.StepInputExpressionRequirement) 135 | self.assertIsInstance(wf.requirements[2], Requirements.SubworkflowFeatureRequirement) 136 | 137 | wfd = wf.get_dict() 138 | expected = { 139 | 'MultipleInputFeatureRequirement': {}, 140 | 'StepInputExpressionRequirement': {}, 141 | 'SubworkflowFeatureRequirement': {} 142 | } 143 | self.assertDictEqual(expected, wfd["requirements"]) 144 | 145 | def test_issue_25_requirements_dict(self): 146 | wfstr = """\ 147 | class: Workflow 148 | cwlVersion: v1.0 149 | 150 | label: joint calling workflow 151 | doc: Perform joint calling on multiple sets aligned reads from the same family. 152 | 153 | inputs: {} 154 | outputs: {} 155 | steps: {} 156 | 157 | requirements: 158 | MultipleInputFeatureRequirement: {} 159 | StepInputExpressionRequirement: {} 160 | SubworkflowFeatureRequirement: {}""" 161 | wf = parse_cwl_string(wfstr) 162 | self.assertEqual(3, len(wf.requirements)) 163 | 164 | wfd = wf.get_dict() 165 | expected = { 166 | 'MultipleInputFeatureRequirement': {}, 167 | 'StepInputExpressionRequirement': {}, 168 | 'SubworkflowFeatureRequirement': {} 169 | } 170 | self.assertDictEqual(expected, wfd["requirements"]) 171 | -------------------------------------------------------------------------------- /test/test_unit_requirements.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import cwlgen 4 | 5 | 6 | class TestAddRequirements(unittest.TestCase): 7 | 8 | def test_inlinejs(self): 9 | w = cwlgen.Workflow() 10 | req = cwlgen.InlineJavascriptRequirement(["expression"]) 11 | w.requirements.append(req) 12 | d = w.get_dict() 13 | self.assertIn("requirements", d) 14 | dr = d["requirements"] 15 | self.assertIn(req.get_class(), dr) 16 | drr = dr[req.get_class()] 17 | self.assertIn("expressionLib", drr) 18 | self.assertEqual(drr["expressionLib"], ["expression"]) 19 | 20 | def test_commandtool(self): 21 | c = cwlgen.CommandLineTool("reqs") 22 | docker = cwlgen.DockerRequirement(docker_pull="ubuntu:latest") 23 | c.requirements.append(docker) 24 | 25 | d = c.get_dict() 26 | self.assertIn("requirements", d) 27 | self.assertIn(docker.get_class(), d["requirements"]) 28 | 29 | 30 | class TestInlineJavascriptReq(unittest.TestCase): 31 | 32 | def setUp(self): 33 | self.js_req = cwlgen.InlineJavascriptRequirement(expression_lib=['expression']) 34 | self.js_req_nolib = cwlgen.InlineJavascriptRequirement() 35 | 36 | def test_init(self): 37 | self.assertEqual(self.js_req.get_class(), 'InlineJavascriptRequirement') 38 | self.assertEqual(self.js_req.expressionLib, ['expression']) 39 | self.assertEqual(self.js_req_nolib.get_class(), 'InlineJavascriptRequirement') 40 | self.assertIsNone(self.js_req_nolib.expressionLib) 41 | 42 | def test_export(self): 43 | tool = self.js_req.get_dict() 44 | self.assertEqual(tool, {'expressionLib': ['expression']}) 45 | 46 | def test_export_without_lib(self): 47 | tool = self.js_req_nolib.get_dict() 48 | self.assertDictEqual(tool, {}) 49 | 50 | 51 | class TestDockerRequirement(unittest.TestCase): 52 | 53 | def setUp(self): 54 | self.dock_req = cwlgen.DockerRequirement( 55 | docker_pull='pull', 56 | docker_load='load', 57 | docker_file='file', 58 | docker_import='import', 59 | docker_image_id='id', 60 | docker_output_dir='dir' 61 | ) 62 | 63 | def test_init(self): 64 | self.assertEqual(self.dock_req.get_class(), 'DockerRequirement') 65 | self.assertEqual(self.dock_req.dockerPull, 'pull') 66 | self.assertEqual(self.dock_req.dockerLoad, 'load') 67 | self.assertEqual(self.dock_req.dockerFile, 'file') 68 | self.assertEqual(self.dock_req.dockerImport, 'import') 69 | self.assertEqual(self.dock_req.dockerImageId, 'id') 70 | self.assertEqual(self.dock_req.dockerOutputDirectory, 'dir') 71 | 72 | def test_export(self): 73 | d = self.dock_req.get_dict() 74 | comparison = { 75 | 'dockerFile': 'file', 76 | 'dockerImageId': 'id', 77 | 'dockerImport': 'import', 78 | 'dockerLoad': 'load', 79 | 'dockerOutputDirectory': 'dir', 80 | 'dockerPull': 'pull' 81 | } 82 | self.assertEqual(d, comparison) 83 | 84 | def test_missing_export(self): 85 | dr = cwlgen.DockerRequirement() 86 | self.assertDictEqual(dr.get_dict(), {}) 87 | 88 | 89 | class TestSubworkflowFeatureRequirement(unittest.TestCase): 90 | 91 | def setUp(self): 92 | self.req = cwlgen.SubworkflowFeatureRequirement() 93 | 94 | def test_init(self): 95 | self.assertEqual(self.req.get_class(), 'SubworkflowFeatureRequirement') 96 | 97 | def test_add(self): 98 | tool = self.req.get_dict() 99 | self.assertDictEqual(tool, {}) 100 | 101 | 102 | class TestSoftwareRequirement(unittest.TestCase): 103 | 104 | def setUp(self): 105 | self.r = cwlgen.SoftwareRequirement() 106 | self.r.packages.append(cwlgen.SoftwareRequirement.SoftwarePackage("package", "version", None)) 107 | 108 | def test_properties(self): 109 | self.assertEqual(self.r.get_class(), "SoftwareRequirement") 110 | self.assertEqual(len(self.r.packages), 1) 111 | p = self.r.packages[0] 112 | self.assertEqual(p.package, "package") 113 | self.assertEqual(p.version, "version") 114 | self.assertIsNone(p.specs) 115 | 116 | def test_serialization(self): 117 | d = self.r.get_dict() 118 | self.assertIn("packages", d) 119 | pks = d["packages"] 120 | self.assertEqual(len(pks), 1) 121 | p = pks[0] 122 | self.assertEqual(p.get("package"), "package") 123 | self.assertEqual(p.get("version"), "version") 124 | self.assertNotIn("specs", p) 125 | 126 | 127 | class TestInitialWorkDirRequirement(unittest.TestCase): 128 | 129 | def setUp(self): 130 | self.i1 = cwlgen.InitialWorkDirRequirement("listing1") 131 | self.i2 = cwlgen.InitialWorkDirRequirement([ 132 | cwlgen.InitialWorkDirRequirement.Dirent("entry", "entryname") 133 | ]) 134 | 135 | def test_properties_i1(self): 136 | self.assertEqual(self.i1.get_class(), "InitialWorkDirRequirement") 137 | self.assertEqual(self.i1.listing, "listing1") 138 | 139 | def test_properties_i2(self): 140 | self.assertEqual(len(self.i2.listing), 1) 141 | l = self.i2.listing[0] 142 | self.assertEqual(l.entry, "entry") 143 | self.assertEqual(l.entryname, "entryname") 144 | self.assertIsNone(l.writable) 145 | 146 | def test_serialization_i1(self): 147 | d = self.i1.get_dict() 148 | self.assertIn("listing", d) 149 | self.assertIsInstance(d["listing"], str) 150 | self.assertEqual(d["listing"], "listing1") 151 | 152 | def test_serialization_i2(self): 153 | d = self.i2.get_dict() 154 | self.assertIn("listing", d) 155 | listing = d.get("listing") 156 | self.assertIsInstance(listing, list) 157 | self.assertEqual(len(listing), 1) 158 | l = listing[0] 159 | self.assertIn("entry", l) 160 | self.assertIn("entryname", l) 161 | self.assertNotIn("writable", l) 162 | 163 | self.assertEqual(l["entry"], "entry") 164 | self.assertEqual(l["entryname"], "entryname") 165 | 166 | 167 | class TestScatterFeatureRequirement(unittest.TestCase): 168 | 169 | def setUp(self): 170 | self.req = cwlgen.ScatterFeatureRequirement() 171 | 172 | def test_init(self): 173 | self.assertEqual(self.req.get_class(), 'ScatterFeatureRequirement') 174 | 175 | def test_add(self): 176 | tool = self.req.get_dict() 177 | self.assertDictEqual(tool, {}) 178 | 179 | 180 | class TestStepInputExpressionRequirement(unittest.TestCase): 181 | 182 | def setUp(self): 183 | self.req = cwlgen.StepInputExpressionRequirement() 184 | 185 | def test_init(self): 186 | self.assertEqual(self.req.get_class(), 'StepInputExpressionRequirement') 187 | 188 | def test_add(self): 189 | tool = self.req.get_dict() 190 | self.assertDictEqual(tool, {}) 191 | 192 | 193 | class TestEnvVarRequirement(unittest.TestCase): 194 | 195 | def setUp(self): 196 | self.e = cwlgen.EnvVarRequirement([ 197 | cwlgen.EnvVarRequirement.EnvironmentDef("env_name", "env_value") 198 | ]) 199 | 200 | def test_properties(self): 201 | self.assertEqual(self.e.get_class(), "EnvVarRequirement") 202 | self.assertIsInstance(self.e.envDef, list) 203 | self.assertEqual(len(self.e.envDef), 1) 204 | e = self.e.envDef[0] 205 | self.assertEqual(e.envName, "env_name") 206 | self.assertEqual(e.envValue, "env_value") 207 | 208 | def test_serialization(self): 209 | d = self.e.get_dict() 210 | self.assertIn("envDef", d) 211 | ed = d["envDef"] 212 | self.assertIsInstance(ed, list) 213 | self.assertEqual(len(ed), 1) 214 | e = ed[0] 215 | self.assertEqual(e.get("envName"), "env_name") 216 | self.assertEqual(e.get("envValue"), "env_value") 217 | 218 | 219 | class TestShellCommandRequirement(unittest.TestCase): 220 | 221 | def setUp(self): 222 | self.req = cwlgen.ShellCommandRequirement() 223 | 224 | def test_init(self): 225 | self.assertEqual(self.req.get_class(), 'ShellCommandRequirement') 226 | 227 | def test_add(self): 228 | tool = self.req.get_dict() 229 | self.assertDictEqual(tool, {}) 230 | 231 | 232 | class TestResourceRequirement(unittest.TestCase): 233 | 234 | def setUp(self): 235 | self.req = cwlgen.ResourceRequirement( 236 | cores_min=1, cores_max=3, 237 | ram_min=2, ram_max=4, 238 | tmpdir_min="tmp/min", tmpdir_max="tmp/max", 239 | outdir_min="out/min", outdir_max="out/max" 240 | ) 241 | self.empty = cwlgen.ResourceRequirement() 242 | 243 | def test_init(self): 244 | self.assertEqual(self.req.get_class(), 'ResourceRequirement') 245 | 246 | self.assertEqual(1, self.req.coresMin) 247 | self.assertEqual(3, self.req.coresMax) 248 | 249 | self.assertEqual(2, self.req.ramMin) 250 | self.assertEqual(4, self.req.ramMax) 251 | 252 | self.assertEqual("tmp/min", self.req.tmpdirMin) 253 | self.assertEqual("tmp/max", self.req.tmpdirMax) 254 | self.assertEqual("out/min", self.req.outdirMin) 255 | self.assertEqual("out/max", self.req.outdirMax) 256 | 257 | def test_empty(self): 258 | self.assertDictEqual(self.empty.get_dict(), {}) 259 | 260 | def test_add(self): 261 | r = self.req.get_dict() 262 | self.assertEqual(r.get("coresMin"), 1) 263 | self.assertEqual(r.get("coresMax"), 3) 264 | 265 | self.assertEqual(r.get("ramMin"), 2) 266 | self.assertEqual(r.get("ramMax"), 4) 267 | 268 | self.assertEqual(r.get("tmpdirMin"), "tmp/min") 269 | self.assertEqual(r.get("tmpdirMax"), "tmp/max") 270 | self.assertEqual(r.get("outdirMin"), "out/min") 271 | self.assertEqual(r.get("outdirMax"), "out/max") 272 | 273 | 274 | class TestParseRequirements(unittest.TestCase): 275 | 276 | def test_parse_requirement_docker(self): 277 | d = { 278 | "class": "DockerRequirement", 279 | "dockerPull": "ubuntu/latest" 280 | } 281 | 282 | req = cwlgen.Requirement.parse_dict(d) 283 | self.assertIsInstance(req, cwlgen.DockerRequirement) 284 | self.assertEqual(d["dockerPull"], req.dockerPull) 285 | 286 | def test_parse_expression_lib(self): 287 | d = { 288 | "class": "InlineJavascriptRequirement", 289 | "expressionLib": ["expression", "lib"] 290 | } 291 | req = cwlgen.Requirement.parse_dict(d) 292 | self.assertIsInstance(req, cwlgen.InlineJavascriptRequirement) 293 | self.assertEqual(d["expressionLib"], req.expressionLib) 294 | 295 | def test_parse_software_requirement(self): 296 | d = { 297 | "class": "SoftwareRequirement", 298 | "packages": [{ 299 | "package": "Half-Life", 300 | "version": 3, 301 | "specs": "Classified" 302 | }] 303 | } 304 | req = cwlgen.Requirement.parse_dict(d) 305 | self.assertIsInstance(req, cwlgen.SoftwareRequirement) 306 | hl3 = req.packages[0] 307 | self.assertIsInstance(hl3, cwlgen.SoftwareRequirement.SoftwarePackage) 308 | self.assertEqual(d["packages"][0]["package"], hl3.package) 309 | self.assertEqual(d["packages"][0]["version"], hl3.version) 310 | self.assertEqual(d["packages"][0]["specs"], hl3.specs) 311 | 312 | def test_parse_initialworkdir_requirement_1(self): 313 | d = { 314 | "class": "InitialWorkDirRequirement", 315 | "listing": [{ 316 | "entry": "test1", 317 | "entryname": "test", 318 | "writable": True 319 | }] 320 | } 321 | req = cwlgen.Requirement.parse_dict(d) 322 | self.assertIsInstance(req, cwlgen.InitialWorkDirRequirement) 323 | l = req.listing[0] 324 | self.assertIsInstance(l, cwlgen.InitialWorkDirRequirement.Dirent) 325 | self.assertEqual(d["listing"][0]["entry"], l.entry) 326 | self.assertEqual(d["listing"][0]["entryname"], l.entryname) 327 | self.assertEqual(d["listing"][0]["writable"], l.writable) 328 | 329 | def test_parse_initialworkdir_requirement_2(self): 330 | d = { 331 | "class": "InitialWorkDirRequirement", 332 | "listing": ["test2"] 333 | } 334 | req = cwlgen.Requirement.parse_dict(d) 335 | self.assertIsInstance(req, cwlgen.InitialWorkDirRequirement) 336 | self.assertListEqual(d["listing"], req.listing) 337 | 338 | def test_parse_initialworkdir_requirement_3(self): 339 | d = { 340 | "class": "InitialWorkDirRequirement", 341 | "listing": "test3" 342 | } 343 | req = cwlgen.Requirement.parse_dict(d) 344 | self.assertIsInstance(req, cwlgen.InitialWorkDirRequirement) 345 | self.assertEqual(d["listing"], req.listing) 346 | 347 | # def test_schema_def_requirement_1(self): 348 | # d = { 349 | # "class": "SchemaDefRequirement", 350 | # "types": [ 351 | # { 352 | # "type": "record", 353 | # "name": "test_inputRecordSchema", 354 | # "fields": [{ 355 | # "name": "test_inputRecordSchema_field1", 356 | # "type": "string", 357 | # "inputBinding": None, 358 | # }] 359 | # }, 360 | # { 361 | # "type": "enum", 362 | # "symbols": ["one", "two", "three"], 363 | # "name": "test_inputEnumSchema" 364 | # }, 365 | # { 366 | # "type": "array", 367 | # "items": "string" 368 | # } 369 | # ] 370 | # } 371 | # 372 | # req = cwlgen.Requirement.parse_dict(d) 373 | # print(req) 374 | -------------------------------------------------------------------------------- /test/test_unit_typing.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from cwlgen import get_type_dict 4 | from cwlgen.common import parse_type, CwlTypes, CommandInputArraySchema 5 | import logging 6 | 7 | 8 | class TestParamTyping(unittest.TestCase): 9 | 10 | def test_types(self): 11 | for cwl_type in CwlTypes.NON_NULL_TYPES: 12 | self.assertEqual(parse_type(cwl_type), cwl_type) 13 | 14 | def test_incorrect_type(self): 15 | invalid_type = "invalid" 16 | should_be_def_type = parse_type(invalid_type) 17 | self.assertNotEqual(should_be_def_type, invalid_type) 18 | self.assertEqual(should_be_def_type, CwlTypes.DEF_TYPE) 19 | 20 | def test_optional_string(self): 21 | for cwl_type in CwlTypes.NON_NULL_TYPES: 22 | optional_type = cwl_type + "?" 23 | self.assertEqual(parse_type(optional_type), optional_type) 24 | 25 | def test_typed_array(self): 26 | array_string_type = "string[]" 27 | q = parse_type(array_string_type) 28 | self.assertIsInstance(q, CommandInputArraySchema) 29 | self.assertEqual(q.items, "string") 30 | 31 | def test_incorrectly_typed_array(self): 32 | array_string_type = "invalid[]" 33 | q = parse_type(array_string_type) 34 | self.assertIsInstance(q, CommandInputArraySchema) 35 | self.assertNotEqual(q.items, "invalid") 36 | self.assertEqual(q.items, CwlTypes.DEF_TYPE) 37 | 38 | def test_optionally_typed_array(self): 39 | array_string_type = "string?[]" 40 | q = parse_type(array_string_type) 41 | self.assertIsInstance(q, CommandInputArraySchema) 42 | self.assertEqual(q.items, "string?") 43 | 44 | def test_optional_typed_array(self): 45 | optional_array_string_type = "string[]?" 46 | q = parse_type(optional_array_string_type) 47 | self.assertIsInstance(q, list) 48 | self.assertEqual(len(q), 2) 49 | null_idx = q.index(CwlTypes.DEF_TYPE) 50 | array_type = q[1 - null_idx] 51 | self.assertIsInstance(array_type, CommandInputArraySchema) 52 | self.assertEqual(array_type.items, "string") 53 | 54 | def test_command_input_array_schema(self): 55 | ar = CommandInputArraySchema(items="string") 56 | self.assertIsInstance(ar, CommandInputArraySchema) 57 | self.assertEqual(parse_type(ar), ar) 58 | self.assertEqual(ar.items, "string") 59 | 60 | def test_optional_type_input_array_schema(self): 61 | ar = CommandInputArraySchema(items="string?") 62 | self.assertIsInstance(ar, CommandInputArraySchema) 63 | self.assertEqual(parse_type(ar), ar) 64 | self.assertEqual(ar.items, "string?") 65 | 66 | def test_array_of_array_of_strings(self): 67 | arstr = CommandInputArraySchema(items="string") 68 | ar = CommandInputArraySchema(items=arstr) 69 | self.assertEqual(ar.get_dict(), {'type': 'array', 'items': {'type': 'array', 'items': 'string'}}) 70 | 71 | def test_parse_array_of_string_array(self): 72 | ar = CommandInputArraySchema(items="string[]") 73 | self.assertEqual(ar.get_dict(), {'type': 'array', 'items': {'type': 'array', 'items': 'string'}}) 74 | 75 | def test_parse_list_of_items(self): 76 | t = ["string", "int"] 77 | types = parse_type(t) 78 | self.assertListEqual(t, types) 79 | 80 | def test_intentional_fail(self): 81 | try: 82 | parse_type("not_a_type", requires_type=True) 83 | self.fail("Failed to throw exception") 84 | except Exception as e: 85 | self.assertTrue(True) 86 | 87 | def test_intentional_fail_not_string(self): 88 | try: 89 | parse_type({}, requires_type=True) 90 | self.fail("Failed to throw exception") 91 | except: 92 | self.assertTrue(True) 93 | 94 | def test_get_type_dict(self): 95 | 96 | self.assertEqual("any_string_input", get_type_dict("any_string_input")) 97 | self.assertListEqual(["any", "list", "of", "strings"], get_type_dict(["any", "list", "of", "strings"])) 98 | self.assertDictEqual({"any": "dict"}, get_type_dict({"any": "dict"})) 99 | 100 | try: 101 | get_type_dict(self) 102 | self.assertTrue(False, "Failed to throw exception") 103 | except Exception as e: 104 | self.assertTrue(True) 105 | -------------------------------------------------------------------------------- /test/test_unit_workflow.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Unit tests for cwlgen library 5 | ''' 6 | 7 | # Import ------------------------------ 8 | 9 | # General libraries 10 | import os 11 | import filecmp 12 | import unittest 13 | from tempfile import NamedTemporaryFile 14 | 15 | # External libraries 16 | import cwlgen 17 | 18 | # Class(es) ------------------------------ 19 | 20 | class TWorkflow: 21 | # class TestWorkflow(unittest.TestCase): 22 | 23 | def capture_tempfile(self, func): 24 | with NamedTemporaryFile() as f: 25 | func(f.name) 26 | return f.read() 27 | 28 | def test_generates_workflow_two_steps(self): 29 | 30 | w = cwlgen.Workflow("test_generates_workflow_two_steps") 31 | tool = cwlgen.parse_cwl("test/import_cwl.cwl") 32 | 33 | f = cwlgen.File("input_file") 34 | 35 | o1 = w.add('step-a', tool, {"INPUT1" : f}) 36 | o2 = w.add('step-b', tool, {"INPUT1" : o1['OUTPUT1']}) 37 | o2['OUTPUT1'].store() 38 | generated = self.capture_tempfile(w.export) 39 | expected = b"""#!/usr/bin/env cwl-runner 40 | 41 | class: Workflow 42 | cwlVersion: v1.0 43 | inputs: 44 | INPUT1: {id: INPUT1, type: File} 45 | outputs: 46 | step-b_OUTPUT1: {id: step-b_OUTPUT1, outputSource: step-b/OUTPUT1, type: File} 47 | steps: 48 | step-a: 49 | id: step-a 50 | in: {INPUT1: INPUT1} 51 | out: [OUTPUT1] 52 | run: test/import_cwl.cwl 53 | step-b: 54 | id: step-b 55 | in: {INPUT1: step-a/OUTPUT1} 56 | out: [OUTPUT1] 57 | run: test/import_cwl.cwl 58 | """ 59 | self.assertEqual(expected, generated) 60 | 61 | def test_generates_workflow_int_inputs(self): 62 | 63 | w = cwlgen.Workflow("test_generates_workflow_int_inputs") 64 | tool = cwlgen.parse_cwl("test/int_tool.cwl") 65 | 66 | i = cwlgen.workflow.InputParameter('INTEGER', param_type='int') 67 | o1 = w.add('step', tool, {"INTEGER": i}) 68 | o1['OUTPUT1'].store() 69 | 70 | expected = b"""#!/usr/bin/env cwl-runner 71 | 72 | class: Workflow 73 | cwlVersion: v1.0 74 | inputs: 75 | INTEGER: {id: INTEGER, type: int} 76 | outputs: 77 | step_OUTPUT1: {id: step_OUTPUT1, outputSource: step/OUTPUT1, type: File} 78 | steps: 79 | step: 80 | id: step 81 | in: {INTEGER: INTEGER} 82 | out: [OUTPUT1] 83 | run: test/int_tool.cwl 84 | """ 85 | generated = self.capture_tempfile(w.export) 86 | self.assertEqual(expected, generated) 87 | 88 | def test_add_requirements(self): 89 | w = cwlgen.Workflow("test_add_requirements") 90 | req = cwlgen.InlineJavascriptRequirement() 91 | w.requirements.append(req) 92 | generated = self.capture_tempfile(w.export) 93 | expected = b"""#!/usr/bin/env cwl-runner 94 | 95 | class: Workflow 96 | cwlVersion: v1.0 97 | inputs: {} 98 | outputs: {} 99 | requirements: 100 | InlineJavascriptRequirement: {} 101 | """ 102 | self.assertEqual(expected, generated) 103 | 104 | --------------------------------------------------------------------------------