├── .dockerignore ├── .github └── workflows │ ├── test_backend.yml │ ├── test_docker.yml │ └── test_frontend.yml ├── .gitignore ├── LICENSE ├── Postman ├── Local-env.postman_environment.json ├── Node-endpoints.postman_collection.json ├── PyWorkflow-runner.postman_collection.json └── Workflow-endpoints.postman_collection.json ├── README.md ├── back-end ├── CLI │ ├── __init__.py │ ├── cli.py │ ├── setup.py │ └── test.py ├── Dockerfile ├── Pipfile ├── Pipfile.lock ├── pyworkflow │ ├── pyworkflow │ │ ├── .coveragerc │ │ ├── __init__.py │ │ ├── node.py │ │ ├── node_factory.py │ │ ├── nodes │ │ │ ├── __init__.py │ │ │ ├── flow_control │ │ │ │ ├── __init__.py │ │ │ │ ├── integer_input.py │ │ │ │ └── string_input.py │ │ │ ├── io │ │ │ │ ├── __init__.py │ │ │ │ ├── read_csv.py │ │ │ │ ├── table_creator.py │ │ │ │ └── write_csv.py │ │ │ ├── manipulation │ │ │ │ ├── __init__.py │ │ │ │ ├── filter.py │ │ │ │ ├── join.py │ │ │ │ └── pivot.py │ │ │ └── visualization │ │ │ │ ├── __init__.py │ │ │ │ └── graph.py │ │ ├── parameters.py │ │ ├── tests │ │ │ ├── sample_test_data.py │ │ │ ├── test_node.py │ │ │ ├── test_parameters.py │ │ │ ├── test_pyworkflow.py │ │ │ └── test_workflow.py │ │ └── workflow.py │ └── setup.py └── vp │ ├── manage.py │ ├── node │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── migrations │ │ └── __init__.py │ ├── models.py │ ├── tests.py │ ├── urls.py │ └── views.py │ ├── vp │ ├── __init__.py │ ├── asgi.py │ ├── settings.py │ ├── urls.py │ ├── views.py │ └── wsgi.py │ └── workflow │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── middleware.py │ ├── migrations │ └── __init__.py │ ├── models.py │ ├── tests.py │ ├── urls.py │ └── views.py ├── docker-compose.yml ├── docs ├── cli.md ├── custom_nodes.md ├── media │ ├── custom_node_missing_package.png │ ├── pyworkflow-ui.png │ ├── pyworkflow_coverage.svg │ └── ui_coverage.svg └── tests.md └── front-end ├── .gitignore ├── Dockerfile ├── README.md ├── __mocks__ └── css │ └── styleMock.js ├── __tests__ ├── components │ ├── CustomNode │ │ ├── GraphView.test.js │ │ ├── NodeConfig.test.js │ │ └── __snapshots__ │ │ │ ├── GraphView.test.js.snap │ │ │ └── NodeConfig.test.js.snap │ ├── VPLink │ │ └── VPLinkModel.test.js │ └── VPPort │ │ ├── VPPortFactory.test.js │ │ └── VPPortModel.test.js └── utils.test.js ├── babel.config.js ├── jest.config.js ├── package-lock.json ├── package.json ├── public ├── favicon.ico ├── index.html ├── logo192.png ├── logo512.png ├── manifest.json ├── robots.txt └── tabular-icon.png ├── setupTests.js └── src ├── API.js ├── components ├── About.js ├── App.js ├── CustomNode │ ├── CustomNodeFactory.js │ ├── CustomNodeModel.js │ ├── CustomNodeWidget.js │ ├── GraphView.js │ └── NodeConfig.js ├── CustomNodeUpload.js ├── GlobalFlowMenu.js ├── NodeMenu.js ├── StatusLight.js ├── VPLink │ ├── VPLinkFactory.js │ ├── VPLinkModel.js │ └── VPLinkWidget.js ├── VPPort │ ├── VPPortFactory.js │ └── VPPortModel.js └── Workspace.js ├── index.js ├── serviceWorker.js ├── styles ├── CustomNode.css ├── GraphView.css ├── NodeConfig.css ├── StatusLight.css ├── Workspace.css └── index.css └── utils.js /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .vscode 3 | 4 | # dependencies 5 | /node_modules 6 | /.pnp 7 | .pnp.js 8 | 9 | # testing 10 | /coverage 11 | 12 | # production 13 | /build 14 | 15 | # misc 16 | .DS_Store 17 | .env.local 18 | .env.development.local 19 | .env.test.local 20 | .env.production.local 21 | 22 | npm-debug.log* 23 | yarn-debug.log* 24 | yarn-error.log* 25 | -------------------------------------------------------------------------------- /.github/workflows/test_backend.yml: -------------------------------------------------------------------------------- 1 | # This workflow tests starting the back-end server, unit and API tests 2 | 3 | name: Test back-end 4 | 5 | on: 6 | push: 7 | pull_request: 8 | branches: [ master ] 9 | 10 | jobs: 11 | setup-back-end: 12 | runs-on: ubuntu-latest 13 | steps: 14 | # Checks-out your repository, for access in the workflow 15 | - uses: actions/checkout@v2 16 | 17 | # Setup python 18 | - name: Set up Python 3.x 19 | uses: actions/setup-python@v1 20 | with: 21 | python-version: 3.x 22 | 23 | # Install dependencies 24 | - name: Install dependencies 25 | run: | 26 | python3 -m pip install --upgrade pip 27 | pip3 install pipenv 28 | cd back-end 29 | pipenv install --dev 30 | 31 | # Add .env and test data 32 | - name: Add test/env data 33 | run: | 34 | cd back-end 35 | pipenv run echo "SECRET_KEY='TEMPORARY SECRET KEY'" > vp/.environment 36 | echo ",key,A 37 | 0,K0,A0 38 | 1,K1,A1 39 | 2,K2,A2 40 | 3,K3,A3 41 | 4,K4,A4 42 | 5,K5,A5" > /tmp/sample.csv 43 | 44 | # Run unittests 45 | - name: Run unittests 46 | run: | 47 | cd back-end/pyworkflow/pyworkflow 48 | pipenv run coverage run -m unittest tests/*.py 49 | pipenv run coverage report 50 | 51 | # Start server in background for API tests 52 | - name: Start server 53 | run: | 54 | cd back-end/vp 55 | pipenv run nohup python3 manage.py runserver & 56 | 57 | 58 | postman-tests: 59 | needs: setup-back-end 60 | runs-on: ubuntu-latest 61 | 62 | steps: 63 | - uses: actions/checkout@v2 64 | - uses: actions/checkout@master 65 | - uses: matt-ball/newman-action@master 66 | with: 67 | collection: Postman/Visual\ Programming-Tests.postman_collection.json 68 | environment: Postman/Local\ Testing.postman_environment.json 69 | -------------------------------------------------------------------------------- /.github/workflows/test_docker.yml: -------------------------------------------------------------------------------- 1 | # This workflow tests building the Docker images and starting containers 2 | 3 | name: Test Docker build 4 | 5 | on: 6 | push: 7 | pull_request: 8 | branches: [ master ] 9 | 10 | jobs: 11 | build-docker: 12 | runs-on: ubuntu-latest 13 | steps: 14 | # Checks-out your repository, for access in the workflow 15 | - uses: actions/checkout@v2 16 | 17 | # Build containers 18 | - name: Build docker container 19 | run: docker-compose build 20 | 21 | # Run containers 22 | - name: Start Docker containers 23 | run: docker-compose up -d 24 | -------------------------------------------------------------------------------- /.github/workflows/test_frontend.yml: -------------------------------------------------------------------------------- 1 | # This workflow tests front-end 2 | 3 | name: Test front-end 4 | 5 | on: 6 | push: 7 | pull_request: 8 | branches: [ master ] 9 | 10 | jobs: 11 | test-front-end: 12 | runs-on: ubuntu-latest 13 | steps: 14 | # Checks-out your repository, for access in the workflow 15 | - uses: actions/checkout@v2 16 | - name: Install dependencies 17 | run: | 18 | cd front-end 19 | npm ci 20 | 21 | - name: Run unit tests 22 | run: | 23 | cd front-end 24 | npm test 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .environment 2 | .vscode 3 | .idea/ 4 | .DS_Store 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | pip-wheel-metadata/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | session_tmp/ 70 | 71 | # Flask stuff: 72 | instance/ 73 | .webassets-cache 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | 81 | # PyBuilder 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 104 | __pypackages__/ 105 | 106 | # Celery stuff 107 | celerybeat-schedule 108 | celerybeat.pid 109 | 110 | # SageMath parsed files 111 | *.sage.py 112 | 113 | # Environments 114 | .env 115 | .venv 116 | env/ 117 | venv/ 118 | ENV/ 119 | env.bak/ 120 | venv.bak/ 121 | 122 | # Spyder project settings 123 | .spyderproject 124 | .spyproject 125 | 126 | # Rope project settings 127 | .ropeproject 128 | 129 | # mkdocs documentation 130 | /site 131 | 132 | # mypy 133 | .mypy_cache/ 134 | .dmypy.json 135 | dmypy.json 136 | 137 | # Pyre type checker 138 | .pyre/ 139 | 140 | # pytype static type analyzer 141 | .pytype/ 142 | 143 | # Cython debug symbols 144 | cython_debug/ 145 | 146 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Matthew Smith 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Postman/Local-env.postman_environment.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "084b8360-2b41-436d-b00f-7273831c9521", 3 | "name": "Local Testing", 4 | "values": [ 5 | { 6 | "key": "environment", 7 | "value": "localhost:8000", 8 | "enabled": true 9 | } 10 | ], 11 | "_postman_variable_scope": "environment", 12 | "_postman_exported_at": "2020-03-18T17:59:37.488Z", 13 | "_postman_exported_using": "Postman/7.10.0" 14 | } -------------------------------------------------------------------------------- /Postman/Node-endpoints.postman_collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "_postman_id": "2abea1fb-028c-4b16-8bd1-4c9898e0f383", 4 | "name": "Visual Programming-Nodes", 5 | "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" 6 | }, 7 | "item": [ 8 | { 9 | "name": "Get node", 10 | "request": { 11 | "method": "GET", 12 | "header": [], 13 | "url": { 14 | "raw": "{{environment}}/node/1", 15 | "host": [ 16 | "{{environment}}" 17 | ], 18 | "path": [ 19 | "node", 20 | "1" 21 | ] 22 | } 23 | }, 24 | "response": [] 25 | }, 26 | { 27 | "name": "Add ReadCsvNode", 28 | "request": { 29 | "method": "POST", 30 | "header": [], 31 | "body": { 32 | "mode": "raw", 33 | "raw": "{\n \"name\": null,\n \"node_id\": \"1\",\n \"node_type\": \"IONode\",\n \"node_key\": \"ReadCsvNode\",\n \"options\": {\n \t\"filepath_or_buffer\": \"/tmp/join1.csv\"\n }\n}", 34 | "options": { 35 | "raw": { 36 | "language": "json" 37 | } 38 | } 39 | }, 40 | "url": { 41 | "raw": "http://localhost:8000/node/", 42 | "protocol": "http", 43 | "host": [ 44 | "localhost" 45 | ], 46 | "port": "8000", 47 | "path": [ 48 | "node", 49 | "" 50 | ] 51 | } 52 | }, 53 | "response": [] 54 | }, 55 | { 56 | "name": "Add JoinNode", 57 | "request": { 58 | "method": "POST", 59 | "header": [], 60 | "body": { 61 | "mode": "raw", 62 | "raw": "{\n \"name\": null,\n \"node_id\": \"3\",\n \"node_type\": \"ManipulationNode\",\n \"node_key\": \"JoinNode\"\n}", 63 | "options": { 64 | "raw": { 65 | "language": "json" 66 | } 67 | } 68 | }, 69 | "url": { 70 | "raw": "http://localhost:8000/node/", 71 | "protocol": "http", 72 | "host": [ 73 | "localhost" 74 | ], 75 | "port": "8000", 76 | "path": [ 77 | "node", 78 | "" 79 | ] 80 | } 81 | }, 82 | "response": [] 83 | }, 84 | { 85 | "name": "Add WriteCsvNode", 86 | "request": { 87 | "method": "POST", 88 | "header": [], 89 | "body": { 90 | "mode": "raw", 91 | "raw": "{\n \"name\": null,\n \"node_id\": \"4\",\n \"node_type\": \"IONode\",\n \"node_key\": \"WriteCsvNode\",\n \"options\": {\n \t\"path_or_buf\": \"/tmp/join_test_output.csv\"\n }\n}", 92 | "options": { 93 | "raw": { 94 | "language": "json" 95 | } 96 | } 97 | }, 98 | "url": { 99 | "raw": "http://localhost:8000/node/", 100 | "protocol": "http", 101 | "host": [ 102 | "localhost" 103 | ], 104 | "port": "8000", 105 | "path": [ 106 | "node", 107 | "" 108 | ] 109 | } 110 | }, 111 | "response": [] 112 | }, 113 | { 114 | "name": "Update node", 115 | "request": { 116 | "method": "POST", 117 | "header": [], 118 | "body": { 119 | "mode": "raw", 120 | "raw": "{\n \"name\": null,\n \"node_id\": \"1\",\n \"node_type\": \"IONode\",\n \"node_key\": \"ReadCsvNode\",\n \"options\": {\n \t\"file\": \"test.csv\",\n \t\"delimiter\": \"\\n\"\n }\n}", 121 | "options": { 122 | "raw": { 123 | "language": "json" 124 | } 125 | } 126 | }, 127 | "url": { 128 | "raw": "{{environment}}/node/1", 129 | "host": [ 130 | "{{environment}}" 131 | ], 132 | "path": [ 133 | "node", 134 | "1" 135 | ] 136 | } 137 | }, 138 | "response": [] 139 | }, 140 | { 141 | "name": "Delete node", 142 | "request": { 143 | "method": "DELETE", 144 | "header": [], 145 | "url": { 146 | "raw": "{{environment}}/node/1", 147 | "host": [ 148 | "{{environment}}" 149 | ], 150 | "path": [ 151 | "node", 152 | "1" 153 | ] 154 | } 155 | }, 156 | "response": [] 157 | }, 158 | { 159 | "name": "Execute node", 160 | "request": { 161 | "method": "GET", 162 | "header": [], 163 | "url": { 164 | "raw": "{{environment}}/node/1/execute", 165 | "host": [ 166 | "{{environment}}" 167 | ], 168 | "path": [ 169 | "node", 170 | "1", 171 | "execute" 172 | ] 173 | } 174 | }, 175 | "response": [] 176 | }, 177 | { 178 | "name": "Add edge", 179 | "request": { 180 | "method": "POST", 181 | "header": [], 182 | "body": { 183 | "mode": "formdata", 184 | "formdata": [ 185 | { 186 | "key": "node_from_id", 187 | "value": "1", 188 | "type": "text", 189 | "disabled": true 190 | }, 191 | { 192 | "key": "node_to_id", 193 | "value": "2", 194 | "type": "text", 195 | "disabled": true 196 | } 197 | ] 198 | }, 199 | "url": { 200 | "raw": "{{environment}}/node/edge/1/2", 201 | "host": [ 202 | "{{environment}}" 203 | ], 204 | "path": [ 205 | "node", 206 | "edge", 207 | "1", 208 | "2" 209 | ] 210 | } 211 | }, 212 | "response": [] 213 | }, 214 | { 215 | "name": "Remove edge", 216 | "request": { 217 | "method": "DELETE", 218 | "header": [], 219 | "body": { 220 | "mode": "formdata", 221 | "formdata": [ 222 | { 223 | "key": "node_from_id", 224 | "value": "1", 225 | "type": "text", 226 | "disabled": true 227 | }, 228 | { 229 | "key": "node_to_id", 230 | "value": "2", 231 | "type": "text", 232 | "disabled": true 233 | } 234 | ] 235 | }, 236 | "url": { 237 | "raw": "{{environment}}/node/edge/1/2", 238 | "host": [ 239 | "{{environment}}" 240 | ], 241 | "path": [ 242 | "node", 243 | "edge", 244 | "1", 245 | "2" 246 | ] 247 | } 248 | }, 249 | "response": [] 250 | }, 251 | { 252 | "name": "Retrieve data", 253 | "request": { 254 | "method": "GET", 255 | "header": [], 256 | "url": { 257 | "raw": "{{environment}}/node/1/retrieve_data", 258 | "host": [ 259 | "{{environment}}" 260 | ], 261 | "path": [ 262 | "node", 263 | "1", 264 | "retrieve_data" 265 | ] 266 | } 267 | }, 268 | "response": [] 269 | } 270 | ], 271 | "protocolProfileBehavior": {} 272 | } -------------------------------------------------------------------------------- /Postman/Workflow-endpoints.postman_collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "_postman_id": "94c6f073-1978-444b-99b3-0123cef01056", 4 | "name": "Visual Programming-Workflow", 5 | "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" 6 | }, 7 | "item": [ 8 | { 9 | "name": "Get Info", 10 | "request": { 11 | "method": "GET", 12 | "header": [], 13 | "url": { 14 | "raw": "localhost:8000/info", 15 | "host": [ 16 | "localhost" 17 | ], 18 | "port": "8000", 19 | "path": [ 20 | "info" 21 | ] 22 | } 23 | }, 24 | "response": [] 25 | }, 26 | { 27 | "name": "New workflow", 28 | "event": [ 29 | { 30 | "listen": "test", 31 | "script": { 32 | "id": "f36bfdc4-a83f-4f7f-a91e-af704ea64c2e", 33 | "exec": [ 34 | "const response = pm.response.json()", 35 | "", 36 | "pm.environment.set('graph', response)" 37 | ], 38 | "type": "text/javascript" 39 | } 40 | } 41 | ], 42 | "request": { 43 | "method": "GET", 44 | "header": [], 45 | "url": { 46 | "raw": "localhost:8000/workflow/new", 47 | "host": [ 48 | "localhost" 49 | ], 50 | "port": "8000", 51 | "path": [ 52 | "workflow", 53 | "new" 54 | ] 55 | } 56 | }, 57 | "response": [] 58 | }, 59 | { 60 | "name": "Open file (200)", 61 | "request": { 62 | "method": "POST", 63 | "header": [], 64 | "body": { 65 | "mode": "formdata", 66 | "formdata": [ 67 | { 68 | "key": "file", 69 | "type": "file", 70 | "src": "/Users/mthomas/Desktop/combined.json" 71 | } 72 | ], 73 | "options": { 74 | "raw": { 75 | "language": "json" 76 | } 77 | } 78 | }, 79 | "url": { 80 | "raw": "localhost:8000/workflow/open", 81 | "host": [ 82 | "localhost" 83 | ], 84 | "port": "8000", 85 | "path": [ 86 | "workflow", 87 | "open" 88 | ] 89 | } 90 | }, 91 | "response": [] 92 | }, 93 | { 94 | "name": "Open file (404)", 95 | "request": { 96 | "method": "POST", 97 | "header": [], 98 | "body": { 99 | "mode": "formdata", 100 | "formdata": [ 101 | { 102 | "key": "file", 103 | "type": "file", 104 | "src": [] 105 | } 106 | ], 107 | "options": { 108 | "raw": { 109 | "language": "json" 110 | } 111 | } 112 | }, 113 | "url": { 114 | "raw": "localhost:8000/workflow/open", 115 | "host": [ 116 | "localhost" 117 | ], 118 | "port": "8000", 119 | "path": [ 120 | "workflow", 121 | "open" 122 | ] 123 | } 124 | }, 125 | "response": [] 126 | }, 127 | { 128 | "name": "Save workflow", 129 | "request": { 130 | "method": "POST", 131 | "header": [], 132 | "body": { 133 | "mode": "raw", 134 | "raw": "{\n \"id\": \"d6a3e610-4b4b-4455-b9da-d5dba9b4d948\",\n \"offsetX\": 0,\n \"offsetY\": 0,\n \"zoom\": 100,\n \"gridSize\": 0,\n \"layers\": [{\n \"id\": \"1163e2c6-99b2-45d2-a0e3-40edc9bb816d\",\n \"type\": \"diagram-links\",\n \"isSvg\": true,\n \"transformed\": true,\n \"models\": {\n \"4e9075df-0538-4fb0-9d3e-45a47d981c7e\": {\n \"id\": \"4e9075df-0538-4fb0-9d3e-45a47d981c7e\",\n \"type\": \"default\",\n \"selected\": true,\n \"source\": \"e7f27af1-e0ce-4c6c-8609-e5b494c7b713\",\n \"sourcePort\": \"5be5cc4f-1f35-4ed8-b143-b52597dd93a4\",\n \"target\": null,\n \"targetPort\": null,\n \"points\": [{\n \"id\": \"c8253115-d3b6-44c5-b45d-69ae6ca26ec9\",\n \"type\": \"point\",\n \"x\": 191.20001220703125,\n \"y\": 174.58750915527344\n }, {\n \"id\": \"07070038-8b29-4bc8-b0b9-c0c1c887f88a\",\n \"type\": \"point\",\n \"x\": 332.79998779296875,\n \"y\": 213.59999084472656\n }],\n \"labels\": [],\n \"width\": 5,\n \"color\": \"orange\",\n \"curvyness\": 50,\n \"selectedColor\": \"rgb(0,192,255)\"\n }\n }\n }, {\n \"id\": \"ef159b45-93f1-44a3-804a-d35fe7f15f21\",\n \"type\": \"diagram-nodes\",\n \"isSvg\": false,\n \"transformed\": true,\n \"models\": {\n \"e7f27af1-e0ce-4c6c-8609-e5b494c7b713\": {\n \"id\": \"e7f27af1-e0ce-4c6c-8609-e5b494c7b713\",\n \"type\": \"custom-node\",\n \"x\": 131.79998779296875,\n \"y\": 130.59999084472656,\n \"ports\": [{\n \"id\": \"5be5cc4f-1f35-4ed8-b143-b52597dd93a4\",\n \"type\": \"vp-port\",\n \"x\": 186.20001220703125,\n \"y\": 168.58750915527344,\n \"name\": \"out-0\",\n \"alignment\": \"right\",\n \"parentNode\": \"e7f27af1-e0ce-4c6c-8609-e5b494c7b713\",\n \"links\": [\"4e9075df-0538-4fb0-9d3e-45a47d981c7e\"],\n \"in\": false,\n \"label\": \"out-0\"\n }],\n \"options\": {\n \"id\": \"e7f27af1-e0ce-4c6c-8609-e5b494c7b713\",\n \"name\": \"Read CSV\",\n \"type\": \"custom-node\",\n \"num_in\": 0,\n \"num_out\": 1,\n \"color\": \"black\",\n \"doc\": \"ReadCsvNode\\n\\n Reads a CSV file into a pandas DataFrame.\\n\\n Raises:\\n NodeException: any error reading CSV file, converting\\n to DataFrame.\\n \"\n }\n },\n \"d9d4485c-847c-4514-9caa-628cf1999c06\": {\n \"id\": \"d9d4485c-847c-4514-9caa-628cf1999c06\",\n \"type\": \"custom-node\",\n \"x\": 314.79998779296875,\n \"y\": 167.59999084472656,\n \"ports\": [{\n \"id\": \"e2c88ded-3969-4231-8b47-85c82730e4f0\",\n \"type\": \"vp-port\",\n \"x\": 322.7874755859375,\n \"y\": 205.58750915527344,\n \"name\": \"in-0\",\n \"alignment\": \"left\",\n \"parentNode\": \"d9d4485c-847c-4514-9caa-628cf1999c06\",\n \"links\": [],\n \"in\": true,\n \"label\": \"in-0\"\n }],\n \"options\": {\n \"id\": \"d9d4485c-847c-4514-9caa-628cf1999c06\",\n \"name\": \"Write CSV\",\n \"type\": \"custom-node\",\n \"num_in\": 1,\n \"num_out\": 0,\n \"color\": \"green\",\n \"doc\": \"WriteCsvNode\\n\\n Writes the current DataFrame to a CSV file.\\n\\n Raises:\\n NodeException: any error writing CSV file, converting\\n from DataFrame.\\n \"\n }\n }\n }\n }]\n}", 135 | "options": { 136 | "raw": { 137 | "language": "json" 138 | } 139 | } 140 | }, 141 | "url": { 142 | "raw": "{{environment}}/workflow/save", 143 | "host": [ 144 | "{{environment}}" 145 | ], 146 | "path": [ 147 | "workflow", 148 | "save" 149 | ] 150 | } 151 | }, 152 | "response": [] 153 | }, 154 | { 155 | "name": "Workflow execution order", 156 | "event": [ 157 | { 158 | "listen": "test", 159 | "script": { 160 | "id": "f36bfdc4-a83f-4f7f-a91e-af704ea64c2e", 161 | "exec": [ 162 | "" 163 | ], 164 | "type": "text/javascript" 165 | } 166 | } 167 | ], 168 | "request": { 169 | "method": "GET", 170 | "header": [], 171 | "url": { 172 | "raw": "{{environment}}/workflow/execute", 173 | "host": [ 174 | "{{environment}}" 175 | ], 176 | "path": [ 177 | "workflow", 178 | "execute" 179 | ] 180 | } 181 | }, 182 | "response": [] 183 | }, 184 | { 185 | "name": "Node successors", 186 | "event": [ 187 | { 188 | "listen": "test", 189 | "script": { 190 | "id": "f36bfdc4-a83f-4f7f-a91e-af704ea64c2e", 191 | "exec": [ 192 | "const response = pm.response.json()", 193 | "", 194 | "pm.environment.set('graph', response)" 195 | ], 196 | "type": "text/javascript" 197 | } 198 | } 199 | ], 200 | "request": { 201 | "method": "GET", 202 | "header": [], 203 | "url": { 204 | "raw": "localhost:8000/workflow/execute/1/successors", 205 | "host": [ 206 | "localhost" 207 | ], 208 | "port": "8000", 209 | "path": [ 210 | "workflow", 211 | "execute", 212 | "1", 213 | "successors" 214 | ] 215 | } 216 | }, 217 | "response": [] 218 | }, 219 | { 220 | "name": "Retrieve node list", 221 | "request": { 222 | "method": "GET", 223 | "header": [], 224 | "url": { 225 | "raw": "{{environment}}/workflow/nodes", 226 | "host": [ 227 | "{{environment}}" 228 | ], 229 | "path": [ 230 | "workflow", 231 | "nodes" 232 | ] 233 | } 234 | }, 235 | "response": [] 236 | } 237 | ], 238 | "protocolProfileBehavior": {} 239 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyWorkflow 2 | | | Status | 3 | |------------|--------| 4 | | Docker | ![Test Docker build](https://github.com/matthew-t-smith/visual-programming/workflows/Test%20Docker%20build/badge.svg) | 5 | | Back-end | ![Test back-end](https://github.com/matthew-t-smith/visual-programming/workflows/Test%20back-end/badge.svg) | 6 | | Front-end | ![Test front-end](https://github.com/matthew-t-smith/visual-programming/workflows/Test%20front-end/badge.svg) | 7 | | PyWorkflow | ![Code Coverage](./docs/media/pyworkflow_coverage.svg) | 8 | | UI | ![Code Coverage](./docs/media/ui_coverage.svg) | 9 | 10 | PyWorkflow is a visual programming application for building data science 11 | pipelines and workflows. It is inspired by [KNIME](https://www.knime.com) 12 | and aims to bring the desktop-based experience to a web-based environment. 13 | PyWorkflow takes a Python-first approach and leverages the power of *pandas* 14 | DataFrames to bring data-science to the masses. 15 | 16 | ![Pyworkflow UI](./docs/media/pyworkflow-ui.png) 17 | 18 | # Introduction 19 | PyWorkflow was developed with a few key principles in mind: 20 | 21 | 1) Easily deployed. PyWorkflow can be deployed locally or remotely with pre-built 22 | Docker containers. 23 | 24 | 2) Highly extensible. PyWorkflow has a few key nodes built-in to perform common 25 | operations, but it is built with custom nodes in mind. Any user can write a 26 | custom node of their own to perform *pandas* operations, or other data science 27 | packages. 28 | 29 | 3) Advanced features for everyone. PyWorkflow is meant to cater to users with 30 | no programming experience, all the way to someone who writes Python code daily. 31 | An easy-to-use command line interface allows for batch workflow execution and 32 | scheduled runs with a tool like `cron`. 33 | 34 | To meet these principles, the user interface is built on 35 | [react-diagrams](https://github.com/projectstorm/react-diagrams) 36 | to enable drag-and-drop nodes and edge creation. These packaged nodes provide 37 | basic *pandas* functionality and easy customization options for users to create 38 | workflows tailored to their specific needs. For users looking to create custom 39 | nodes, please [reference the documentation on how to write your own class](docs/custom_nodes.md). 40 | 41 | On the back-end, a computational graph stores the nodes, edges, and 42 | configuration options using the [NetworkX package](https://networkx.github.io). 43 | All data operations are saved in JSON format which allows for easy readability 44 | and transfer of data to other environments. 45 | 46 | # Getting Started 47 | The back-end consists of the PyWorkflow package, to perform all graph-based 48 | operations, file storage/retrieval, and execution. These methods are triggered 49 | either via API calls from the Django web-server, or from the CLI application. 50 | 51 | The front-end is a SPA React app (bootstrapped with create-react-app). For React 52 | to request data from Django, the `proxy` field is set in `front-end/package.json`, 53 | telling the dev server to fetch non-static data from `localhost:8000` **where 54 | the Django app must be running**. 55 | 56 | ## Docker 57 | 58 | The easiest way to get started is by deploying both Docker containers on your 59 | local machine. For help installing Docker, [reference the documentation for your 60 | specific system](https://docs.docker.com/get-docker/). 61 | 62 | To run the application with `docker-compose`, run `docker-compose up` from the root directory 63 | (or `docker-compose up --build` to rebuild the images first). 64 | 65 | Use `docker-compose down` to shut down the application gracefully. 66 | 67 | The application comprises running containers of two images: the `front-end` and 68 | the `back-end`. The `docker-compose.yml` defines how to combine and run the two. 69 | 70 | In order to build each image individually, from the root of the application: 71 | - `docker build front-end --tag FE_IMAGE[:TAG]` 72 | - `docker build back-end --tag BE_IMAGE[:TAG]` 73 | ex. - `docker build back-end --tag backendtest:2.0` 74 | 75 | Each individual container can be run by changing to the `front-end` or `back-end` directory and running: 76 | - `docker run -p 3000:3000 --name FE_CONTAINER_NAME FE_IMAGE[:TAG]` 77 | - `docker run -p 8000:8000 --name BE_CONTAINER_NAME BE_IMAGE[:TAG]` 78 | ex. - `docker run -p 8000:8000 --name pyworkflow-be backendtest:2.0` 79 | 80 | Note: there [is a known issue with `react-scripts` v3.4.1](https://github.com/facebook/create-react-app/issues/8688) 81 | that may cause the front-end container to exit with code 0. If this happens, 82 | you can add `-e CI=true` to the `docker-run` command above for the front-end. 83 | 84 | NOTE: For development outside of Docker, change `./front-end/package.json` 85 | from `"proxy": "http://back-end:8000"` to `"proxy": http://localhost:8000"` to work. 86 | 87 | After the Docker containers are started, and both front- and back-ends are running, 88 | you can access the application by loading [http://localhost:3000/](http://localhost:3000/) 89 | in your browser. 90 | 91 | ## Serve locally 92 | 93 | Alternatively, the front- and back-ends can be compiled separately and run on 94 | your local machine. 95 | 96 | ### Server (Django) 97 | 98 | 1. Install `pipenv` 99 | 100 | - **Homebrew** 101 | 102 | ``` 103 | brew install pipenv 104 | ``` 105 | 106 | - **pip** 107 | 108 | ``` 109 | pip install pipenv OR pip3 install pipenv 110 | ``` 111 | 2. Install dependencies 112 | Go to the `back-end` directory with `Pipfile` and `Pipfile.lock`. 113 | ``` 114 | cd back-end 115 | pipenv install 116 | ``` 117 | 3. Setup your local environment 118 | 119 | - Create environment file with app secret 120 | ``` 121 | echo "SECRET_KEY='TEMPORARY SECRET KEY'" > vp/.environment 122 | ``` 123 | 124 | 4. Start dev server from app root 125 | ``` 126 | cd vp 127 | pipenv run python3 manage.py runserver 128 | ``` 129 | 130 | If you have trouble running commands individually, you can also enter the 131 | virtual environment created by `pipenv` by running `pipenv shell`. 132 | 133 | ### Client (react-diagrams) 134 | In a separate terminal window, perform the following steps to start the 135 | front-end. 136 | 137 | 1. Install Prerequisites 138 | ``` 139 | cd front-end 140 | npm install 141 | ``` 142 | 2. Start dev server 143 | ``` 144 | npm start 145 | ``` 146 | 147 | By default, the `react-scripts` should open your default browser to the main 148 | application page. If not, you can go to [http://localhost:3000/](http://localhost:3000/) 149 | in your browser. 150 | 151 | # CLI 152 | PyWorkflow also provides a command-line interface to execute pre-built workflows 153 | without the client or server running. The CLI is packaged in the `back-end` 154 | directory and can be accessed through a deployed Docker container, or locally 155 | through the `pipenv shell`. 156 | 157 | The CLI syntax for PyWorkflow is: 158 | ``` 159 | pyworkflow execute workflow-file... 160 | ``` 161 | 162 | For help reading from stdin, writing to stdout, batch-processing, and more 163 | [check out the CLI docs](docs/cli.md) for more information. 164 | 165 | # Tests 166 | PyWorkflow has several automated tests that are run on each push to the GitHub 167 | repository through GitHub Actions. The status of each can be seen in the various 168 | badges at the top of this README. 169 | 170 | PyWorkflow currently has unit tests for both the back-end (the PyWorkflow 171 | package) and the front-end (react-diagrams). There are also API tests 172 | using Postman to test the integration between the front- and back-ends. For more 173 | information on these tests, and how to run them, [read the documentation for more 174 | information](docs/tests.md). 175 | -------------------------------------------------------------------------------- /back-end/CLI/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/CLI/__init__.py -------------------------------------------------------------------------------- /back-end/CLI/cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | import json 3 | 4 | from pyworkflow import Workflow, WorkflowException 5 | from pyworkflow import NodeException 6 | from pyworkflow.nodes import ReadCsvNode, WriteCsvNode 7 | 8 | 9 | class Config(object): 10 | def __init__(self): 11 | self.verbose = False 12 | 13 | 14 | pass_config = click.make_pass_decorator(Config, ensure=True) 15 | 16 | 17 | @click.group() 18 | def cli(): 19 | pass 20 | 21 | 22 | @cli.command() 23 | @click.argument('filenames', type=click.Path(exists=True), nargs=-1) 24 | @click.option('--verbose', is_flag=True, help='Enables verbose mode.') 25 | def execute(filenames, verbose): 26 | """Execute Workflow file(s).""" 27 | # Check whether to log to terminal, or redirect output 28 | log = click.get_text_stream('stdout').isatty() 29 | 30 | # Execute each workflow in the args 31 | for workflow_file in filenames: 32 | 33 | if workflow_file is None: 34 | click.echo('Please specify a workflow to run', err=True) 35 | return 36 | 37 | if log: 38 | click.echo('Loading workflow file from %s' % workflow_file) 39 | 40 | try: 41 | workflow = open_workflow(workflow_file) 42 | execute_workflow(workflow, log, verbose) 43 | except OSError as e: 44 | click.echo(f"Issues loading workflow file: {e}", err=True) 45 | except WorkflowException as e: 46 | click.echo(f"Issues during workflow execution\n{e}", err=True) 47 | 48 | 49 | def execute_workflow(workflow, log, verbose): 50 | """Execute a workflow file, node-by-node. 51 | 52 | Retrieves the execution order from the Workflow and iterates through nodes. 53 | If any I/O nodes are present AND stdin/stdout redirection is provided in the 54 | command-line, overwrite the stored options and then replace before saving. 55 | 56 | Args: 57 | workflow - Workflow object loaded from file 58 | log - True, for outputting to terminal; False for stdout redirection 59 | verbose - True, for outputting debug information; False otherwise 60 | """ 61 | execution_order = workflow.execution_order() 62 | 63 | # Execute each node in the order returned by the Workflow 64 | for node in execution_order: 65 | try: 66 | node_to_execute = workflow.get_node(node) 67 | original_file_option = pre_execute(workflow, node_to_execute, log) 68 | 69 | if verbose: 70 | print('Executing node of type ' + str(type(node_to_execute))) 71 | 72 | # perform execution 73 | executed_node = workflow.execute(node) 74 | 75 | # If file was replaced with stdin/stdout, restore original option 76 | if original_file_option is not None: 77 | executed_node.option_values["file"] = original_file_option 78 | 79 | # Update Node in Workflow with changes (saved data file) 80 | workflow.update_or_add_node(executed_node) 81 | except NodeException as e: 82 | click.echo(f"Issues during node execution\n{e}", err=True) 83 | 84 | if verbose: 85 | click.echo('Completed workflow execution!') 86 | 87 | 88 | def pre_execute(workflow, node_to_execute, log): 89 | """Pre-execution steps, to overwrite file options with stdin/stdout. 90 | 91 | If stdin is not a tty, and the Node is ReadCsv, replace file with buffer. 92 | If stdout is not a tty, and the Node is WriteCsv, replace file with buffer. 93 | 94 | Args: 95 | workflow - Workflow object loaded from file 96 | node_to_execute - The Node to execute 97 | log - True, for outputting to terminal; False for stdout redirection 98 | """ 99 | stdin = click.get_text_stream('stdin') 100 | 101 | if type(node_to_execute) is ReadCsvNode and not stdin.isatty(): 102 | new_file_location = stdin 103 | elif type(node_to_execute) is WriteCsvNode and not log: 104 | new_file_location = click.get_text_stream('stdout') 105 | else: 106 | # No file redirection needed 107 | return None 108 | 109 | # save original file info 110 | original_file_option = node_to_execute.option_values["file"] 111 | 112 | # replace with value from stdin and save 113 | node_to_execute.option_values["file"] = new_file_location 114 | workflow.update_or_add_node(node_to_execute) 115 | 116 | return original_file_option 117 | 118 | 119 | def open_workflow(workflow_file): 120 | with open(workflow_file) as f: 121 | json_content = json.load(f) 122 | 123 | return Workflow.from_json(json_content['pyworkflow']) 124 | -------------------------------------------------------------------------------- /back-end/CLI/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup(name='CLI', 4 | version='0.0.0', 5 | py_modules= ['cli'], 6 | install_requires=[ 7 | 'Click', 8 | ], 9 | entry_points=''' 10 | [console_scripts] 11 | pyworkflow=cli:cli 12 | ''', 13 | description='CLI application for pyworkflow virtual programming tool', 14 | author='Visual Programming Team', 15 | license='MIT', 16 | zip_safe=False) -------------------------------------------------------------------------------- /back-end/CLI/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/CLI/test.py -------------------------------------------------------------------------------- /back-end/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | 3 | WORKDIR /visual-programming/back-end 4 | 5 | COPY Pipfile Pipfile.lock ./ 6 | COPY CLI/ ./CLI/ 7 | COPY pyworkflow/ ./pyworkflow/ 8 | 9 | RUN pip install pipenv 10 | RUN pipenv install --dev --ignore-pipfile 11 | 12 | COPY vp/ ./vp 13 | RUN echo "SECRET_KEY=tmp" > vp/.environment 14 | 15 | EXPOSE 8000 16 | 17 | WORKDIR /visual-programming/back-end/vp 18 | 19 | CMD pipenv run python manage.py runserver 0.0.0.0:8000 20 | -------------------------------------------------------------------------------- /back-end/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | coverage = "~=5.1" 8 | coverage-badge = "~=1.0" 9 | v = {editable = true,version = "*"} 10 | 11 | [packages] 12 | autopep8 = "~=1.5" 13 | django = "~=3.0.7" 14 | pandas = "~=1.0.1" 15 | python-dotenv = "~=0.12.0" 16 | networkx = "~=2.4" 17 | pyworkflow = {path = "./pyworkflow",editable = true} 18 | djangorestframework = "*" 19 | drf-yasg = "*" 20 | click = "*" 21 | altair = "~=4.1.0" 22 | cli = {path = "./CLI",editable = true} 23 | 24 | [requires] 25 | python_version = "3.8" 26 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit= 3 | */.local/share/virtualenvs/* 4 | ./tests/* 5 | ./nodes/custom_nodes/* -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/__init__.py: -------------------------------------------------------------------------------- 1 | from .workflow import Workflow, WorkflowException 2 | from .node import * 3 | from .node_factory import node_factory 4 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/node.py: -------------------------------------------------------------------------------- 1 | from .parameters import * 2 | import io 3 | 4 | class Node: 5 | """Node object 6 | 7 | """ 8 | options = Options() 9 | option_types = OptionTypes() 10 | 11 | def __init__(self, node_info): 12 | self.name = node_info.get('name') 13 | self.node_id = node_info.get('node_id') 14 | self.node_type = node_info.get('node_type') 15 | self.node_key = node_info.get('node_key') 16 | self.data = node_info.get('data') 17 | self.filename = node_info.get('filename') 18 | self.is_global = node_info.get('is_global') is True 19 | 20 | self.option_values = dict() 21 | if node_info.get("options"): 22 | self.option_values.update(node_info["options"]) 23 | 24 | self.option_replace = dict() 25 | if node_info.get("option_replace"): 26 | self.option_replace.update(node_info["option_replace"]) 27 | 28 | def execute(self, predecessor_data, flow_vars): 29 | raise NotImplementedError() 30 | 31 | def get_execution_options(self, workflow, flow_nodes): 32 | """Replace Node options with flow variables. 33 | 34 | If the user has specified any flow variables to replace Node options, 35 | perform the replacement and return a dict with all options to use for 36 | execution. If no flow variables are included, this method will return 37 | a copy of all Node options. 38 | 39 | For any 'file' options, the value will be replaced with a path based on 40 | the Workflow's root directory. 41 | 42 | Args: 43 | workflow: Workflow object to construct file paths 44 | flow_nodes: dict of FlowNodes used to replace options 45 | 46 | Returns: 47 | dict containing options to use for execution 48 | """ 49 | execution_options = dict() 50 | 51 | for key, option in self.options.items(): 52 | 53 | if key in flow_nodes: 54 | replacement_value = flow_nodes[key].get_replacement_value() 55 | option.set_value(replacement_value) 56 | else: 57 | replacement_value = option.get_value() 58 | 59 | if key == 'file' and type(replacement_value) == io.TextIOWrapper: 60 | # For files specified via stdin/stdout, store directly 61 | option.set_value(replacement_value) 62 | elif key == 'file': 63 | # Otherwise, point to filepath stored in Workflow directory 64 | option.set_value(workflow.path(replacement_value)) 65 | 66 | execution_options[key] = option 67 | 68 | return execution_options 69 | 70 | def validate(self): 71 | """Validate Node configuration 72 | 73 | Checks all Node options and validates all Parameter classes using 74 | their validation method. 75 | 76 | Raises: 77 | ParameterValidationError: invalid Parameter value 78 | """ 79 | for key, option in self.options.items(): 80 | if key not in self.option_replace: 81 | option.validate() 82 | 83 | def validate_input_data(self, num_input_data): 84 | """Validate Node input data. 85 | 86 | Checks that input data, if any, matches with required number of input 87 | ports. 88 | 89 | Args: 90 | num_input_data: Number of input data passed in 91 | 92 | Raises: 93 | NodeException on mis-matched input ports/data 94 | """ 95 | if num_input_data != self.num_in: 96 | raise NodeException( 97 | 'execute', 98 | f'{self.node_key} requires {self.num_in} inputs. {num_input_data} were provided' 99 | ) 100 | 101 | def to_json(self): 102 | return { 103 | "name": self.name, 104 | "node_id": self.node_id, 105 | "node_type": self.node_type, 106 | "node_key": self.node_key, 107 | "data": self.data, 108 | "is_global": self.is_global, 109 | "option_values": self.option_values, 110 | "option_replace": self.option_replace, 111 | } 112 | 113 | def __str__(self): 114 | return self.name 115 | 116 | 117 | class FlowNode(Node): 118 | """FlowNodes object. 119 | 120 | FlowNodes do not execute. They specify a variable name and value to pass 121 | to other Nodes as a way to dynamically change other parameter values. 122 | """ 123 | display_name = "Flow Control" 124 | 125 | def execute(self, predecessor_data, flow_vars): 126 | return 127 | 128 | def get_replacement_value(self): 129 | return self.options['default_value'].get_value() 130 | 131 | 132 | class IONode(Node): 133 | """IONodes deal with file-handling in/out of the Workflow.""" 134 | color = "green" 135 | 136 | def execute(self, predecessor_data, flow_vars): 137 | raise NotImplementedError() 138 | 139 | 140 | class ManipulationNode(Node): 141 | """ManipulationNodes deal with data manipulation.""" 142 | color = "goldenrod" 143 | 144 | def execute(self, predecessor_data, flow_vars): 145 | raise NotImplementedError() 146 | 147 | 148 | class VizNode(Node): 149 | """VizNodes deal with graphical display of data.""" 150 | color = "red" 151 | 152 | def execute(self, predecessor_data, flow_vars): 153 | raise NotImplementedError() 154 | 155 | 156 | class NodeException(Exception): 157 | def __init__(self, action: str, reason: str): 158 | self.action = action 159 | self.reason = reason 160 | 161 | def __str__(self): 162 | return self.action + ': ' + self.reason 163 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/node_factory.py: -------------------------------------------------------------------------------- 1 | from .nodes import * 2 | import importlib 3 | 4 | 5 | def node_factory(node_info): 6 | """Create a new Node with info.""" 7 | node_type = node_info.get('node_type') 8 | node_key = node_info.get('node_key') 9 | 10 | if node_type == 'io': 11 | new_node = io_node(node_key, node_info) 12 | elif node_type == 'manipulation': 13 | new_node = manipulation_node(node_key, node_info) 14 | elif node_type == 'flow_control': 15 | new_node = flow_node(node_key, node_info) 16 | elif node_type == 'visualization': 17 | new_node = visualization_node(node_key, node_info) 18 | else: 19 | new_node = custom_node(node_key, node_info) 20 | 21 | return new_node 22 | 23 | 24 | def flow_node(node_key, node_info): 25 | if node_key == 'StringNode': 26 | return StringNode(node_info) 27 | elif node_key == 'IntegerNode': 28 | return IntegerNode(node_info) 29 | else: 30 | return None 31 | 32 | 33 | def io_node(node_key, node_info): 34 | if node_key == 'ReadCsvNode': 35 | return ReadCsvNode(node_info) 36 | elif node_key == 'TableCreatorNode': 37 | return TableCreatorNode(node_info) 38 | elif node_key == 'WriteCsvNode': 39 | return WriteCsvNode(node_info) 40 | else: 41 | return None 42 | 43 | 44 | def manipulation_node(node_key, node_info): 45 | if node_key == 'JoinNode': 46 | return JoinNode(node_info) 47 | elif node_key == 'PivotNode': 48 | return PivotNode(node_info) 49 | elif node_key == 'FilterNode': 50 | return FilterNode(node_info) 51 | else: 52 | return None 53 | 54 | 55 | def visualization_node(node_key, node_info): 56 | if node_key == 'GraphNode': 57 | return GraphNode(node_info) 58 | else: 59 | return None 60 | 61 | 62 | def custom_node(node_key, node_info): 63 | try: 64 | filename = node_info.get('filename') 65 | module = importlib.import_module(f'pyworkflow.nodes.custom_nodes.{filename}') 66 | my_class = getattr(module, node_key) 67 | instance = my_class(node_info) 68 | 69 | return instance 70 | except Exception as e: 71 | # print(str(e)) 72 | return None 73 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/__init__.py: -------------------------------------------------------------------------------- 1 | from .flow_control import * 2 | from .io import * 3 | from .manipulation import * 4 | from .visualization import * 5 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/flow_control/__init__.py: -------------------------------------------------------------------------------- 1 | from .string_input import StringNode 2 | from .integer_input import IntegerNode 3 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/flow_control/integer_input.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import FlowNode 2 | from pyworkflow.parameters import * 3 | 4 | 5 | class IntegerNode(FlowNode): 6 | """IntegerNode object 7 | 8 | Allows for Integers to replace fields representing numbers in Nodes 9 | """ 10 | name = "Integer Input" 11 | num_in = 0 12 | num_out = 0 13 | color = 'purple' 14 | 15 | OPTIONS = { 16 | "default_value": IntegerParameter( 17 | "Default Value", 18 | docstring="Value this node will pass as a flow variable" 19 | ), 20 | "var_name": StringParameter( 21 | "Variable Name", 22 | default="my_var", 23 | docstring="Name of the variable to use in another Node" 24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/flow_control/string_input.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import FlowNode, NodeException 2 | from pyworkflow.parameters import * 3 | 4 | 5 | class StringNode(FlowNode): 6 | """StringNode object 7 | 8 | Allows for Strings to replace 'string' fields in Nodes 9 | """ 10 | name = "String Input" 11 | num_in = 0 12 | num_out = 0 13 | color = 'purple' 14 | 15 | OPTIONS = { 16 | "default_value": StringParameter( 17 | "Default Value", 18 | docstring="Value this node will pass as a flow variable" 19 | ), 20 | "var_name": StringParameter( 21 | "Variable Name", 22 | default="my_var", 23 | docstring="Name of the variable to use in another Node" 24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/io/__init__.py: -------------------------------------------------------------------------------- 1 | from .read_csv import ReadCsvNode 2 | from .write_csv import WriteCsvNode 3 | from .table_creator import TableCreatorNode 4 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/io/read_csv.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import IONode, NodeException 2 | from pyworkflow.parameters import * 3 | 4 | import pandas as pd 5 | 6 | 7 | class ReadCsvNode(IONode): 8 | """Reads a CSV file into a pandas DataFrame. 9 | 10 | Raises: 11 | NodeException: any error reading CSV file, converting to DataFrame. 12 | """ 13 | name = "Read CSV" 14 | num_in = 0 15 | num_out = 1 16 | 17 | OPTIONS = { 18 | "file": FileParameter( 19 | "File", 20 | docstring="CSV File" 21 | ), 22 | "sep": StringParameter( 23 | "Delimiter", 24 | default=",", 25 | docstring="Column delimiter" 26 | ), 27 | # user-specified headers are probably integers, but haven't figured out 28 | # arguments with multiple possible types 29 | "header": StringParameter( 30 | "Header Row", 31 | default="infer", 32 | docstring="Row number containing column names (0-indexed)" 33 | ), 34 | } 35 | 36 | def execute(self, predecessor_data, flow_vars): 37 | try: 38 | df = pd.read_csv( 39 | flow_vars["file"].get_value(), 40 | sep=flow_vars["sep"].get_value(), 41 | header=flow_vars["header"].get_value() 42 | ) 43 | return df.to_json() 44 | except Exception as e: 45 | raise NodeException('read csv', str(e)) 46 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/io/table_creator.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import IONode, NodeException 2 | from pyworkflow.parameters import * 3 | 4 | import pandas as pd 5 | import io 6 | 7 | 8 | class TableCreatorNode(IONode): 9 | """Accepts raw-text CSV input to create data tables. 10 | 11 | Raises: 12 | NodeException: any error reading CSV file, converting 13 | to DataFrame. 14 | """ 15 | name = "Table Creator" 16 | num_in = 0 17 | num_out = 1 18 | 19 | OPTIONS = { 20 | "input": TextParameter( 21 | "Input", 22 | default="", 23 | docstring="Text input" 24 | ), 25 | "sep": StringParameter( 26 | "Delimiter", 27 | default=",", 28 | docstring="Column delimiter" 29 | ), 30 | # user-specified headers are probably integers, but haven't figured out 31 | # arguments with multiple possible types 32 | "header": StringParameter( 33 | "Header Row", 34 | default="infer", 35 | docstring="Row number containing column names (0-indexed)" 36 | ), 37 | } 38 | 39 | def execute(self, predecessor_data, flow_vars): 40 | try: 41 | df = pd.read_csv( 42 | io.StringIO(flow_vars["input"].get_value()), 43 | sep=flow_vars["sep"].get_value(), 44 | header=flow_vars["header"].get_value() 45 | ) 46 | return df.to_json() 47 | except Exception as e: 48 | raise NodeException('read csv', str(e)) 49 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/io/write_csv.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import IONode, NodeException 2 | from pyworkflow.parameters import * 3 | 4 | import pandas as pd 5 | 6 | 7 | class WriteCsvNode(IONode): 8 | """Writes the current DataFrame to a CSV file. 9 | 10 | Raises: 11 | NodeException: any error writing CSV file, converting from DataFrame. 12 | """ 13 | name = "Write CSV" 14 | num_in = 1 15 | num_out = 0 16 | download_result = True 17 | 18 | OPTIONS = { 19 | "file": StringParameter( 20 | "Filename", 21 | docstring="CSV file to write" 22 | ), 23 | "sep": StringParameter( 24 | "Delimiter", 25 | default=",", 26 | docstring="Column delimiter" 27 | ), 28 | "index": BooleanParameter( 29 | "Write Index", 30 | default=True, 31 | docstring="Write index as column?" 32 | ), 33 | } 34 | 35 | def execute(self, predecessor_data, flow_vars): 36 | try: 37 | # Convert JSON data to DataFrame 38 | df = pd.DataFrame.from_dict(predecessor_data[0]) 39 | 40 | # Write to CSV and save 41 | df.to_csv( 42 | flow_vars["file"].get_value(), 43 | sep=flow_vars["sep"].get_value(), 44 | index=flow_vars["index"].get_value() 45 | ) 46 | return df.to_json() 47 | except Exception as e: 48 | raise NodeException('write csv', str(e)) 49 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/manipulation/__init__.py: -------------------------------------------------------------------------------- 1 | from .filter import FilterNode 2 | from .join import JoinNode 3 | from .pivot import PivotNode 4 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/manipulation/filter.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import ManipulationNode, NodeException 2 | from pyworkflow.parameters import * 3 | 4 | import pandas as pd 5 | 6 | 7 | class FilterNode(ManipulationNode): 8 | """Subset the DataFrame rows or columns according to the specified index labels. 9 | 10 | pandas API reference: 11 | https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.filter.html 12 | 13 | Raises: 14 | NodeException: catches exceptions when dealing with pandas DataFrames. 15 | """ 16 | name = "Filter" 17 | num_in = 1 18 | num_out = 1 19 | 20 | OPTIONS = { 21 | 'items': StringParameter( 22 | 'Items', 23 | docstring='Keep labels from axis which are in items' 24 | ), 25 | 'like': StringParameter( 26 | 'Like', 27 | docstring='Keep labels from axis for which like in label == True.' 28 | ), 29 | 'regex': StringParameter( 30 | 'Regex', 31 | docstring='Keep labels from axis for which re.search(regex, label) == True.' 32 | ), 33 | 'axis': StringParameter( 34 | 'Axis', 35 | docstring='The axis to filter on.' 36 | ) 37 | } 38 | 39 | def execute(self, predecessor_data, flow_vars): 40 | try: 41 | input_df = pd.DataFrame.from_dict(predecessor_data[0]) 42 | output_df = pd.DataFrame.filter( 43 | input_df, 44 | items=flow_vars['items'].get_value(), 45 | like=flow_vars['like'].get_value(), 46 | regex=flow_vars['regex'].get_value(), 47 | axis=flow_vars['axis'].get_value(), 48 | ) 49 | return output_df.to_json() 50 | except Exception as e: 51 | raise NodeException('filter', str(e)) 52 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/manipulation/join.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import ManipulationNode, NodeException 2 | from pyworkflow.parameters import * 3 | 4 | import pandas as pd 5 | 6 | 7 | class JoinNode(ManipulationNode): 8 | """Merge DataFrame or named Series objects with a database-style join. 9 | 10 | pandas API reference: 11 | https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.merge.html 12 | 13 | Raises: 14 | NodeException: catches exceptions when dealing with pandas DataFrames. 15 | """ 16 | name = "Joiner" 17 | num_in = 2 18 | num_out = 1 19 | 20 | OPTIONS = { 21 | "on": StringParameter( 22 | "Join Column", 23 | docstring="Name of column to join on" 24 | ) 25 | } 26 | 27 | def execute(self, predecessor_data, flow_vars): 28 | try: 29 | first_df = pd.DataFrame.from_dict(predecessor_data[0]) 30 | second_df = pd.DataFrame.from_dict(predecessor_data[1]) 31 | combined_df = pd.merge( 32 | first_df, 33 | second_df, 34 | on=flow_vars["on"].get_value() 35 | ) 36 | return combined_df.to_json() 37 | except Exception as e: 38 | raise NodeException('join', str(e)) 39 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/manipulation/pivot.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import ManipulationNode, NodeException 2 | from pyworkflow.parameters import * 3 | 4 | import pandas as pd 5 | 6 | 7 | class PivotNode(ManipulationNode): 8 | """Create a spreadsheet-style pivot table as a DataFrame. 9 | 10 | pandas reference: 11 | https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.pivot_table.html 12 | 13 | Raises: 14 | NodeException: catches exceptions when dealing with pandas DataFrames. 15 | """ 16 | name = "Pivoting" 17 | num_in = 1 18 | num_out = 3 19 | 20 | OPTIONS = { 21 | 'index': StringParameter( 22 | 'Index', 23 | docstring='Column to aggregate (column, grouper, array or list)' 24 | ), 25 | 'values': StringParameter( 26 | 'Values', 27 | docstring='Column name to use to populate new frame\'s values (column, grouper, array or list)' 28 | ), 29 | 'columns': StringParameter( 30 | 'Column Name Row', 31 | docstring='Column(s) to use for populating new frame values. (column, grouper, array or list)' 32 | ), 33 | 'aggfunc': StringParameter( 34 | 'Aggregation function', 35 | default='mean', 36 | docstring='Function used for aggregation (function, list of functions, dict, default numpy.mean)' 37 | ), 38 | 'fill_value': StringParameter( 39 | 'Fill value', 40 | docstring='Value to replace missing values with (scalar)' 41 | ), 42 | 'margins': BooleanParameter( 43 | 'Margins name', 44 | default=False, 45 | docstring='Add all rows/columns' 46 | ), 47 | 'dropna': BooleanParameter( 48 | 'Drop NaN columns', 49 | default=True, 50 | docstring='Ignore columns with all NaN entries' 51 | ), 52 | 'margins_name': StringParameter( 53 | 'Margins name', 54 | default='All', 55 | docstring='Name of the row/column that will contain the totals when margins is True' 56 | ), 57 | 'observed': BooleanParameter( 58 | 'Column Name Row', 59 | default=False, 60 | docstring='Row number with column names (0-indexed) or "infer"' 61 | ) 62 | } 63 | 64 | def execute(self, predecessor_data, flow_vars): 65 | try: 66 | input_df = pd.DataFrame.from_dict(predecessor_data[0]) 67 | output_df = pd.DataFrame.pivot_table( 68 | input_df, 69 | index=flow_vars['index'].get_value(), 70 | values=flow_vars['values'].get_value(), 71 | columns=flow_vars['columns'].get_value(), 72 | aggfunc=flow_vars['aggfunc'].get_value(), 73 | fill_value=flow_vars['fill_value'].get_value(), 74 | margins=flow_vars['margins'].get_value(), 75 | dropna=flow_vars['dropna'].get_value(), 76 | margins_name=flow_vars['margins_name'].get_value(), 77 | observed=flow_vars['observed'].get_value(), 78 | ) 79 | return output_df.to_json() 80 | except Exception as e: 81 | raise NodeException('pivot', str(e)) 82 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/visualization/__init__.py: -------------------------------------------------------------------------------- 1 | from .graph import GraphNode 2 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/nodes/visualization/graph.py: -------------------------------------------------------------------------------- 1 | from pyworkflow.node import VizNode, NodeException 2 | from pyworkflow.parameters import * 3 | 4 | import pandas as pd 5 | import altair as alt 6 | 7 | 8 | class GraphNode(VizNode): 9 | """Displays a pandas DataFrame in a visual graph. 10 | 11 | Raises: 12 | NodeException: any error generating Altair Chart. 13 | """ 14 | name = "Graph Node" 15 | num_in = 1 16 | num_out = 0 17 | 18 | OPTIONS = { 19 | "graph_type": SelectParameter( 20 | "Graph Type", 21 | options=["area", "bar", "line", "point"], 22 | default="bar", 23 | docstring="Graph viz type" 24 | ), 25 | "mark_options": BooleanParameter( 26 | "Specify mark options", 27 | default=False, 28 | docstring="Specify mark options" 29 | ), 30 | "width": IntegerParameter( 31 | "Mark width", 32 | default=10, 33 | docstring="Width of marks" 34 | ), 35 | "height": IntegerParameter( 36 | "Mark height", 37 | default=10, 38 | docstring="Height of marks" 39 | ), 40 | "encode_options": BooleanParameter( 41 | "Specify encoding options", 42 | default=True, 43 | docstring="Specify encoding options" 44 | ), 45 | "x_axis": StringParameter( 46 | "X-Axis", 47 | default="a", 48 | docstring="X-axis values" 49 | ), 50 | "y_axis": StringParameter( 51 | "Y-Axis", 52 | default="average(b)", 53 | docstring="Y-axis values" 54 | ) 55 | } 56 | 57 | def execute(self, predecessor_data, flow_vars): 58 | try: 59 | df = pd.DataFrame.from_dict(predecessor_data[0]) 60 | 61 | if flow_vars["mark_options"].get_value(): 62 | mark_options = { 63 | "height": flow_vars["height"].get_value(), 64 | "width": flow_vars["width"].get_value(), 65 | } 66 | else: 67 | mark_options = {} 68 | 69 | if flow_vars["encode_options"].get_value(): 70 | encode_options = { 71 | "x": flow_vars["x_axis"].get_value(), 72 | "y": flow_vars["y_axis"].get_value(), 73 | } 74 | else: 75 | encode_options = {} 76 | 77 | graph_type = flow_vars["graph_type"].get_value() 78 | 79 | # Generate requested chart with options 80 | if graph_type == "area": 81 | chart = alt.Chart(df).mark_area(**mark_options).encode(**encode_options) 82 | elif graph_type == "bar": 83 | chart = alt.Chart(df).mark_bar(**mark_options).encode(**encode_options) 84 | elif graph_type == "line": 85 | chart = alt.Chart(df).mark_line(**mark_options).encode(**encode_options) 86 | elif graph_type == "point": 87 | chart = alt.Chart(df).mark_point(**mark_options).encode(**encode_options) 88 | else: 89 | chart = None 90 | 91 | return chart.to_json() 92 | except Exception as e: 93 | print(e) 94 | raise NodeException('graph node', str(e)) 95 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/parameters.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | class Options: 5 | """ 6 | Descriptor for accessing node parameters as Parameter instances 7 | 8 | Clones the values in the class variable `OPTIONS` and sets their values 9 | with the values in in the instance variable `option_values`. 10 | """ 11 | 12 | def __get__(self, obj, objtype): 13 | # return class variable OPTIONS if invoked from class 14 | if obj is None: 15 | return getattr(objtype, "OPTIONS", dict()) 16 | # otherwise clone class's options and set values from instance 17 | options = dict() 18 | for k, v in obj.OPTIONS.items(): 19 | options[k] = v.clone() 20 | for k, v in getattr(obj, "option_values", dict()).items(): 21 | if k in options: 22 | options[k].set_value(v) 23 | return options 24 | 25 | 26 | class OptionTypes: 27 | """ 28 | Descriptor for accessing parameter names, types, and descriptions. 29 | 30 | This will never reference instance parameter values, only turn the 31 | class OPTIONS into a dict. 32 | """ 33 | 34 | def __get__(self, obj, objtype): 35 | # handle both instance- and class-callers 36 | item = obj or objtype 37 | if getattr(item, "OPTIONS", None) is None: 38 | return dict() 39 | return {k: v.to_json() for k, v in item.OPTIONS.items()} 40 | 41 | 42 | class Parameter: 43 | type = None 44 | 45 | def __init__(self, label="", default=None, docstring=None): 46 | self._label = label 47 | self._value = None 48 | self._default = default 49 | self._docstring = docstring 50 | 51 | def clone(self): 52 | return self.__class__(self.label, self.default, self.docstring) 53 | 54 | def get_value(self): 55 | if self._value is None: 56 | return self.default 57 | return self._value 58 | 59 | def set_value(self, value): 60 | self._value = value 61 | 62 | @property 63 | def label(self): 64 | return self._label 65 | 66 | @property 67 | def default(self): 68 | return self._default 69 | 70 | @property 71 | def docstring(self): 72 | return self._docstring 73 | 74 | def validate(self): 75 | raise NotImplementedError() 76 | 77 | def to_json(self): 78 | return { 79 | "type": self.type, 80 | "label": self.label, 81 | "value": self.get_value(), 82 | "docstring": self.docstring 83 | } 84 | 85 | 86 | class FileParameter(Parameter): 87 | type = "file" 88 | 89 | def validate(self): 90 | value = self.get_value() 91 | if (value is None) or (not os.path.exists(value)): 92 | raise ParameterValidationError(self) 93 | 94 | 95 | class StringParameter(Parameter): 96 | type = "string" 97 | 98 | def validate(self): 99 | value = self.get_value() 100 | if not isinstance(value, str): 101 | raise ParameterValidationError(self) 102 | 103 | 104 | class TextParameter(Parameter): 105 | type = "text" 106 | 107 | def validate(self): 108 | value = self.get_value() 109 | if not isinstance(value, str): 110 | raise ParameterValidationError(self) 111 | 112 | 113 | class IntegerParameter(Parameter): 114 | type = "int" 115 | 116 | def validate(self): 117 | value = self.get_value() 118 | if not isinstance(value, int): 119 | raise ParameterValidationError(self) 120 | 121 | 122 | class BooleanParameter(Parameter): 123 | type = "boolean" 124 | 125 | def validate(self): 126 | value = self.get_value() 127 | if not isinstance(value, bool): 128 | raise ParameterValidationError(self) 129 | 130 | 131 | class SelectParameter(Parameter): 132 | type = "select" 133 | 134 | def __init__(self, label="", options=None, default=None, docstring=None): 135 | super().__init__(label, default, docstring) 136 | self.options = options or [] 137 | 138 | def to_json(self): 139 | out = super().to_json() 140 | out["options"] = self.options 141 | return out 142 | 143 | def validate(self): 144 | value = self.get_value() 145 | if not isinstance(value, str): 146 | raise ParameterValidationError(self) 147 | 148 | 149 | class ParameterValidationError(Exception): 150 | 151 | def __init__(self, parameter): 152 | self.parameter = parameter 153 | 154 | def __str__(self): 155 | param = self.parameter 156 | value = param.get_value() 157 | value_type = type(value).__name__ 158 | param_type = type(param).__name__ 159 | return f"Invalid value '{value}' (type '{value_type}') for {param_type}" 160 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/tests/sample_test_data.py: -------------------------------------------------------------------------------- 1 | from pyworkflow import * 2 | 3 | GOOD_NODES = { 4 | "read_csv_node": { 5 | "name": "Read CSV", 6 | "node_id": "1", 7 | "node_type": "io", 8 | "node_key": "ReadCsvNode", 9 | "is_global": False, 10 | "options": { 11 | "file": "/tmp/sample1.csv" 12 | } 13 | }, 14 | "write_csv_node": { 15 | "name": "Write CSV", 16 | "node_id": "2", 17 | "node_type": "io", 18 | "node_key": "WriteCsvNode", 19 | "is_global": False, 20 | "options": { 21 | "file": "/tmp/sample_out.csv" 22 | } 23 | }, 24 | "join_node": { 25 | "name": "Joiner", 26 | "node_id": "3", 27 | "node_type": "manipulation", 28 | "node_key": "JoinNode", 29 | "is_global": False, 30 | "options": { 31 | "on": "to_replace" 32 | }, 33 | "option_replace": { 34 | "on": { 35 | "node_id": "7", 36 | "is_global": False, 37 | } 38 | } 39 | }, 40 | "filter_node": { 41 | "name": "Filter", 42 | "node_id": "4", 43 | "node_type": "manipulation", 44 | "node_key": "FilterNode", 45 | "is_global": False, 46 | "options": { 47 | "on": "key" 48 | } 49 | }, 50 | "pivot_node": { 51 | "name": "Pivoting", 52 | "node_id": "5", 53 | "node_type": "manipulation", 54 | "node_key": "PivotNode", 55 | "is_global": False, 56 | "options": { 57 | "on": "key" 58 | } 59 | }, 60 | "graph_node": { 61 | "name": "Graph", 62 | "node_id": "6", 63 | "node_type": "visualization", 64 | "node_key": "GraphNode", 65 | "is_global": False, 66 | }, 67 | "string_input": { 68 | "name": "String Input", 69 | "node_id": "7", 70 | "node_type": "flow_control", 71 | "node_key": "StringNode", 72 | "is_global": False, 73 | "options": { 74 | "default_value": "key", 75 | "var_name": "local_flow_var" 76 | } 77 | }, 78 | "integer_input": { 79 | "name": "Integer Input", 80 | "node_id": "8", 81 | "node_type": "flow_control", 82 | "node_key": "IntegerNode", 83 | "is_global": False, 84 | "options": { 85 | "default_value": 42, 86 | "var_name": "my_var" 87 | } 88 | }, 89 | "global_flow_var": { 90 | "name": "String Input", 91 | "node_id": "1", 92 | "node_type": "flow_control", 93 | "node_key": "StringNode", 94 | "is_global": True, 95 | "options": { 96 | "default_value": ",", 97 | "var_name": "global_flow_var" 98 | } 99 | }, 100 | } 101 | 102 | BAD_NODES = { 103 | "bad_flow_node": { 104 | "name": "Foobar", 105 | "node_id": "1", 106 | "node_type": "flow_control", 107 | "node_key": "foobar", 108 | "is_global": False, 109 | }, 110 | "bad_io_node": { 111 | "name": "Foobar", 112 | "node_id": "1", 113 | "node_type": "io", 114 | "node_key": "foobar", 115 | "is_global": False, 116 | }, 117 | "bad_manipulation_node": { 118 | "name": "Foobar", 119 | "node_id": "1", 120 | "node_type": "manipulation", 121 | "node_key": "foobar", 122 | "is_global": False, 123 | }, 124 | "bad_visualization_node": { 125 | "name": "Foobar", 126 | "node_id": "1", 127 | "node_type": "visualization", 128 | "node_key": "foobar", 129 | "is_global": False, 130 | }, 131 | "bad_node_type": { 132 | "name": "Foobar", 133 | "node_id": "1", 134 | "node_type": "foobar", 135 | "node_key": "foobar", 136 | "is_global": False, 137 | }, 138 | } 139 | 140 | GOOD_PARAMETERS = { 141 | "string_param": StringParameter( 142 | 'Index', 143 | default='my value', 144 | docstring='my docstring' 145 | ), 146 | "text_param": TextParameter( 147 | 'CSV Input', 148 | default='my value', 149 | docstring='my docstring' 150 | ), 151 | "bool_param": BooleanParameter( 152 | 'Drop NaN columns', 153 | default=True, 154 | docstring='Ignore columns with all NaN entries' 155 | ), 156 | "file_param": FileParameter( 157 | "File", 158 | docstring="CSV File" 159 | ), 160 | "int_param": IntegerParameter( 161 | 'Integer', 162 | default=42, 163 | docstring="CSV File" 164 | ), 165 | "select_param": SelectParameter( 166 | 'Graph type', 167 | options=["area", "bar", "line", "point"], 168 | default='bar', 169 | docstring='my docstring' 170 | ), 171 | } 172 | 173 | BAD_PARAMETERS = { 174 | "bad_string_param": StringParameter( 175 | 'Bad String', 176 | default=42, 177 | docstring="CSV File" 178 | ), 179 | "bad_file_param": FileParameter( 180 | 'Bad File', 181 | default='fobar.csv', 182 | docstring="CSV File" 183 | ), 184 | "bad_int_param": IntegerParameter( 185 | 'Bad Integer', 186 | default="foobar", 187 | docstring="CSV File" 188 | ), 189 | "bad_bool_param": BooleanParameter( 190 | 'Bad Bool Param', 191 | default=42, 192 | docstring="CSV File" 193 | ), 194 | "bad_text_param": TextParameter( 195 | 'Bad Bool Param', 196 | default=42, 197 | docstring="CSV File" 198 | ), 199 | "bad_select_param": SelectParameter( 200 | 'Bad Bool Param', 201 | default=42, 202 | docstring="CSV File" 203 | ), 204 | } 205 | 206 | DATA_FILES = { 207 | "sample1": (',key,A\n' 208 | '0,K0,A0\n' 209 | '1,K1,A1\n' 210 | '2,K2,A2\n' 211 | '3,K3,A3\n' 212 | '4,K4,A4\n' 213 | '5,K5,A5\n'), 214 | "sample2": (',key,B\n' 215 | '0,K0,B0\n' 216 | '1,K1,B1\n' 217 | '2,K2,B2\n'), 218 | "good_custom_node": ('from pyworkflow.node import Node, NodeException\n' 219 | 'from pyworkflow.parameters import *\n' 220 | 'class MyGoodCustomNode(Node):\n' 221 | '\tname="Custom Node"\n' 222 | '\tnum_in=1\n' 223 | '\tnum_out=1\n' 224 | '\tdef execute(self, predecessor_data, flow_vars):\n' 225 | '\t\tprint("Hello world")\n'), 226 | "bad_custom_node": ('from pyworkflow.node import Node, NodeException\n' 227 | 'from pyworkflow.parameters import *\n' 228 | 'import torch\n' 229 | 'class MyBadCustomNode(Node):\n' 230 | '\tname="Custom Node"\n' 231 | '\tnum_in=1\n' 232 | '\tnum_out=1\n' 233 | '\tdef execute(self, predecessor_data, flow_vars):\n' 234 | '\t\tprint("Hello world")\n'), 235 | } -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/tests/test_node.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from pyworkflow import * 3 | from pyworkflow.nodes import * 4 | from pyworkflow.tests.sample_test_data import GOOD_NODES, BAD_NODES, DATA_FILES 5 | 6 | 7 | class NodeTestCase(unittest.TestCase): 8 | def test_add_join_csv_node(self): 9 | node_to_add = node_factory(GOOD_NODES["join_node"]) 10 | self.assertIsInstance(node_to_add, JoinNode) 11 | 12 | def test_add_filter_csv_node(self): 13 | node_to_add = node_factory(GOOD_NODES["filter_node"]) 14 | self.assertIsInstance(node_to_add, FilterNode) 15 | 16 | def test_add_pivot_csv_node(self): 17 | node_to_add = node_factory(GOOD_NODES["pivot_node"]) 18 | self.assertIsInstance(node_to_add, PivotNode) 19 | 20 | def test_add_graph_csv_node(self): 21 | node_to_add = node_factory(GOOD_NODES["graph_node"]) 22 | self.assertIsInstance(node_to_add, GraphNode) 23 | 24 | def test_add_string_node(self): 25 | node_to_add = node_factory(GOOD_NODES["string_input"]) 26 | self.assertIsInstance(node_to_add, StringNode) 27 | 28 | def test_add_integer_node(self): 29 | node_to_add = node_factory(GOOD_NODES["integer_input"]) 30 | self.assertIsInstance(node_to_add, IntegerNode) 31 | 32 | def test_fail_add_node(self): 33 | bad_nodes = [ 34 | node_factory(BAD_NODES["bad_node_type"]), 35 | node_factory(BAD_NODES["bad_flow_node"]), 36 | node_factory(BAD_NODES["bad_io_node"]), 37 | node_factory(BAD_NODES["bad_manipulation_node"]), 38 | node_factory(BAD_NODES["bad_visualization_node"]) 39 | ] 40 | 41 | for bad_node in bad_nodes: 42 | self.assertIsNone(bad_node) 43 | 44 | def test_flow_node_replacement_value(self): 45 | node_to_add = node_factory(GOOD_NODES["string_input"]) 46 | self.assertEqual(node_to_add.get_replacement_value(), "key") 47 | 48 | def test_node_to_string(self): 49 | node_to_add = node_factory(GOOD_NODES["string_input"]) 50 | self.assertEqual(str(node_to_add), "String Input") 51 | 52 | def test_node_to_json(self): 53 | node_to_add = node_factory(GOOD_NODES["string_input"]) 54 | 55 | dict_to_compare = { 56 | "name": "String Input", 57 | "node_id": "7", 58 | "node_type": "flow_control", 59 | "node_key": "StringNode", 60 | "data": None, 61 | "is_global": False, 62 | "option_replace": {}, 63 | "option_values": { 64 | "default_value": "key", 65 | "var_name": "local_flow_var" 66 | } 67 | } 68 | 69 | self.assertDictEqual(node_to_add.to_json(), dict_to_compare) 70 | 71 | def test_node_execute_not_implemented(self): 72 | test_node = Node(dict()) 73 | test_io_node = IONode(dict()) 74 | test_manipulation_node = ManipulationNode(dict()) 75 | test_visualization_node = VizNode(dict()) 76 | 77 | nodes = [test_node, test_io_node, test_manipulation_node, test_visualization_node] 78 | 79 | for node_to_execute in nodes: 80 | with self.assertRaises(NotImplementedError): 81 | node_to_execute.execute(None, None) 82 | 83 | def test_node_execute_exception(self): 84 | read_csv_node = node_factory(GOOD_NODES["read_csv_node"]) 85 | write_csv_node = node_factory(GOOD_NODES["write_csv_node"]) 86 | join_node = node_factory(GOOD_NODES["join_node"]) 87 | 88 | nodes = [read_csv_node, write_csv_node, join_node] 89 | for node_to_execute in nodes: 90 | with self.assertRaises(NodeException): 91 | node_to_execute.execute(dict(), dict()) 92 | 93 | def test_validate_node(self): 94 | node_to_validate = node_factory(GOOD_NODES["string_input"]) 95 | node_to_validate.validate() 96 | 97 | def test_validate_input_data(self): 98 | node_to_validate = node_factory(GOOD_NODES["join_node"]) 99 | node_to_validate.validate_input_data(2) 100 | 101 | def test_validate_input_data_exception(self): 102 | node_to_validate = node_factory(GOOD_NODES["join_node"]) 103 | 104 | try: 105 | node_to_validate.validate_input_data(0) 106 | except NodeException as e: 107 | self.assertEqual(str(e), "execute: JoinNode requires 2 inputs. 0 were provided") 108 | 109 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/tests/test_parameters.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from pyworkflow import * 3 | import networkx as nx 4 | from pyworkflow.tests.sample_test_data import GOOD_PARAMETERS, BAD_PARAMETERS 5 | 6 | 7 | class ParameterTestCase(unittest.TestCase): 8 | def test_string_param(self): 9 | full_json = { 10 | 'type': 'string', 11 | 'label': 'Index', 12 | 'value': 'my value', 13 | 'docstring': 'my docstring' 14 | } 15 | 16 | self.assertDictEqual(GOOD_PARAMETERS["string_param"].to_json(), full_json) 17 | 18 | def test_parameter_validate_not_implemented(self): 19 | test_param = Parameter(dict()) 20 | params = [test_param] 21 | 22 | for param_to_validate in params: 23 | with self.assertRaises(NotImplementedError): 24 | param_to_validate.validate() 25 | 26 | def test_validate_string_param(self): 27 | with self.assertRaises(ParameterValidationError): 28 | BAD_PARAMETERS["bad_string_param"].validate() 29 | 30 | def test_validate_integer_param(self): 31 | with self.assertRaises(ParameterValidationError): 32 | BAD_PARAMETERS["bad_int_param"].validate() 33 | 34 | def test_validate_boolean_param(self): 35 | with self.assertRaises(ParameterValidationError): 36 | BAD_PARAMETERS["bad_bool_param"].validate() 37 | 38 | def test_validate_text_param(self): 39 | with self.assertRaises(ParameterValidationError): 40 | BAD_PARAMETERS["bad_text_param"].validate() 41 | 42 | def test_validate_select_param(self): 43 | with self.assertRaises(ParameterValidationError): 44 | BAD_PARAMETERS["bad_select_param"].validate() 45 | 46 | def test_validate_file_param(self): 47 | with self.assertRaises(ParameterValidationError): 48 | BAD_PARAMETERS["bad_file_param"].validate() 49 | 50 | def test_parameter_validation_error(self): 51 | try: 52 | BAD_PARAMETERS["bad_string_param"].validate() 53 | except ParameterValidationError as e: 54 | self.assertEqual(str(e), "Invalid value '42' (type 'int') for StringParameter") 55 | 56 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/tests/test_pyworkflow.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from pyworkflow import Workflow, WorkflowException, Node, NodeException, node_factory 3 | from pyworkflow.nodes import * 4 | import networkx as nx 5 | 6 | from pyworkflow.tests.sample_test_data import GOOD_NODES, BAD_NODES, DATA_FILES 7 | 8 | 9 | class PyWorkflowTestCase(unittest.TestCase): 10 | def setUp(self): 11 | 12 | with open('/tmp/sample1.csv', 'w') as f: 13 | f.write(DATA_FILES["sample1"]) 14 | 15 | with open('/tmp/sample2.csv', 'w') as f: 16 | f.write(DATA_FILES["sample2"]) 17 | 18 | self.pyworkflow = Workflow("My Workflow", root_dir="/tmp") 19 | 20 | self.read_csv_node_1 = Node(GOOD_NODES["read_csv_node"]) 21 | 22 | self.read_csv_node_2 = Node({ 23 | "name": "Read CSV", 24 | "node_id": "2", 25 | "node_type": "io", 26 | "node_key": "ReadCsvNode", 27 | "is_global": False, 28 | "options": { 29 | "file": "/tmp/sample2.csv", 30 | "sep": ";", 31 | }, 32 | "option_replace": { 33 | "sep": { 34 | "node_id": "1", 35 | "is_global": True, 36 | } 37 | } 38 | }) 39 | 40 | self.join_node = Node(GOOD_NODES["join_node"]) 41 | 42 | self.write_csv_node = Node({ 43 | "name": "Write CSV", 44 | "node_id": "4", 45 | "node_type": "io", 46 | "node_key": "WriteCsvNode", 47 | "is_global": False, 48 | "options": { 49 | "file": "/tmp/sample_out.csv" 50 | } 51 | }) 52 | 53 | self.string_flow_node = Node(GOOD_NODES["string_input"]) 54 | self.string_global_flow_node = Node(GOOD_NODES["global_flow_var"]) 55 | 56 | self.nodes = [ 57 | self.read_csv_node_1, 58 | self.read_csv_node_2, 59 | self.join_node, 60 | self.write_csv_node, 61 | self.string_flow_node, 62 | self.string_global_flow_node, 63 | ] 64 | self.edges = [("1", "3"), ("2", "3"), ("3", "4"), ("7", "3")] 65 | 66 | def create_workflow(self): 67 | # When created in setUp(), duplicate Node/Edge errors would arise 68 | for node in self.nodes: 69 | self.pyworkflow.update_or_add_node(node) 70 | 71 | for edge in self.edges: 72 | source_node = self.pyworkflow.get_node(edge[0]) 73 | target_node = self.pyworkflow.get_node(edge[1]) 74 | self.pyworkflow.add_edge(source_node, target_node) 75 | 76 | def test_get_local_flow_nodes(self): 77 | node_with_flow = self.pyworkflow.get_node("3") 78 | flow_nodes = self.pyworkflow.load_flow_nodes(node_with_flow.option_replace) 79 | self.assertEqual(len(flow_nodes), 1) 80 | 81 | def test_get_global_flow_nodes(self): 82 | node_with_flow = self.pyworkflow.get_node("2") 83 | flow_nodes = self.pyworkflow.load_flow_nodes(node_with_flow.option_replace) 84 | self.assertEqual(len(flow_nodes), 1) 85 | 86 | def test_get_global_flow_node_exception(self): 87 | node_with_flow = self.pyworkflow.get_node("1") 88 | flow_nodes = self.pyworkflow.load_flow_nodes(node_with_flow.option_replace) 89 | self.assertEqual(len(flow_nodes), 0) 90 | 91 | def test_get_execution_order(self): 92 | self.create_workflow() 93 | order = self.pyworkflow.execution_order() 94 | self.assertEqual(order, ["7", "2", "1", "3", "4"]) 95 | 96 | def test_xexecute_workflow(self): 97 | order = self.pyworkflow.execution_order() 98 | 99 | for node in order: 100 | executed_node = self.pyworkflow.execute(node) 101 | self.pyworkflow.update_or_add_node(executed_node) 102 | 103 | # def test_execute_workflow_load_data(self): 104 | # print(self.pyworkflow.graph.nodes) 105 | # data = self.pyworkflow.load_input_data("3") 106 | 107 | def test_fail_execute_node(self): 108 | with self.assertRaises(WorkflowException): 109 | self.pyworkflow.execute("100") 110 | 111 | def test_upload_file(self): 112 | with open('/tmp/sample1.csv', 'rb') as f: 113 | to_open = '/tmp/sample_upload.csv' 114 | saved_filed = self.pyworkflow.upload_file(f, to_open) 115 | 116 | self.assertEqual(to_open, saved_filed) 117 | -------------------------------------------------------------------------------- /back-end/pyworkflow/pyworkflow/tests/test_workflow.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | from pyworkflow import Workflow, WorkflowException, Node, NodeException, node_factory 4 | import networkx as nx 5 | 6 | from pyworkflow.tests.sample_test_data import GOOD_NODES, BAD_NODES, DATA_FILES 7 | 8 | 9 | class WorkflowTestCase(unittest.TestCase): 10 | def setUp(self): 11 | self.workflow = Workflow("Untitled", root_dir="/tmp", node_dir=os.path.join(os.getcwd(), 'nodes')) 12 | 13 | self.read_csv_node = GOOD_NODES["read_csv_node"] 14 | self.local_flow_node = GOOD_NODES["string_input"] 15 | self.write_csv_node = GOOD_NODES["write_csv_node"] 16 | self.join_node = GOOD_NODES["join_node"] 17 | self.global_flow_var = GOOD_NODES["global_flow_var"] 18 | 19 | def add_node(self, node_info, node_id): 20 | node_info["node_id"] = node_id 21 | node_to_add = Node(node_info) 22 | return self.workflow.update_or_add_node(node_to_add) 23 | 24 | def add_edge(self, source_node, target_node): 25 | response = self.workflow.add_edge(source_node, target_node) 26 | self.assertEqual(response, (source_node.node_id, target_node.node_id)) 27 | 28 | ########################## 29 | # Workflow getters/setters 30 | ########################## 31 | def test_workflow_name(self): 32 | self.assertEqual(self.workflow.name, "Untitled") 33 | 34 | def test_workflow_dir_os_error(self): 35 | try: 36 | os.makedirs("foobar", 0000) 37 | test_workflow = Workflow(node_dir="foobar") 38 | except WorkflowException as e: 39 | self.assertEqual(e.action, "init workflow") 40 | 41 | def test_workflow_node_dir(self): 42 | self.assertEqual(self.workflow.node_dir, os.path.join(os.getcwd(), 'nodes')) 43 | 44 | def test_workflow_node_path(self): 45 | self.assertEqual(self.workflow.node_path('io', 'read_csv.py'), os.path.join(os.getcwd(), 'nodes/io/read_csv.py')) 46 | 47 | def test_set_workflow_name(self): 48 | self.workflow.name = "My Workflow" 49 | self.assertEqual(self.workflow.name, "My Workflow") 50 | 51 | def test_workflow_filename(self): 52 | self.assertEqual(self.workflow.filename, "Untitled.json") 53 | 54 | def test_workflow_from_json(self): 55 | new_workflow = Workflow("Untitled", root_dir="/tmp") 56 | workflow_copy = Workflow.from_json(self.workflow.to_json()) 57 | 58 | self.assertEqual(new_workflow.name, workflow_copy.name) 59 | 60 | def test_workflow_from_json_key_error(self): 61 | with self.assertRaises(WorkflowException): 62 | new_workflow = Workflow.from_json(dict()) 63 | 64 | def test_empty_workflow_to_session(self): 65 | new_workflow = Workflow("Untitled", root_dir="/tmp", node_dir=os.path.join(os.getcwd(), 'nodes')) 66 | saved_workflow = new_workflow.to_json() 67 | 68 | workflow_to_compare = { 69 | 'name': 'Untitled', 70 | 'root_dir': '/tmp', 71 | 'node_dir': os.path.join(os.getcwd(), 'nodes'), 72 | 'graph': Workflow.to_graph_json(new_workflow.graph), 73 | 'flow_vars': Workflow.to_graph_json(new_workflow.flow_vars), 74 | } 75 | self.assertDictEqual(new_workflow.to_json(), workflow_to_compare) 76 | 77 | ########################## 78 | # Node lists 79 | ########################## 80 | def test_workflow_packaged_nodes(self): 81 | nodes = self.workflow.get_packaged_nodes() 82 | self.assertEqual(len(nodes), 5) 83 | 84 | def test_workflow_packaged_nodes_exception(self): 85 | result = self.workflow.get_packaged_nodes(root_path="foobar") 86 | self.assertIsNone(result) 87 | 88 | def test_get_flow_variables(self): 89 | flow_var_options = self.workflow.get_all_flow_var_options("1") 90 | 91 | self.assertEqual(len(flow_var_options), 1) 92 | 93 | def test_get_node_successors(self): 94 | successors = self.workflow.get_node_successors("1") 95 | 96 | self.assertEqual(successors, ["3", "2"]) 97 | 98 | def test_fail_get_node_successors(self): 99 | try: 100 | successors = self.workflow.get_node_successors("100") 101 | except WorkflowException as e: 102 | self.assertEqual(str(e), "get node successors: The node 100 is not in the digraph.") 103 | 104 | def test_fail_get_node_predecessors(self): 105 | with self.assertRaises(WorkflowException): 106 | predecessors = self.workflow.get_node_predecessors("200") 107 | 108 | def test_get_node_predecessors(self): 109 | predecessors = self.workflow.get_node_predecessors("2") 110 | 111 | self.assertEqual(predecessors, ["1"]) 112 | 113 | def test_fail_get_execution_order(self): 114 | copied_workflow = self.workflow 115 | with self.assertRaises(WorkflowException): 116 | copied_workflow._graph = nx.Graph() 117 | copied_workflow.execution_order() 118 | 119 | ########################## 120 | # Node operations 121 | ########################## 122 | def test_add_custom_node(self): 123 | with open(self.workflow.node_path('custom_nodes', 'good_custom_node.py'), 'w') as f: 124 | f.write((DATA_FILES['good_custom_node'])) 125 | 126 | custom_node_info = { 127 | "name": "Custom Node", 128 | "node_id": "50", 129 | "node_type": "custom_node", 130 | "node_key": "MyGoodCustomNode", 131 | "is_global": False, 132 | } 133 | 134 | node_to_add = Node(custom_node_info) 135 | added_node = self.add_node(custom_node_info, "50") 136 | 137 | self.assertDictEqual(node_to_add.__dict__, added_node.__dict__) 138 | 139 | def test_add_read_csv_node(self): 140 | node_to_add = Node(self.read_csv_node) 141 | added_node = self.add_node(self.read_csv_node, "1") 142 | 143 | self.assertDictEqual(node_to_add.__dict__, added_node.__dict__) 144 | 145 | def test_add_write_csv_node(self): 146 | node_to_add = Node(self.write_csv_node) 147 | added_node = self.add_node(self.write_csv_node, "2") 148 | 149 | self.assertDictEqual(node_to_add.__dict__, added_node.__dict__) 150 | 151 | def test_get_node(self): 152 | retrieved_node = self.workflow.get_node("1") 153 | read_csv_node = Node(self.read_csv_node) 154 | 155 | self.assertDictEqual(retrieved_node.__dict__, read_csv_node.__dict__) 156 | 157 | def test_fail_get_node(self): 158 | retrieved_node = self.workflow.get_flow_var("100") 159 | 160 | self.assertIsNone(retrieved_node) 161 | 162 | def test_remove_node(self): 163 | node_to_remove = self.workflow.get_node("1") 164 | removed_node = self.workflow.remove_node(node_to_remove) 165 | 166 | self.assertDictEqual(node_to_remove.__dict__, removed_node.__dict__) 167 | 168 | def test_remove_node_error(self): 169 | node_to_remove = self.workflow.get_node("1") 170 | with self.assertRaises(WorkflowException): 171 | self.workflow.remove_node(node_to_remove) 172 | 173 | ########################## 174 | # Flow variable operations 175 | ########################## 176 | def test_add_string_node(self): 177 | node_to_add = Node(self.global_flow_var) 178 | added_node = self.add_node(self.global_flow_var, "1") 179 | 180 | self.assertDictEqual(node_to_add.__dict__, added_node.__dict__) 181 | 182 | def test_get_flow_var(self): 183 | retrieved_node = self.workflow.get_flow_var("1") 184 | global_flow_var = Node(self.global_flow_var) 185 | 186 | self.assertDictEqual(retrieved_node.__dict__, global_flow_var.__dict__) 187 | 188 | ########################## 189 | # Edge operations 190 | ########################## 191 | def test_add_node_edge_1_to_2(self): 192 | node_1 = self.workflow.get_node("1") 193 | node_2 = self.workflow.get_node("2") 194 | self.add_edge(node_1, node_2) 195 | return 196 | 197 | def test_add_node_edge_duplicated(self): 198 | node_1 = self.workflow.get_node("1") 199 | node_2 = self.workflow.get_node("2") 200 | 201 | with self.assertRaises(WorkflowException): 202 | self.workflow.add_edge(node_1, node_2) 203 | 204 | def test_remove_edge(self): 205 | node_1 = self.workflow.get_node("1") 206 | node_2 = self.workflow.get_node("2") 207 | response = self.workflow.remove_edge(node_1, node_2) 208 | 209 | self.assertEqual(response, ("1", "2")) 210 | 211 | def test_remove_edge_error(self): 212 | node_1 = self.workflow.get_node("1") 213 | node_2 = self.workflow.get_node("2") 214 | 215 | with self.assertRaises(WorkflowException): 216 | self.workflow.remove_edge(node_1, node_2) 217 | 218 | ########################## 219 | # Flow variable operations 220 | ########################## 221 | def test_execute_node(self): 222 | node_to_execute = self.workflow.get_node("1") 223 | 224 | response = self.workflow.execute("1") 225 | node_to_execute.data = 'Untitled-1' 226 | 227 | self.assertDictEqual(node_to_execute.__dict__, response.__dict__) 228 | 229 | def test_fail_execute_node(self): 230 | with self.assertRaises(WorkflowException): 231 | self.workflow.execute("100") 232 | 233 | ########################## 234 | # File I/O operations 235 | ########################## 236 | def test_download_file(self): 237 | file = self.workflow.download_file("1") 238 | 239 | self.assertEqual(file.name, "/tmp/sample1.csv") 240 | file.close() 241 | 242 | def test_download_file_error(self): 243 | self.assertIsNone(self.workflow.download_file("100")) 244 | 245 | def test_download_file_wrong_type(self): 246 | with self.assertRaises(WorkflowException): 247 | self.workflow.download_file("3") 248 | 249 | -------------------------------------------------------------------------------- /back-end/pyworkflow/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup(name='pyworkflow', 4 | version='0.0.0', 5 | description='Python representation of visual programming workflows', 6 | author='Visual Programming Team', 7 | license='MIT', 8 | packages=['pyworkflow'], 9 | zip_safe=False) -------------------------------------------------------------------------------- /back-end/vp/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | import dotenv 7 | 8 | 9 | def main(): 10 | dotenv.load_dotenv(".environment") 11 | 12 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'vp.settings') 13 | try: 14 | from django.core.management import execute_from_command_line 15 | except ImportError as exc: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) from exc 21 | execute_from_command_line(sys.argv) 22 | 23 | 24 | if __name__ == '__main__': 25 | main() 26 | -------------------------------------------------------------------------------- /back-end/vp/node/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/vp/node/__init__.py -------------------------------------------------------------------------------- /back-end/vp/node/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /back-end/vp/node/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class NodeConfig(AppConfig): 5 | name = 'node' 6 | -------------------------------------------------------------------------------- /back-end/vp/node/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/vp/node/migrations/__init__.py -------------------------------------------------------------------------------- /back-end/vp/node/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/vp/node/models.py -------------------------------------------------------------------------------- /back-end/vp/node/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /back-end/vp/node/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from . import views 3 | 4 | urlpatterns = [ 5 | path('', views.node, name='node'), 6 | path('', views.handle_node, name='handle node'), 7 | path('global/', views.handle_node, name='handle node'), 8 | path('/execute', views.execute_node, name='execute node'), 9 | path('/retrieve_data', views.retrieve_data, name='retrieve data'), 10 | path('edge//', views.handle_edge, name='handle edge') 11 | ] 12 | -------------------------------------------------------------------------------- /back-end/vp/vp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/vp/vp/__init__.py -------------------------------------------------------------------------------- /back-end/vp/vp/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for vp project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'vp.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /back-end/vp/vp/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for vp project. 3 | 4 | Generated by 'django-admin startproject' using Django 3.0.4. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/3.0/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = os.environ['SECRET_KEY'] 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = ['back-end:8000', 29 | 'back-end', 30 | 'localhost', 31 | '127.0.0.1', 32 | '0.0.0.0', 33 | '[::1]'] 34 | 35 | 36 | # Application definition 37 | 38 | INSTALLED_APPS = [ 39 | 'django.contrib.admin', 40 | 'django.contrib.auth', 41 | 'django.contrib.contenttypes', 42 | 'django.contrib.sessions', 43 | 'django.contrib.messages', 44 | 'django.contrib.staticfiles', 45 | # Project apps 46 | 'workflow.apps.WorkflowConfig', 47 | 'node.apps.NodeConfig', 48 | 'drf_yasg' 49 | ] 50 | 51 | MIDDLEWARE = [ 52 | 'django.middleware.security.SecurityMiddleware', 53 | 'django.contrib.sessions.middleware.SessionMiddleware', 54 | 'django.middleware.common.CommonMiddleware', 55 | 'django.middleware.csrf.CsrfViewMiddleware', 56 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 57 | 'django.contrib.messages.middleware.MessageMiddleware', 58 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 59 | 60 | # Custom middleware 61 | 'workflow.middleware.WorkflowMiddleware', 62 | ] 63 | 64 | ROOT_URLCONF = 'vp.urls' 65 | 66 | TEMPLATES = [ 67 | { 68 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 69 | 'DIRS': [], 70 | 'APP_DIRS': True, 71 | 'OPTIONS': { 72 | 'context_processors': [ 73 | 'django.template.context_processors.debug', 74 | 'django.template.context_processors.request', 75 | 'django.contrib.auth.context_processors.auth', 76 | 'django.contrib.messages.context_processors.messages', 77 | ], 78 | }, 79 | }, 80 | ] 81 | 82 | SESSION_ENGINE = 'django.contrib.sessions.backends.file' 83 | 84 | WSGI_APPLICATION = 'vp.wsgi.application' 85 | 86 | 87 | # Database 88 | # https://docs.djangoproject.com/en/3.0/ref/settings/#databases 89 | # Not yet setup 90 | DATABASES = {} 91 | 92 | MEDIA_ROOT = '/tmp' 93 | 94 | # Password validation 95 | # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators 96 | 97 | AUTH_PASSWORD_VALIDATORS = [ 98 | { 99 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 100 | }, 101 | { 102 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 103 | }, 104 | { 105 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 106 | }, 107 | { 108 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 109 | }, 110 | ] 111 | 112 | 113 | # Internationalization 114 | # https://docs.djangoproject.com/en/3.0/topics/i18n/ 115 | 116 | LANGUAGE_CODE = 'en-us' 117 | 118 | TIME_ZONE = 'UTC' 119 | 120 | USE_I18N = True 121 | 122 | USE_L10N = True 123 | 124 | USE_TZ = True 125 | 126 | 127 | # Static files (CSS, JavaScript, Images) 128 | # https://docs.djangoproject.com/en/3.0/howto/static-files/ 129 | 130 | STATIC_URL = '/static/' 131 | -------------------------------------------------------------------------------- /back-end/vp/vp/urls.py: -------------------------------------------------------------------------------- 1 | """vp URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/3.0/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import path, include 18 | from rest_framework import permissions 19 | 20 | from . import views 21 | 22 | from drf_yasg.views import get_schema_view 23 | from drf_yasg import openapi 24 | 25 | schema_view = get_schema_view( 26 | openapi.Info( 27 | title="Visual Programming API", 28 | default_version='v1', 29 | description="Back-end documentation for Visual Programming KNIME based application." 30 | ), 31 | public=True, 32 | permission_classes=(permissions.AllowAny,) 33 | ) 34 | 35 | urlpatterns = [ 36 | path('schema/', schema_view.without_ui(cache_timeout=0), name='schema-json'), 37 | path('swagger/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), 38 | path('redoc/', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), 39 | path('admin/', admin.site.urls), 40 | path('info/', views.info), 41 | path('node/', include('node.urls')), 42 | path('workflow/', include('workflow.urls')) 43 | ] 44 | -------------------------------------------------------------------------------- /back-end/vp/vp/views.py: -------------------------------------------------------------------------------- 1 | from django.http import JsonResponse 2 | from rest_framework.decorators import api_view 3 | from drf_yasg.utils import swagger_auto_schema 4 | 5 | 6 | @swagger_auto_schema(method='get', responses={200: 'JSON response with data'}) 7 | @api_view(['GET']) 8 | def info(request): 9 | """Retrieve app info. 10 | 11 | Args: 12 | request: Django request Object 13 | 14 | Returns: 15 | 200 - JSON response with data. 16 | """ 17 | data = { 18 | "application": "visual_programming", 19 | "version": "negative something", 20 | "about": "super-duper workflows!" 21 | } 22 | return JsonResponse(data) 23 | -------------------------------------------------------------------------------- /back-end/vp/vp/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for vp project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'vp.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /back-end/vp/workflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/vp/workflow/__init__.py -------------------------------------------------------------------------------- /back-end/vp/workflow/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /back-end/vp/workflow/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class WorkflowConfig(AppConfig): 5 | name = 'workflow' 6 | -------------------------------------------------------------------------------- /back-end/vp/workflow/middleware.py: -------------------------------------------------------------------------------- 1 | from pyworkflow import Workflow, WorkflowException 2 | from django.http import JsonResponse 3 | 4 | 5 | class WorkflowMiddleware: 6 | """ Custom middleware 7 | 8 | https://docs.djangoproject.com/en/3.0/topics/http/middleware/ 9 | """ 10 | def __init__(self, get_response): 11 | self.get_response = get_response 12 | 13 | # One-time configuration and initialization. 14 | 15 | def __call__(self, request): 16 | # Code executed each request before view (and later middleware) called 17 | 18 | path = request.path 19 | 20 | if not path.startswith('/workflow/') and not path.startswith('/node/'): 21 | # Workflow needed only for /workflow and /node routes 22 | pass 23 | elif path == '/workflow/open' or path == '/workflow/new': 24 | # 'open' loads from file upload, 'new' inits new Workflow 25 | pass 26 | else: 27 | # All other cases, load workflow from session 28 | try: 29 | request.pyworkflow = Workflow.from_json(request.session) 30 | 31 | # Check if a graph is present 32 | if request.pyworkflow.graph is None: 33 | return JsonResponse({ 34 | 'message': 'A workflow has not been created yet.' 35 | }, status=404) 36 | except WorkflowException as e: 37 | return JsonResponse({e.action: e.reason}, status=500) 38 | 39 | response = self.get_response(request) 40 | 41 | # Code executed for each request/response after the view is called 42 | 43 | # Request should have 'pyworkflow' attribute, but do not crash if not 44 | if hasattr(request, 'pyworkflow'): 45 | # Save Workflow back to session 46 | request.session.update(request.pyworkflow.to_json()) 47 | 48 | return response 49 | -------------------------------------------------------------------------------- /back-end/vp/workflow/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/vp/workflow/migrations/__init__.py -------------------------------------------------------------------------------- /back-end/vp/workflow/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PyWorkflowApp/visual-programming/af02519a57062f4569b009a7f217718533a5f653/back-end/vp/workflow/models.py -------------------------------------------------------------------------------- /back-end/vp/workflow/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /back-end/vp/workflow/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from . import views 3 | 4 | urlpatterns = [ 5 | path('new', views.new_workflow, name='new workflow'), 6 | path('open', views.open_workflow, name='open workflow'), 7 | path('edit', views.edit_workflow, name='edit workflow'), 8 | path('save', views.save_workflow, name='save'), 9 | path('execute', views.execute_workflow, name='execute workflow'), 10 | path('execute//successors', views.get_successors, name='get node successors'), 11 | path('globals', views.global_vars, name="retrieve global variables"), 12 | path('upload', views.upload_file, name='upload file'), 13 | path('download', views.download_file, name='download file'), 14 | path('nodes', views.retrieve_nodes_for_user, name='retrieve node list'), 15 | ] 16 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | services: 3 | back-end: 4 | build: ./back-end 5 | ports: 6 | - "8000:8000" 7 | command: bash -c "pipenv run python manage.py runserver 0.0.0.0:8000" 8 | environment: 9 | - DJANGO_ENV=development 10 | front-end: 11 | build: ./front-end 12 | stdin_open: true 13 | ports: 14 | - "3000:3000" 15 | environment: 16 | - NODE_ENV=development 17 | depends_on: 18 | - back-end 19 | links: 20 | - back-end 21 | command: npm start 22 | -------------------------------------------------------------------------------- /docs/cli.md: -------------------------------------------------------------------------------- 1 | # Command-line Interface 2 | 3 | PyWorkflow is first-and-foremost a visual programming application, designed to 4 | help data scientists and many others build workflows to view, manipulate, and 5 | output their data into new formats. Therefore, all workflows must first be 6 | created via the user-interface and saved for later execution. 7 | 8 | However, it may not always be ideal to have the client and server deployed 9 | locally or on a remote server just to run your workflows. Power-users want the 10 | ability to running multiple workflows at once, schedule workflow runs, and 11 | dynamically pass data from workflows via stdin/stdout in traditional shell 12 | scripts. This is where the inclusion of PyWorkflow's CLI really shines. 13 | 14 | ## Command-line syntax 15 | 16 | ``` 17 | pyworkflow execute workflow-file... 18 | ``` 19 | ### Commands 20 | 21 | #### Execute 22 | Accepts one or more workflow files as arguments to execute. PyWorkflow will load 23 | the file(s) specified and output status messages to `stdout`. If a workflow 24 | fails to run because of an exception, these will be logged to `stderr`. 25 | 26 | **Single-file example** 27 | ``` 28 | pyworkflow execute ./workflows/my_workflow.json 29 | ``` 30 | 31 | **Batch processing** 32 | 33 | Many shells offer different wildcards that can be used to work with multiple 34 | files on the command line, or in scripts. A useful one is the `*` wildcard that 35 | matches matches anything. Used in the following example, it has the effect of 36 | passing all files located within the `workflows` directory to the `execute` 37 | command. 38 | 39 | ``` 40 | pyworkflow execute ./workflows/* 41 | ``` 42 | 43 | ## Using `stdin`/`stdout` to modify workflows 44 | 45 | Two powerful tools when writing shell scripts are redirection and pipes, which 46 | allow you to dynamically pass data from one command to another. Using these 47 | tools, you can pass different data in to and out of workflows that define what 48 | standard behavior should occur. 49 | 50 | PyWorkflow comes with a Read CSV input node and Write CSV output node. When data 51 | is provided via `stdin` on the command-line, it will modify the workflow 52 | behavior to redirect the Read CSV node to that data. Similarly, if a destination 53 | is specified for `stdout`, the Write CSV node output will be redirected there. 54 | 55 | Input data can be passed to PyWorkflow in a few ways. 56 | 1) Redirection 57 | ``` 58 | # Data from sample_file.csv is passed to a Read CSV node 59 | pyworkflow execute my_workflow.json < sample_file.csv 60 | ``` 61 | 2) Pipes 62 | ``` 63 | # Two CSV files are combined and passed in to a Read CSV node 64 | cat sample_file.csv more_data.csv | pyworkflow execute my_workflow.json 65 | 66 | # Data from a 'csv_exporter' tool is passed to a Read CSV node 67 | csv_exporter generate | pyworkflow execute my_workflow.json 68 | ``` 69 | 70 | Output data can be passed from PyWorkflow in a few ways. 71 | 1) Redirection 72 | ``` 73 | # Output from a Write CSV node is stored in a new file 'output.csv' 74 | pyworkflow execute my_workflow.json > output.csv 75 | ``` 76 | 2) Pipes 77 | ``` 78 | # Output from a Write CSV node is searched for the phrase 'foobar' 79 | pyworkflow execute my_workflow.json | grep "foobar" 80 | ``` 81 | -------------------------------------------------------------------------------- /docs/custom_nodes.md: -------------------------------------------------------------------------------- 1 | # Custom Nodes 2 | The power of PyWorkflow comes from its support for custom nodes. New data 3 | science and other Python packages are being constantly developed. With custom 4 | nodes, you can write workflows tailored to your specific needs and packages 5 | needed for your specific field. 6 | 7 | Custom nodes were designed to be easily written and greatly expandable. You 8 | don't need to worry about React, Django, or any specifics of PyWorkflow to get 9 | started. All you need is: 10 | 11 | 1) Create a `.py` file that subclass the main Node class. 12 | 2) Add any parameters you need for your node might need for execution. 13 | 3) Write an `execute()` method using your package of choice. 14 | 4) That's it! 15 | 16 | The rest is handled for you, from flow variable overrides, to input data from 17 | other nodes in the workflow. 18 | 19 | # Getting started 20 | A custom node will look something like the following. 21 | ```python 22 | from pyworkflow.node import Node, NodeException 23 | from pyworkflow.parameters import * 24 | import pandas as pd 25 | 26 | 27 | class MyCustomNode(Node): 28 | name = "My Node Name" 29 | num_in = 1 30 | num_out = 1 31 | 32 | OPTIONS = { 33 | "input": StringParameter( 34 | "My Input Parameter", 35 | default="", 36 | docstring="A place to provide input" 37 | ) 38 | } 39 | 40 | def execute(self, predecessor_data, flow_vars): 41 | try: 42 | # Do custom node operations here 43 | my_json_data = {"message": flow_vars["input"].get_value()} 44 | return my_json_data 45 | except Exception as e: 46 | raise NodeException('my_node', str(e)) 47 | ``` 48 | 49 | Let's break it down to see how you can take this example and make your own 50 | custom node! 51 | 52 | ## Imports 53 | All custom nodes require a few classes defined by the PyWorkflow package. In the 54 | example above, we import `Node`, `NodeException`, and all (`*`) classes from 55 | the `parameters.py` file. If you take a look at `pyworkflow/node.py`, you'll see 56 | there's several subclasses defined in addition to `Node`. These classes are 57 | described in their docstring comments and include: 58 | - FlowNode: for flow variable parameter overrides 59 | - IONode: for reading/writing data 60 | - ManipulationNode: for altering data 61 | - VizNode: for visualizing data with graphs, charts, etc. 62 | 63 | In the example above, we subclass the main `Node` class, but you can also 64 | import/subclass one of the others mentioned above depending on your use case. 65 | 66 | The final line, `import pandas as pd` is important as all PyWorkflow nodes use 67 | a pandas DataFrame as the atom of data representation. If your custom node 68 | reads or writes data, it must start or end with a pandas DataFrame. 69 | 70 | ## Class attributes 71 | You'll see there are three class-level attributes defined in the example above. 72 | This information is used by both the front- and back-ends to properly display 73 | and validate your custom node. The attributes are: 74 | - `name`: The display name you want your node to have in the UI. 75 | - `num_in`: The number of 'in' ports your node accepts. 76 | - `num_out`: The number of 'out' ports your node accepts. 77 | 78 | ## Parameter options 79 | The next part of the example is the `OPTIONS` dictionary that defines any number 80 | of parameters your custom node might need for execution. You can find out more 81 | about the different parameter types in `pyworkflow/parameters.py`, but for a 82 | general overview, there is: 83 | - `FileParameter`: accepts a file-upload in the configuration form 84 | - `StringParameter`: accepts any string input, displayed as `` 85 | - `TextParameter`: accepts any string input, but displayed as an HTML `