├── .gitignore ├── LICENSE ├── README.md ├── docker ├── Dockerfile ├── Makefile └── start.sh ├── docs └── sample_add.png ├── repo └── repo.json ├── src ├── fbp │ ├── __init__.py │ ├── flow.py │ ├── node.py │ ├── port.py │ └── repository.py ├── message │ └── __init__.py ├── node_specs │ ├── __init__.py │ ├── add.py │ ├── cli.py │ ├── jsonpath.py │ └── split.py ├── nodemaker.py ├── repo.db ├── server.py └── static │ ├── bower.json │ ├── css │ ├── OpenSans-Bold.ttf │ ├── OpenSans-BoldItalic.ttf │ ├── OpenSans-ExtraBold.ttf │ ├── OpenSans-ExtraBoldItalic.ttf │ ├── OpenSans-Italic.ttf │ ├── OpenSans-Light.ttf │ ├── OpenSans-LightItalic.ttf │ ├── OpenSans-Regular.ttf │ ├── OpenSans-Semibold.ttf │ ├── OpenSans-SemiboldItalic.ttf │ ├── OpenSans.woff │ ├── images │ │ ├── bg_hr.png │ │ ├── blacktocat.png │ │ ├── collapse.png │ │ ├── expand.png │ │ ├── icon_download.png │ │ └── sprite_download.png │ └── pyflow.css │ ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.svg │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ └── glyphicons-halflings-regular.woff2 │ ├── img │ └── animated-progress.gif │ ├── index.html │ ├── js │ ├── app.js │ ├── components │ │ ├── flowCanvas.js │ │ ├── flowInspector.js │ │ ├── nodeCodePanel.js │ │ ├── nodeListPanel.js │ │ ├── nodePropertyPanel.js │ │ └── treeview.js │ ├── flow.js │ ├── model │ │ ├── flow.js │ │ ├── node.js │ │ └── repo.js │ ├── node.js │ └── util.js │ ├── package.json │ └── yarn.lock └── tests ├── rest └── test.yaml └── unit ├── __init__.py ├── test_flow.py ├── test_message.py ├── test_node.py ├── test_port.py ├── test_repo.py └── test_run.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | .DS_* 92 | src/static/bower_components 93 | src/static/node_modules 94 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Gang Tao 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyFlow 2 | PyFlow is a python based flow programming engine/platform 3 | 4 | According to wikipedia, here is the definition of [FBP](http://www.jpaulmorrison.com/fbp/) 5 | 6 | > In computer programming, flow-based programming (FBP) is a programming paradigm that defines applications as networks of "black box" processes, which exchange data across predefined connections by message passing, where the connections are specified externally to the processes. These black box processes can be reconnected endlessly to form different applications without having to be changed internally. FBP is thus naturally component-oriented. 7 | 8 | There are lots of software or tools or product that enbrace the flow concept for example: 9 | - [Apache NiFi](https://nifi.apache.org/) 10 | - [DAG](https://data-flair.training/blogs/dag-in-apache-spark/) in spark 11 | - [AWS Step Functions](https://aws.amazon.com/step-functions/) 12 | - [Azure ML Studio](https://studio.azureml.net/) 13 | - [TensorBoard](https://www.tensorflow.org/get_started/graph_viz) from Tensorflow 14 | - [Scratch](http://scratched.gse.harvard.edu/) programing language 15 | - [argo](https://github.com/argoproj/argo) an open source container-native workflow engine 16 | 17 | I am building this tool to help you visualize your function building and execution procedures. 18 | 19 | 20 | # Concept 21 | Core concept of the PyFlow includs `Node`, `Port`, `Flow` and `Repository` 22 | 23 | ### Node and Port 24 | Node is the basic building block for flow. 25 | Each node contains: 26 | - a python function that represent the logic of this node 27 | - zero or more input port which defines the input of the function 28 | - 1 or more output port which represent the running result of the function 29 | 30 | A nodespec is the scheme or meta data of a node instance. you can create multiple node instance based on specific nodespec, that works like a nodespec is a class and a node instance is an object of that class. 31 | 32 | A nodespec contains following information: 33 | - title : the name of this node 34 | - ports : defines input and output port of this node 35 | - func : the python function of this node 36 | - id : unique identifier of the node specifiction 37 | 38 | here is a sample of nodespec for a function of add operation: 39 | ``` 40 | { 41 | "title": "add", 42 | "port": { 43 | "input": [{ 44 | "name": "a", 45 | "order": 0 46 | }, { 47 | "name": "b", 48 | "order": 1 49 | }] 50 | }, 51 | "func": "def func(a, b):\n return a + b\n", 52 | "id": "pyflow.transform.add" 53 | } 54 | ``` 55 | 56 | note, each port has a name and for input port, it has an order. so here `a` is the first input for add operation node and `b` is the second input. 57 | 58 | ### Flow 59 | A flow is a DAG that composed by nodes and links between nodes. 60 | A link can only link the output port of a node to an input of another node. 61 | 62 | A flow description contains: 63 | - id : unique id of this flow 64 | - name : display name of this flow 65 | - nodes : the node instance that compose this flow 66 | - links : the links that connect the ports of the nodes 67 | 68 | Here is a sample definition of a flow: 69 | ``` 70 | { 71 | "id": "pyflow.builder.gen", 72 | "name": "SampleFlow", 73 | "nodes": [{ 74 | "id": "node1515719064230", 75 | "spec_id": "pyflow.transform.add", 76 | "name": "add", 77 | "ports": [{ 78 | "name": "a", 79 | "value": "1" 80 | }, { 81 | "name": "b", 82 | "value": "2" 83 | }] 84 | }, { 85 | "id": "node1515719065769", 86 | "spec_id": "pyflow.transform.add", 87 | "name": "add", 88 | "ports": [{ 89 | "name": "b", 90 | "value": "3" 91 | }] 92 | "is_end": 1 93 | }], 94 | "links": [{ 95 | "source": "node1515719064230:out", 96 | "target": "node1515719065769:a" 97 | }] 98 | } 99 | ``` 100 | ![sample](https://github.com/gangtao/pyflow/raw/master/docs/sample_add.png) 101 | 102 | note, each input port can have input values in case there is no output port connected to it. and if the is_end is true which means this node is the last node to run for a flow. 103 | 104 | 105 | 106 | # Flow Engine 107 | A simple flow engine is implemented that will scan the the flow, create a stack orderred by the link of the node to make sure running node sequetially according to the dependency of the node. So the node will no depenency will run first and then the linked nodes to those node that run complete. 108 | This model is a very simple one and has some limitations. 109 | - call `exec(spec.get("func"))` to execute the python function is not secure, considering to support run the function in container 110 | - does not support parallel processing 111 | 112 | 113 | # Flow Rest API 114 | PyFlow server is a flask based REST server, refer to `tests/rest/test.yaml` for all the support Rest API 115 | 116 | # UI and Work Flow 117 | PyFlow Web UI leverages [jsplumb](https://jsplumbtoolkit.com/) to provide flow building and visulaizing functions. Through this Web UI, the user can: 118 | - create, edit, test, delete nodes 119 | - create, view, run flows 120 | - import/export the repository that contains all the definition of the flows and nodes 121 | 122 | # Build and Test 123 | ## Run locally 124 | install dependecny for front end code 125 | ``` 126 | cd /src/static 127 | yarn install 128 | ``` 129 | and start the server 130 | ``` 131 | cd /src 132 | python server.py 133 | ``` 134 | then open `http://localhost:5000` for the pyflow web UI 135 | 136 | ## Run with docker 137 | ``` 138 | docker run -P naughtytao/pyflow:latest 139 | ``` 140 | 141 | ## Docker Build and Run 142 | ``` 143 | cd /docker 144 | make docker 145 | ``` 146 | then run `docker run -P pyflow:latest` to run the pyflow in docker 147 | 148 | 149 | ## Unit Test 150 | ``` 151 | cd /tests/unit 152 | python -m unittest discover 153 | ``` 154 | 155 | ## Rest Test 156 | start the pyflow server and then run following command to test the rest API 157 | ``` 158 | cd /tests/rest 159 | resttest.py http://localhost:5000 test.yaml --verbose 160 | ``` 161 | 162 | ## Open Issues and Todo list 163 | 164 | - Now a static execution engine (run once per a node in a seperated process) is supported, no streaming support 165 | - Port type validations is not implemented 166 | - Support running each function as a docker instance instead of python eval which is much more secure and flexible 167 | - UI improvements, lots of things to do -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | ENV PYTHON_VERSION 2.7.13 4 | 5 | RUN apk add --update nodejs nodejs-npm 6 | 7 | RUN apk add --update \ 8 | python \ 9 | python-dev \ 10 | py-pip \ 11 | build-base \ 12 | nodejs \ 13 | nodejs-npm \ 14 | yarn \ 15 | git \ 16 | && rm -rf /var/cache/apk/* 17 | 18 | RUN cd /home \ 19 | && git clone https://github.com/gangtao/pyflow.git 20 | 21 | RUN cd /home/pyflow/src \ 22 | && pip install --target . flask 23 | 24 | RUN cd /home/pyflow/src/static \ 25 | && yarn install 26 | RUN cd /home/pyflow/src \ 27 | && pip install --target . flask 28 | 29 | COPY ./start.sh / 30 | RUN chmod +x /start.sh 31 | 32 | WORKDIR / 33 | EXPOSE 5000 34 | 35 | CMD ["/bin/sh","./start.sh"] 36 | -------------------------------------------------------------------------------- /docker/Makefile: -------------------------------------------------------------------------------- 1 | BIN_NAME ?= pyflow 2 | VERSION ?= 0.2.2 3 | IMAGE_NAME ?= $(BIN_NAME):$(VERSION) 4 | DOCKER_ID_USER ?= naughtytao 5 | 6 | docker: Dockerfile 7 | docker build --no-cache -t $(IMAGE_NAME) . 8 | 9 | push: 10 | docker tag $(IMAGE_NAME) ${DOCKER_ID_USER}/$(BIN_NAME):$(VERSION) 11 | docker tag $(IMAGE_NAME) ${DOCKER_ID_USER}/$(BIN_NAME):latest 12 | docker push ${DOCKER_ID_USER}/$(BIN_NAME):$(VERSION) 13 | docker push ${DOCKER_ID_USER}/$(BIN_NAME):latest -------------------------------------------------------------------------------- /docker/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd /home/pyflow/src 3 | #cp /usr/local/lib/python2.7/dist-packages/* . -r 4 | python server.py -------------------------------------------------------------------------------- /docs/sample_add.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/docs/sample_add.png -------------------------------------------------------------------------------- /repo/repo.json: -------------------------------------------------------------------------------- 1 | { 2 | "flow": { 3 | "pyflow.sample.error": { 4 | "nodes": [ 5 | { 6 | "spec_id": "pyflow.transform.append", 7 | "ui": { 8 | "y": "48px", 9 | "x": "77px" 10 | }, 11 | "ports": [ 12 | { 13 | "name": "a", 14 | "value": "ab" 15 | }, 16 | { 17 | "name": "b", 18 | "value": "cd" 19 | } 20 | ], 21 | "name": "Append", 22 | "id": "node1527137681280" 23 | }, 24 | { 25 | "name": "Append", 26 | "id": "node1527137684469", 27 | "ui": { 28 | "y": "206px", 29 | "x": "553px" 30 | }, 31 | "is_end": 1, 32 | "spec_id": "pyflow.transform.append", 33 | "ports": [ 34 | { 35 | "name": "b", 36 | "value": "ef" 37 | } 38 | ] 39 | }, 40 | { 41 | "spec_id": "pyflow.transform.error", 42 | "ui": { 43 | "y": "118px", 44 | "x": "312px" 45 | }, 46 | "ports": [], 47 | "name": "Error", 48 | "id": "node1527137688270" 49 | } 50 | ], 51 | "id": "pyflow.sample.error", 52 | "links": [ 53 | { 54 | "source": "node1527137681280:out", 55 | "target": "node1527137688270:a" 56 | }, 57 | { 58 | "source": "node1527137688270:out", 59 | "target": "node1527137684469:a" 60 | } 61 | ], 62 | "name": "Error" 63 | }, 64 | "pyflow.sample.xor": { 65 | "nodes": [ 66 | { 67 | "spec_id": "pyflow.logic.self", 68 | "ui": { 69 | "y": "70px", 70 | "x": "37px" 71 | }, 72 | "ports": [ 73 | { 74 | "name": "a", 75 | "value": "true" 76 | } 77 | ], 78 | "name": "Self", 79 | "id": "node1527108601395" 80 | }, 81 | { 82 | "spec_id": "pyflow.logic.self", 83 | "ui": { 84 | "y": "184px", 85 | "x": "40px" 86 | }, 87 | "ports": [ 88 | { 89 | "name": "a", 90 | "value": "true" 91 | } 92 | ], 93 | "name": "Self", 94 | "id": "node1527108602911" 95 | }, 96 | { 97 | "spec_id": "pyflow.logic.not", 98 | "ui": { 99 | "y": "9px", 100 | "x": "270px" 101 | }, 102 | "ports": [], 103 | "name": "Not", 104 | "id": "node1527108607514" 105 | }, 106 | { 107 | "spec_id": "pyflow.logic.not", 108 | "ui": { 109 | "y": "280px", 110 | "x": "274px" 111 | }, 112 | "ports": [], 113 | "name": "Not", 114 | "id": "node1527108611102" 115 | }, 116 | { 117 | "spec_id": "pyflow.logic.and", 118 | "ui": { 119 | "y": "119px", 120 | "x": "511px" 121 | }, 122 | "ports": [], 123 | "name": "And", 124 | "id": "node1527108634951" 125 | }, 126 | { 127 | "spec_id": "pyflow.logic.and", 128 | "ui": { 129 | "y": "218px", 130 | "x": "537px" 131 | }, 132 | "ports": [], 133 | "name": "And", 134 | "id": "node1527108637547" 135 | }, 136 | { 137 | "name": "Or", 138 | "id": "node1527108752672", 139 | "ui": { 140 | "y": "176px", 141 | "x": "771px" 142 | }, 143 | "is_end": 1, 144 | "spec_id": "pyflow.logic.or", 145 | "ports": [] 146 | } 147 | ], 148 | "id": "pyflow.sample.xor", 149 | "links": [ 150 | { 151 | "source": "node1527108601395:out", 152 | "target": "node1527108607514:a" 153 | }, 154 | { 155 | "source": "node1527108602911:out", 156 | "target": "node1527108611102:a" 157 | }, 158 | { 159 | "source": "node1527108607514:out", 160 | "target": "node1527108634951:b" 161 | }, 162 | { 163 | "source": "node1527108611102:out", 164 | "target": "node1527108637547:b" 165 | }, 166 | { 167 | "source": "node1527108601395:out", 168 | "target": "node1527108637547:a" 169 | }, 170 | { 171 | "source": "node1527108602911:out", 172 | "target": "node1527108634951:a" 173 | }, 174 | { 175 | "source": "node1527108634951:out", 176 | "target": "node1527108752672:a" 177 | }, 178 | { 179 | "source": "node1527108637547:out", 180 | "target": "node1527108752672:b" 181 | } 182 | ], 183 | "name": "Xor" 184 | }, 185 | "pyflow.sample.xor1": { 186 | "nodes": [ 187 | { 188 | "spec_id": "pyflow.logic.selfandnot", 189 | "ui": { 190 | "y": "81px", 191 | "x": "121px" 192 | }, 193 | "ports": [ 194 | { 195 | "name": "a", 196 | "value": "true" 197 | } 198 | ], 199 | "name": "Self and Not", 200 | "id": "node1527448342301" 201 | }, 202 | { 203 | "spec_id": "pyflow.logic.selfandnot", 204 | "ui": { 205 | "y": "200px", 206 | "x": "53px" 207 | }, 208 | "ports": [ 209 | { 210 | "name": "a", 211 | "value": "false" 212 | } 213 | ], 214 | "name": "Self and Not", 215 | "id": "node1527448358756" 216 | }, 217 | { 218 | "spec_id": "pyflow.logic.and", 219 | "ui": { 220 | "y": "10px", 221 | "x": "426px" 222 | }, 223 | "ports": [], 224 | "name": "And", 225 | "id": "node1527448364884" 226 | }, 227 | { 228 | "spec_id": "pyflow.logic.and", 229 | "ui": { 230 | "y": "236px", 231 | "x": "478px" 232 | }, 233 | "ports": [], 234 | "name": "And", 235 | "id": "node1527448366929" 236 | }, 237 | { 238 | "name": "Or", 239 | "id": "node1527448391328", 240 | "ui": { 241 | "y": "112px", 242 | "x": "744px" 243 | }, 244 | "is_end": 1, 245 | "spec_id": "pyflow.logic.or", 246 | "ports": [] 247 | } 248 | ], 249 | "id": "pyflow.sample.xor1", 250 | "links": [ 251 | { 252 | "source": "node1527448342301:notself", 253 | "target": "node1527448364884:b" 254 | }, 255 | { 256 | "source": "node1527448342301:self", 257 | "target": "node1527448366929:a" 258 | }, 259 | { 260 | "source": "node1527448358756:self", 261 | "target": "node1527448364884:a" 262 | }, 263 | { 264 | "source": "node1527448358756:notself", 265 | "target": "node1527448366929:b" 266 | }, 267 | { 268 | "source": "node1527448364884:out", 269 | "target": "node1527448391328:a" 270 | }, 271 | { 272 | "source": "node1527448366929:out", 273 | "target": "node1527448391328:b" 274 | } 275 | ], 276 | "name": "Xor1" 277 | }, 278 | "pyflow.sample.math": { 279 | "nodes": [ 280 | { 281 | "spec_id": "pyflow.math.add", 282 | "ui": { 283 | "y": "69px", 284 | "x": "160px" 285 | }, 286 | "id": "node1527101193950", 287 | "name": "Add", 288 | "ports": [ 289 | { 290 | "name": "a", 291 | "value": "2" 292 | }, 293 | { 294 | "name": "b", 295 | "value": "3" 296 | } 297 | ] 298 | }, 299 | { 300 | "spec_id": "pyflow.math.add", 301 | "ui": { 302 | "y": "213px", 303 | "x": "163px" 304 | }, 305 | "id": "node1527101195573", 306 | "name": "Add", 307 | "ports": [ 308 | { 309 | "name": "a", 310 | "value": "4" 311 | }, 312 | { 313 | "name": "b", 314 | "value": "5" 315 | } 316 | ] 317 | }, 318 | { 319 | "spec_id": "pyflow.math.multiply", 320 | "ui": { 321 | "y": "160px", 322 | "x": "450px" 323 | }, 324 | "id": "node1527101199970", 325 | "name": "Multiply", 326 | "ports": [] 327 | }, 328 | { 329 | "name": "Division", 330 | "id": "node1527101204754", 331 | "ui": { 332 | "y": "169px", 333 | "x": "721px" 334 | }, 335 | "is_end": 1, 336 | "spec_id": "pyflow.math.division", 337 | "ports": [ 338 | { 339 | "name": "b", 340 | "value": "5" 341 | } 342 | ] 343 | } 344 | ], 345 | "id": "pyflow.sample.math", 346 | "links": [ 347 | { 348 | "source": "node1527101193950:out", 349 | "target": "node1527101199970:a" 350 | }, 351 | { 352 | "source": "node1527101195573:out", 353 | "target": "node1527101199970:b" 354 | }, 355 | { 356 | "source": "node1527101199970:out", 357 | "target": "node1527101204754:a" 358 | } 359 | ], 360 | "name": "Math" 361 | } 362 | }, 363 | "nodespec": { 364 | "pyflow.transform.append": { 365 | "port": { 366 | "input": [ 367 | { 368 | "type": "String", 369 | "name": "a", 370 | "order": 0 371 | }, 372 | { 373 | "type": "String", 374 | "name": "b", 375 | "order": 1 376 | } 377 | ], 378 | "output": [ 379 | { 380 | "type": "String", 381 | "name": "out" 382 | } 383 | ] 384 | }, 385 | "id": "pyflow.transform.append", 386 | "func": "def func(a,b):\n\t\"\"\"\n :params: a,b\n :ptypes: String,String\n :returns: out\n :rtype: String\n \"\"\"\n\treturn a + b", 387 | "title": "Append" 388 | }, 389 | "pyflow.transform.error": { 390 | "port": { 391 | "input": [ 392 | { 393 | "type": "String", 394 | "name": "a", 395 | "order": 0 396 | } 397 | ], 398 | "output": [ 399 | { 400 | "type": "String", 401 | "name": "out" 402 | } 403 | ] 404 | }, 405 | "id": "pyflow.transform.error", 406 | "func": "def func(a):\n\t\"\"\"\n :params: a\n :ptypes: String\n :returns: out\n :rtype: String\n \"\"\"\n\traise Exception(\"Always Failed!\")", 407 | "title": "Error" 408 | }, 409 | "pyflow.math.division": { 410 | "id": "pyflow.math.division", 411 | "port": { 412 | "input": [ 413 | { 414 | "type": "Int", 415 | "name": "a", 416 | "order": 0 417 | }, 418 | { 419 | "type": "Int", 420 | "name": "b", 421 | "order": 1 422 | } 423 | ], 424 | "output": [ 425 | { 426 | "type": "Int", 427 | "name": "out" 428 | } 429 | ] 430 | }, 431 | "func": "def func(a,b):\n\t\"\"\"\n :params: a,b\n :ptypes: Int,Int\n :returns: out\n :rtype: Int\n \"\"\"\n\treturn a / b", 432 | "title": "Division" 433 | }, 434 | "pyflow.logic.and": { 435 | "id": "pyflow.logic.and", 436 | "port": { 437 | "input": [ 438 | { 439 | "type": "Boolean", 440 | "name": "a", 441 | "order": 0 442 | }, 443 | { 444 | "type": "Boolean", 445 | "name": "b", 446 | "order": 1 447 | } 448 | ], 449 | "output": [ 450 | { 451 | "type": "Boolean", 452 | "name": "out" 453 | } 454 | ] 455 | }, 456 | "func": "def func(a,b):\n\t\"\"\"\n :params: a,b\n :ptypes: Boolean,Boolean\n :returns: out\n :rtype: Boolean\n \"\"\"\n\treturn a and b", 457 | "title": "And" 458 | }, 459 | "pyflow.logic.self": { 460 | "id": "pyflow.logic.self", 461 | "port": { 462 | "input": [ 463 | { 464 | "type": "Boolean", 465 | "name": "a", 466 | "order": 0 467 | } 468 | ], 469 | "output": [ 470 | { 471 | "type": "Boolean", 472 | "name": "out" 473 | } 474 | ] 475 | }, 476 | "func": "def func(a):\n\t\"\"\"\n :params: a\n :ptypes: Boolean\n :returns: out\n :rtype: Boolean\n \"\"\"\n\treturn bool(a)", 477 | "title": "Self" 478 | }, 479 | "pyflow.math.multiply": { 480 | "id": "pyflow.math.multiply", 481 | "port": { 482 | "input": [ 483 | { 484 | "type": "Int", 485 | "name": "a", 486 | "order": 0 487 | }, 488 | { 489 | "type": "Int", 490 | "name": "b", 491 | "order": 1 492 | } 493 | ], 494 | "output": [ 495 | { 496 | "type": "Int", 497 | "name": "out" 498 | } 499 | ] 500 | }, 501 | "func": "def func(a,b):\n\t\"\"\"\n :params: a,b\n :ptypes: Int,Int\n :returns: out\n :rtype: Int\n \"\"\"\n\treturn a * b", 502 | "title": "Multiply" 503 | }, 504 | "pyflow.math.minus": { 505 | "id": "pyflow.math.minus", 506 | "port": { 507 | "input": [ 508 | { 509 | "type": "Int", 510 | "name": "a", 511 | "order": 0 512 | }, 513 | { 514 | "type": "Int", 515 | "name": "b", 516 | "order": 1 517 | } 518 | ], 519 | "output": [ 520 | { 521 | "type": "Int", 522 | "name": "out" 523 | } 524 | ] 525 | }, 526 | "func": "def func(a,b):\n\t\"\"\"\n :params: a,b\n :ptypes: Int,Int\n :returns: out\n :rtype: Int\n \"\"\"\n\treturn a - b", 527 | "title": "Minus" 528 | }, 529 | "pyflow.logic.or": { 530 | "id": "pyflow.logic.or", 531 | "port": { 532 | "input": [ 533 | { 534 | "type": "Boolean", 535 | "name": "a", 536 | "order": 0 537 | }, 538 | { 539 | "type": "Boolean", 540 | "name": "b", 541 | "order": 1 542 | } 543 | ], 544 | "output": [ 545 | { 546 | "type": "Boolean", 547 | "name": "out" 548 | } 549 | ] 550 | }, 551 | "func": "def func(a,b):\n\t\"\"\"\n :params: a,b\n :ptypes: Boolean,Boolean\n :returns: out\n :rtype: Boolean\n \"\"\"\n\treturn a or b", 552 | "title": "Or" 553 | }, 554 | "pyflow.math.add": { 555 | "id": "pyflow.math.add", 556 | "port": { 557 | "input": [ 558 | { 559 | "type": "Int", 560 | "name": "a", 561 | "order": 0 562 | }, 563 | { 564 | "type": "Int", 565 | "name": "b", 566 | "order": 1 567 | } 568 | ], 569 | "output": [ 570 | { 571 | "type": "Int", 572 | "name": "out" 573 | } 574 | ] 575 | }, 576 | "func": "def func(a,b):\n\t\"\"\"\n :params: a,b\n :ptypes: Int,Int\n :returns: out\n :rtype: Int\n \"\"\"\n\treturn a + b", 577 | "title": "Add" 578 | }, 579 | "pyflow.logic.not": { 580 | "id": "pyflow.logic.not", 581 | "port": { 582 | "input": [ 583 | { 584 | "type": "Boolean", 585 | "name": "a", 586 | "order": 0 587 | } 588 | ], 589 | "output": [ 590 | { 591 | "type": "Boolean", 592 | "name": "out" 593 | } 594 | ] 595 | }, 596 | "func": "def func(a):\n\t\"\"\"\n :params: a\n :ptypes: Boolean\n :returns: out\n :rtype: Boolean\n \"\"\"\n\treturn not a", 597 | "title": "Not" 598 | }, 599 | "pyflow.logic.selfandnot": { 600 | "port": { 601 | "input": [ 602 | { 603 | "type": "Boolean", 604 | "name": "a", 605 | "order": 0 606 | } 607 | ], 608 | "output": [ 609 | { 610 | "type": "Boolean", 611 | "name": "self" 612 | }, 613 | { 614 | "type": "Boolean", 615 | "name": "notself" 616 | } 617 | ] 618 | }, 619 | "id": "pyflow.logic.selfandnot", 620 | "func": "def func(a):\n\t\"\"\"\n :params: a\n :ptypes: Boolean\n :returns: self, notself\n :rtype: Boolean, Boolean\n \"\"\"\n\tresult = dict()\n\tresult[\"self\"] = a\n\tresult[\"notself\"] = not a\n\treturn result", 621 | "title": "Self and Not" 622 | } 623 | } 624 | } -------------------------------------------------------------------------------- /src/fbp/__init__.py: -------------------------------------------------------------------------------- 1 | """Flow library""" 2 | import json 3 | import time 4 | 5 | from node import Node 6 | from flow import Flow 7 | from repository import repository 8 | 9 | 10 | __version_info__ = (0, 0, 1) 11 | __version__ = ".".join(map(str, __version_info__)) 12 | 13 | 14 | def create_node(spec_id, id, name): 15 | spec = repository().get("nodespec", spec_id) 16 | 17 | if spec is None: 18 | raise Exception("No such node specification {}".format(spec_id)) 19 | 20 | if type(spec) is not dict: 21 | try: 22 | spec_obj = json.loads(spec, strict=False) 23 | except Exception as e: 24 | raise Exception("Invalid node specification {}".format(spec)) 25 | 26 | anode = Node(id, name, spec_obj) 27 | return anode 28 | 29 | anode = Node(id, name, spec) 30 | return anode 31 | 32 | # Run a flow based on a defined specification of flow 33 | # Todo consider unify the flow definition spec and running spec 34 | 35 | 36 | def _run_flow(flow_spec): 37 | flow_spec_obj = None 38 | 39 | if type(flow_spec) is not dict: 40 | try: 41 | flow_spec_obj = json.loads(flow_spec, strict=False) 42 | except Exception as e: 43 | # print "invalid flow specification format" 44 | raise e 45 | else: 46 | flow_spec_obj = flow_spec 47 | 48 | aflow = Flow(flow_spec_obj.get("id"), flow_spec_obj.get("name")) 49 | 50 | for node_def in flow_spec_obj.get("nodes"): 51 | anode = create_node(node_def.get("spec_id"), 52 | node_def.get("id"), node_def.get("name")) 53 | aflow.add_node(anode) 54 | if "is_end" in node_def.keys() and node_def.get("is_end") == 1: 55 | end_node = anode 56 | for port_def in node_def.get("ports"): 57 | anode.set_inport_value(port_def.get("name"), port_def.get("value")) 58 | 59 | for link_def in flow_spec_obj.get("links"): 60 | source = link_def.get("source").split(":") 61 | target = link_def.get("target").split(":") 62 | 63 | aflow.link(source[0], source[1], target[0], target[1]) 64 | 65 | stats = aflow.run(end_node) 66 | 67 | return stats 68 | 69 | 70 | def run_flow(flow_spec): 71 | stats = _run_flow(flow_spec) 72 | # TODO : support run in async mode 73 | while not stats.check_stat(): 74 | time.sleep(0.1) 75 | return [i for i in stats.result()] 76 | -------------------------------------------------------------------------------- /src/fbp/flow.py: -------------------------------------------------------------------------------- 1 | """Core Class for Flow.""" 2 | from multiprocessing import Process, Manager 3 | from multiprocessing.managers import BaseManager 4 | import sys 5 | import json 6 | from node import Node 7 | 8 | 9 | EXEC_MODE_BATCH = "batch" 10 | EXEC_MODE_STREAMING = "streaming" 11 | 12 | 13 | class Path(object): 14 | def __init__(self, source_node, source_port, target_node, target_port): 15 | self._name = source_node.id + ":" + source_port.name + \ 16 | "~" + target_node.id + ":" + target_port.name 17 | self._source_node = source_node 18 | self._target_node = target_node 19 | self._source_port = source_port 20 | self._target_port = target_port 21 | 22 | @property 23 | def name(self): 24 | return self._name 25 | 26 | @property 27 | def source_node(self): 28 | return self._source_node 29 | 30 | @property 31 | def source_port(self): 32 | return self._source_port 33 | 34 | @property 35 | def target_node(self): 36 | return self._target_node 37 | 38 | @property 39 | def target_port(self): 40 | return self._target_port 41 | 42 | 43 | def _gen_lable(node, port): 44 | return node.id + ":" + port.name 45 | 46 | 47 | class FlowStates(object): 48 | def __init__(self): 49 | self._result = list() 50 | self._complete = False 51 | 52 | def check_stat(self): 53 | return self._complete 54 | 55 | def result(self): 56 | return self._result 57 | 58 | def append_stat(self, node): 59 | self._result.append(node) 60 | 61 | def set_stat(self, is_complete): 62 | self._complete = is_complete 63 | 64 | def get_result_by_id(self, id): 65 | for r in self._result: 66 | if r["id"] == id: 67 | return r 68 | return None 69 | 70 | 71 | class Flow(object): 72 | 73 | def __init__(self, id, name): 74 | self._name = name 75 | self._id = id 76 | self._nodes = dict() 77 | self._links = dict() 78 | self._mode = EXEC_MODE_BATCH 79 | 80 | def add_node(self, node): 81 | self._nodes[node.id] = node 82 | 83 | def get_node(self, id): 84 | return self._node.get(id) 85 | 86 | def get_nodes(self): 87 | return self._nodes.values() 88 | 89 | def remove_node(self, node_id): 90 | if self._nodes.get(node_id) is not None: 91 | del self._nodes[node_id] 92 | 93 | def link(self, source_node_id, source_port_name, target_node_id, target_port_name): 94 | 95 | # TODO : link should do data transfer if source port contains data 96 | if self._nodes.get(source_node_id) is None: 97 | raise Exception( 98 | "The source node {} is not in the flow".format(source_node_id)) 99 | 100 | if self._nodes.get(target_node_id) is None: 101 | raise Exception( 102 | "The target node {} is not in the flow".format(target_node_id)) 103 | 104 | source_node = self._nodes.get(source_node_id) 105 | target_node = self._nodes.get(target_node_id) 106 | source_port = source_node.get_port(source_port_name, "out") 107 | target_port = target_node.get_port(target_port_name, "in") 108 | 109 | if source_port is None: 110 | raise Exception("The source port {} is not in the node {}".format( 111 | source_port_name, source_node_id)) 112 | 113 | if target_port is None: 114 | raise Exception("The target port {} is not in the node {}".format( 115 | target_port_name, target_node_id)) 116 | 117 | # source lable is not used 118 | source_label = _gen_lable(source_node, source_port) 119 | target_label = _gen_lable(target_node, target_port) 120 | 121 | link_to_target = self._links.get(target_label) 122 | if link_to_target is not None: 123 | raise Exception( 124 | "Link to target port {} already exist, unlink first!".format(target_label)) 125 | 126 | # bi-directional link the port 127 | target_port.point_from(source_port) 128 | source_port.point_to(target_port) 129 | 130 | self._links[target_label] = Path( 131 | source_node, source_port, target_node, target_port) 132 | 133 | def unlink(self, target_node_id, target_port_name): 134 | target_label = target_node_id + ":" + target_port_name 135 | link_to_target = self._links.get(target_label) 136 | if link_to_target is not None: 137 | link_to_target.source_port.un_point_to(link_to_target.target_port) 138 | link_to_target.target_port.point_from(None) 139 | del self._links[target] 140 | 141 | def get_links(self): 142 | return self._links 143 | 144 | def _find_dependant_nodes(self, target_node, source_nodes): 145 | in_ports = target_node.get_ports("in") 146 | children = [] 147 | for p in in_ports: 148 | link_to_p = self._links.get(_gen_lable(target_node, p)) 149 | if link_to_p is not None : 150 | children.append(link_to_p.source_node) 151 | if link_to_p.source_node in source_nodes: 152 | source_nodes.remove(link_to_p.source_node) 153 | source_nodes.append(link_to_p.source_node) 154 | return children 155 | 156 | def _find_source_nodes(self, target_node, source_nodes): 157 | # TODO : Add loop check 158 | children = [target_node] 159 | new_children = [] 160 | while True: 161 | for child in children: 162 | new_children += self._find_dependant_nodes(child, source_nodes) 163 | 164 | if len(new_children) == 0: 165 | break 166 | children = new_children 167 | new_children = [] 168 | 169 | def _run_batch(self, end_node, stat): 170 | nodemap = [end_node] 171 | self._find_source_nodes(end_node, nodemap) 172 | while True: 173 | if len(nodemap) == 0: 174 | break 175 | anode = nodemap.pop() 176 | node_value = anode.get_node_value() 177 | 178 | dep_nodes = list() 179 | find_failure = False 180 | for n in self._find_dependant_nodes(anode, dep_nodes): 181 | if n._status == "fail" or n._status == "skip": 182 | node_value["status"] = "skip" 183 | node_value["error"] = "skip due to denpendency node failure" 184 | stat.append_stat(node_value) 185 | find_failure = True 186 | break 187 | 188 | if find_failure: 189 | # break incase there is depedency failure 190 | break 191 | 192 | try: 193 | anode.run() 194 | node_value = anode.get_node_value() 195 | except Exception as e: 196 | node_value = anode.get_node_value() 197 | node_value["status"] = "fail" 198 | node_value["error"] = str(e) 199 | finally : 200 | stat.append_stat(node_value) 201 | 202 | stat.set_stat(True) 203 | 204 | def _run_streaming(self, end_node): 205 | pass 206 | 207 | def run(self, end_node): 208 | if self._mode == EXEC_MODE_BATCH: 209 | BaseManager.register('FlowStates', FlowStates) 210 | BaseManager.register('Node', Node) 211 | manager = BaseManager() 212 | manager.start() 213 | stat = manager.FlowStates() 214 | 215 | p = Process(target=self._run_batch, args=(end_node, stat)) 216 | p.start() 217 | return stat 218 | elif self._mode == EXEC_MODE_STREAMING: 219 | self._run_streaming(end_node) 220 | -------------------------------------------------------------------------------- /src/fbp/node.py: -------------------------------------------------------------------------------- 1 | """Node Class for Flow.""" 2 | 3 | import traceback, pdb 4 | 5 | from port import Inport, Outport 6 | 7 | OUTPORT_DEFAULT_NAME = "out" 8 | 9 | STATUS_SUCCESS = "success" 10 | STATUS_FAIL = "fail" 11 | STATUS_INIT = "init" 12 | STATUS_RUNNING = "running" 13 | 14 | 15 | class Node(object): 16 | 17 | def __init__(self, id, name, spec): 18 | self._id = id 19 | self._name = name 20 | self._spec = spec 21 | self._port_spec = spec.get("port") 22 | # Tricky and Non-Secure eval 23 | exec(spec.get("func")) 24 | self._func = func 25 | 26 | self._inputports = dict() 27 | self._outputports = dict() 28 | self._initports() 29 | self._is_cache_valid = False 30 | self._status = STATUS_INIT 31 | self._error = None 32 | 33 | @property 34 | def name(self): 35 | return self._name 36 | 37 | @property 38 | def id(self): 39 | return self._id 40 | 41 | def _initports(self): 42 | 43 | def _parse_in_port_port_spec(port_port_spec): 44 | name = port_port_spec.get("name") 45 | 46 | ptype = port_port_spec.get("type") 47 | if ptype is None: 48 | ptype = "String" 49 | 50 | default = port_port_spec.get("default") 51 | 52 | required = port_port_spec.get("required") 53 | if required is None: 54 | required = False 55 | else: 56 | required = required.lower() in ('1', 'true', 'yes', 'y') 57 | 58 | try: 59 | order = int(port_port_spec.get("order")) 60 | except Exception as e: 61 | order = 0 62 | 63 | return [name, ptype, default, required, order] 64 | 65 | def _parse_out_port_port_spec(port_port_spec): 66 | name = port_port_spec.get("name") 67 | 68 | ptype = port_port_spec.get("type") 69 | if ptype is None: 70 | ptype = "String" 71 | 72 | return [name, ptype] 73 | 74 | input_ports = self._port_spec.get("input") 75 | if input_ports: 76 | for p in input_ports: 77 | port_info = _parse_in_port_port_spec(p) 78 | in_port = Inport(port_info[0], port_info[1], port_info[ 79 | 2], port_info[3], port_info[4]) 80 | self._inputports[in_port.name] = in_port 81 | 82 | output_ports = self._port_spec.get("output") 83 | 84 | if output_ports is None: 85 | out_port = Outport(OUTPORT_DEFAULT_NAME) 86 | self._outputports[out_port.name] = out_port 87 | else: 88 | for p in output_ports: 89 | port_info = _parse_out_port_port_spec(p) 90 | out_port = Outport(port_info[0], port_info[1]) 91 | self._outputports[out_port.name] = out_port 92 | 93 | def __str__(self): 94 | out_str = "node id : {}\nnode name : {}".format(self._id, self._name) 95 | 96 | for k, v in self._inputports.items(): 97 | out_str = out_str + "\n" + str(v) 98 | 99 | for k, v in self._outputports.items(): 100 | out_str = out_str + "\n" + str(v) 101 | 102 | out_str = out_str + "\n" + "status : {}".format(self._status) 103 | out_str = out_str + "\n" + "error : {}".format(self._error) 104 | 105 | return out_str 106 | 107 | def set_inport_value(self, port_name, value): 108 | inport = self._inputports.get(port_name) 109 | if inport is None: 110 | raise Exception("No such port {} in current Node {}".format( 111 | port_name, self.id)) # TODO Add Exception definition for Flow 112 | 113 | inport.value = value # TODO cache later 114 | self._is_cache_valid = False 115 | 116 | def get_port(self, port_name, port_type): 117 | if port_type == "in": 118 | return self._inputports.get(port_name) 119 | elif port_type == "out": 120 | return self._outputports.get(port_name) 121 | else: 122 | raise Exception("Invalid port type {}".format(port_type)) 123 | 124 | def get_ports(self, port_type): 125 | if port_type == "in": 126 | return [v for k, v in self._inputports.items()] 127 | elif port_type == "out": 128 | return [v for k, v in self._outputports.items()] 129 | else: 130 | raise Exception("Invalid port type {}".format(port_type)) 131 | 132 | def get_inport_value(self, port_name): 133 | inport = self._inputports.get(port_name) 134 | 135 | if inport is None: 136 | raise Exception("No such port {} in current Node {}".format( 137 | port_name, self.id)) # TODO Add Exception definition for Flow 138 | 139 | return inport.value 140 | 141 | def get_outport_value(self, port_name=OUTPORT_DEFAULT_NAME): 142 | outport = self._outputports.get(port_name) 143 | 144 | if outport is None: 145 | raise Exception("No such port {} in current Node {}".format( 146 | port_name, self.id)) # TODO Add Exception definition for Flow 147 | 148 | return outport.value 149 | 150 | def get_node_value(self): 151 | node = dict() 152 | node["id"] = self._id 153 | node["name"] = self._name 154 | node["inputs"] = [v.get_value() for k, v in self._inputports.items()] 155 | node["outputs"] = [v.get_value() for k, v in self._outputports.items()] 156 | node["status"] = self._status 157 | 158 | node["error"] = str(self._error) 159 | return node 160 | 161 | def run(self): 162 | def _function_wrapper(func, args): 163 | return func(*args) 164 | 165 | self._status = STATUS_RUNNING 166 | 167 | if self._is_cache_valid: 168 | # Cache Hit 169 | self._status = STATUS_SUCCESS 170 | self._error = None 171 | return 172 | 173 | parameter_values = [(v.value, v.order) 174 | for k, v in self._inputports.items()] 175 | 176 | parameter_values = [v[0] for v in sorted( 177 | parameter_values, key=lambda x: x[1])] # sort by order 178 | 179 | try: 180 | return_value = _function_wrapper(self._func, parameter_values) 181 | 182 | self._is_cache_valid = True 183 | self._status = STATUS_SUCCESS 184 | self._error = None 185 | 186 | # Single output case 187 | if OUTPORT_DEFAULT_NAME in self._outputports.keys() and len(self._outputports) == 1: 188 | out_port = self._outputports.get(OUTPORT_DEFAULT_NAME) 189 | out_port.value = return_value 190 | return 191 | 192 | # Mutiple output case 193 | # the multiple output should return a dict where key/value is output name/value 194 | for k, v in self._outputports.items(): 195 | v.value = return_value.get(k) 196 | except Exception as e: 197 | self._status = STATUS_FAIL 198 | self._error = e 199 | print(traceback.format_exc()) 200 | -------------------------------------------------------------------------------- /src/fbp/port.py: -------------------------------------------------------------------------------- 1 | """Port Class for Flow.""" 2 | 3 | import types 4 | import json 5 | 6 | # All Supported Types 7 | TYPES = dict() 8 | TYPES["Boolean"] = types.BooleanType 9 | TYPES["Int"] = types.IntType 10 | TYPES["Long"] = types.LongType 11 | TYPES["Float"] = types.FloatType 12 | TYPES["String"] = types.StringType 13 | TYPES["List"] = types.ListType 14 | TYPES["Json"] = types.DictType 15 | 16 | 17 | def c_int(val): 18 | return int(val) 19 | 20 | 21 | def c_bool(val): 22 | if type(val) is 'bool': 23 | return val 24 | elif type(val) is str or unicode: 25 | return str(val).lower() in ["y", "true", "yes"] 26 | else: 27 | return False 28 | 29 | 30 | def c_long(val): 31 | return long(val) 32 | 33 | 34 | def c_float(val): 35 | return float(val) 36 | 37 | 38 | def c_str(val): 39 | return str(val) 40 | 41 | 42 | def c_list(val): 43 | if type(val) is list: 44 | return val 45 | elif type(val) is str or unicode: 46 | return str(val).split(",") 47 | return [] 48 | 49 | 50 | def c_json(val): 51 | if type(val) is dict: 52 | return val 53 | elif type(val) is str or unicode: 54 | return json.loads(str(val)) 55 | return {} 56 | 57 | 58 | def type_conversion(value, type): 59 | if type == "Boolean": 60 | return c_bool(value) 61 | elif type == "Int": 62 | return c_int(value) 63 | elif type == "Long": 64 | return c_long(value) 65 | elif type == "Float": 66 | return c_float(value) 67 | elif type == "String": 68 | return c_str(value) 69 | elif type == "List": 70 | return c_list(value) 71 | elif type == "Json": 72 | return c_json(value) 73 | else: 74 | return None 75 | 76 | 77 | class Port(object): 78 | def __init__(self, name, type='String'): 79 | self._name = name 80 | self._type = type 81 | if type in TYPES.keys(): 82 | self._type_object = TYPES[type] 83 | else: 84 | print("Port type {} is not supported! default to string".format(type)) 85 | self._type_object = 'String' 86 | self._value = None 87 | 88 | @classmethod 89 | def support_types(cls): 90 | return TYPES.keys() 91 | 92 | @property 93 | def name(self): 94 | return self._name 95 | 96 | @property 97 | def type(self): 98 | return self._type 99 | 100 | @property 101 | def type_object(self): 102 | return self._type_object 103 | 104 | @property 105 | def value(self): 106 | # convert the value to type 107 | return type_conversion(self._value, self._type) 108 | 109 | @value.setter 110 | def value(self, value): 111 | self._value = value 112 | 113 | def __str__(self): 114 | format_str = "Port\nname : {}\nvalue : {}\ntype : {}\n" 115 | return format_str.format(self.name, str(self.value), self.type) 116 | 117 | def get_value(self): 118 | port = dict() 119 | port["name"] = self._name 120 | port["value"] = self._value 121 | port["type"] = self._type 122 | return port 123 | 124 | def valid(self, value): 125 | # TODO Type check 126 | pass 127 | 128 | def clone(self, include_value=False): 129 | # TODO implement clone of port 130 | pass 131 | 132 | 133 | class Inport(Port): 134 | 135 | def __init__(self, name, type='String', default=None, required=False, order=0): 136 | Port.__init__(self, name, type) 137 | self._default = default 138 | self._required = required 139 | self._order = order 140 | self._point_from = None 141 | 142 | @property 143 | def default(self): 144 | return self._default 145 | 146 | @property 147 | def is_required(self): 148 | return self._required 149 | 150 | @property 151 | def order(self): 152 | return self._order 153 | 154 | @property 155 | def value(self): 156 | if self._value is not None: 157 | # return converted value according to the type def 158 | return type_conversion(self._value, self._type) 159 | else: 160 | return self._default 161 | 162 | @value.setter 163 | def value(self, value): 164 | self._value = value 165 | 166 | def point_from(self, port): 167 | self._point_from = port 168 | 169 | def __str__(self): 170 | format_str = "In" + \ 171 | Port.__str__(self) + "default : {}\required : {}\norder : {}\n" 172 | return format_str.format(self.default, self.is_required, self.order) 173 | 174 | 175 | class Outport(Port): 176 | def __init__(self, name, type='String'): 177 | Port.__init__(self, name, type) 178 | self._point_to = [] 179 | 180 | @property 181 | def value(self): 182 | # return converted value according to the type def 183 | return type_conversion(self._value, self._type) 184 | 185 | @value.setter 186 | def value(self, value): 187 | self._value = value 188 | for p in self._point_to: 189 | p.value = value 190 | 191 | def point_to(self, port): 192 | self._point_to.append(port) 193 | 194 | def un_point_to(self, port): 195 | self._point_to.remove(port) 196 | 197 | def __str__(self): 198 | format_str = "Out" + Port.__str__(self) 199 | return format_str 200 | -------------------------------------------------------------------------------- /src/fbp/repository.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import json 3 | 4 | 5 | def singleton(class_): 6 | instances = {} 7 | 8 | def getinstance(*args, **kwargs): 9 | if class_ not in instances: 10 | instances[class_] = class_(*args, **kwargs) 11 | return instances[class_] 12 | return getinstance 13 | 14 | 15 | # Object repository, the repo should store 16 | # key as string and value as json object 17 | class BaseRepo(object): 18 | def register(self, domain, key, value): 19 | pass 20 | 21 | def unregister(self, domain, key): 22 | pass 23 | 24 | def get(self, domain, key=None): 25 | pass 26 | 27 | def domains(self): 28 | pass 29 | 30 | def clean(self): 31 | pass 32 | 33 | 34 | class IMRepo(BaseRepo): 35 | 36 | def __init__(self): 37 | BaseRepo.__init__(self) 38 | self._repo = {} 39 | 40 | def register(self, domain, key, value): 41 | if self._repo.get(domain) is None: 42 | self._repo[domain] = {} 43 | 44 | self._repo[domain][key] = value 45 | 46 | def unregister(self, domain, key): 47 | if self._repo.get(domain) is None: 48 | return 49 | 50 | if self._repo.get(domain).get(key) is None: 51 | return 52 | 53 | del self._repo.get(domain)[key] 54 | 55 | def get(self, domain, key=None): 56 | if domain is None: 57 | return None 58 | 59 | if self._repo.get(domain) is None: 60 | return None 61 | 62 | if key is None: 63 | return self._repo.get(domain) 64 | 65 | if self._repo.get(domain).get(key) is None: 66 | return None 67 | 68 | return self._repo.get(domain)[key] 69 | 70 | def domains(self): 71 | return self._repo.keys() 72 | 73 | def clean(self): 74 | self._repo = dict() 75 | return 76 | 77 | 78 | class SqliteRepo(BaseRepo): 79 | 80 | def __init__(self): 81 | BaseRepo.__init__(self) 82 | # TODO : config the db name here 83 | self._conn = sqlite3.connect('repo.db', check_same_thread=False) 84 | self._domains = set() 85 | 86 | def register(self, domain, key, value): 87 | self._domains.add(domain) 88 | c = self._conn.cursor() 89 | c.execute('''CREATE TABLE IF NOT EXISTS {} 90 | (key text PRIMARY KEY, value text)'''.format(domain)) 91 | 92 | t = (key, json.dumps(value)) 93 | c.execute('''REPLACE INTO {} VALUES 94 | (?,?)'''.format(domain), t) 95 | 96 | self._conn.commit() 97 | # self._conn.close() 98 | 99 | def unregister(self, domain, key): 100 | c = self._conn.cursor() 101 | t = (key,) 102 | c.execute('''DELETE FROM {} WHERE key=?'''.format(domain), t) 103 | 104 | self._conn.commit() 105 | 106 | def get(self, domain, key=None): 107 | c = self._conn.cursor() 108 | t = (domain,) 109 | c.execute( 110 | "SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", t) 111 | count = c.fetchone() 112 | 113 | if int(count[0]) == 0: 114 | return None 115 | 116 | if key == None: 117 | result = dict() 118 | for row in c.execute('SELECT * FROM {}'.format(domain)): 119 | result[row[0]] = json.loads(row[1], strict=False) 120 | return result 121 | 122 | t = (key,) 123 | c.execute('SELECT value FROM {} WHERE key=?'.format(domain), t) 124 | result = c.fetchone() 125 | return json.loads(result[0], strict=False) 126 | 127 | def domains(self): 128 | # todo run this at initialize and update on register/unregister 129 | c = self._conn.cursor() 130 | cursor = c.execute( 131 | "SELECT name FROM sqlite_master WHERE type='table'") 132 | return [ row[0] for row in cursor] 133 | 134 | def clean(self): 135 | c = self._conn.cursor() 136 | for domain in self._domains: 137 | c.execute('''DELETE FROM {} '''.format(domain)) 138 | self._conn.commit() 139 | self._domains = set() 140 | return 141 | 142 | 143 | @singleton 144 | class repository(object): 145 | # a default Sqlite repo is use, need read configuration 146 | # to use different DB for repo 147 | _repo = SqliteRepo() 148 | 149 | def register(self, domain, key, value): 150 | return self._repo.register(domain, key, value) 151 | 152 | def unregister(self, domain, key): 153 | return self._repo.unregister(domain, key) 154 | 155 | def get(self, domain, key=None): 156 | return self._repo.get(domain, key) 157 | 158 | def load(self, repo): 159 | self._repo = repo 160 | 161 | def domains(self): 162 | return self._repo.domains() 163 | 164 | def clean(self): 165 | return self._repo.clean() 166 | 167 | def dumps(self, path): 168 | repo = dict() 169 | for domain in self.domains(): 170 | repo[domain] = self.get(domain) 171 | 172 | with open(path, "w") as f: 173 | f.write(json.dumps(repo, indent=2)) 174 | 175 | def loads(self, path): 176 | self.clean() 177 | with open(path, "r") as f: 178 | repo = json.loads(f.read()) 179 | for domain, domain_value in repo.iteritems(): 180 | for key, value in domain_value.iteritems(): 181 | self.register(domain, key, value) 182 | -------------------------------------------------------------------------------- /src/message/__init__.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import Queue 3 | 4 | 5 | # A simple in memory message bus 6 | 7 | class Subscriber(object): 8 | def __init__(self, name): 9 | self._name = name 10 | 11 | @property 12 | def name(self): 13 | return self._name 14 | 15 | def notify(self, message): 16 | print("{} got notified for {}".format(self._name, message)) 17 | 18 | 19 | class Bus(threading.Thread): 20 | def __init__(self, name): 21 | threading.Thread.__init__(self) 22 | self._name = name 23 | self._subscribers = list() 24 | self._queue = Queue.Queue() 25 | self._stop = False 26 | 27 | @property 28 | def name(self): 29 | return self._name 30 | 31 | def publish(self, message): 32 | self._queue.put(message) 33 | 34 | def subcribe(self, subscriber): 35 | if subscriber not in self._subscribers: 36 | self._subscribers.append(subscriber) 37 | 38 | def unsubscribe(self, subscriber): 39 | if subscriber in self._subscribers: 40 | index = self._subscribers.index(subscriber) 41 | self._subscribers.pop(index) 42 | 43 | def run(self): 44 | print("{} Start".format(self._name)) 45 | while not self._stop: 46 | try: 47 | message = self._queue.get(True,3) 48 | for subscriber in self._subscribers: 49 | subscriber.notify(message) 50 | except: 51 | pass 52 | print("{} Stop".format(self._name)) 53 | 54 | def stop(self): 55 | print("{} set to complete".format(self._name)) 56 | self._stop = True 57 | -------------------------------------------------------------------------------- /src/node_specs/__init__.py: -------------------------------------------------------------------------------- 1 | # this code is used to create initial node contents -------------------------------------------------------------------------------- /src/node_specs/add.py: -------------------------------------------------------------------------------- 1 | spec = { 2 | "title": "add", 3 | "id": "pyflow.transform.add", 4 | } 5 | 6 | 7 | def func(a, b): 8 | """ 9 | :params: a,b 10 | :ptypes: String,String 11 | :returns: out 12 | :rtype: String 13 | """ 14 | 15 | return a + b 16 | -------------------------------------------------------------------------------- /src/node_specs/cli.py: -------------------------------------------------------------------------------- 1 | spec = { 2 | "title": "cli", 3 | "id": "pyflow.source.cli", 4 | } 5 | 6 | 7 | def func(command): 8 | """ 9 | :params: command 10 | :ptypes: String 11 | :returns: out 12 | :rtype: String 13 | """ 14 | 15 | import shlex 16 | import subprocess 17 | # This cli cannot hand code that refresh the screen like top 18 | args = shlex.split(command) 19 | p = subprocess.Popen( 20 | args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 21 | outdata, errdata = p.communicate() 22 | if len(errdata) != 0: 23 | raise Exception( 24 | 'Failed to execut command % , error is {}'.format(command, errdata)) 25 | return outdata 26 | -------------------------------------------------------------------------------- /src/node_specs/jsonpath.py: -------------------------------------------------------------------------------- 1 | spec = { 2 | "title": "JSON Path", 3 | "id": "pyflow.transform.jsonpath", 4 | } 5 | 6 | 7 | def func(source, path): 8 | """ 9 | :params: source, path 10 | :ptypes: String, String 11 | :returns: out 12 | :rtype: String 13 | """ 14 | from jsonpath_rw import jsonpath, parse 15 | import json 16 | 17 | if type(source) == type('') or type(source) == type(u''): 18 | source = json.loads(source) 19 | elif type(source) != type({}): 20 | return "Invalid source type {}".format(type(source)) 21 | 22 | jsonpath_expr = parse(path) 23 | ret = jsonpath_expr.find(source) 24 | 25 | if len(ret) == 1: 26 | return ret[0].value 27 | 28 | ret = [match.value for match in ret] 29 | return ret 30 | -------------------------------------------------------------------------------- /src/node_specs/split.py: -------------------------------------------------------------------------------- 1 | spec = { 2 | "title": "split", 3 | "id": "pyflow.transform.split", 4 | } 5 | output_keys = ["a", "b"] 6 | 7 | 8 | def func(source, delimiter): 9 | """ 10 | :params: source,delimiter 11 | :ptypes: String,String 12 | :returns: a,b 13 | :rtype: String, String 14 | """ 15 | result = source.split(delimiter, 1) 16 | out = dict() 17 | 18 | out["a"] = result[0] 19 | if len(result) > 1: 20 | out["b"] = result[1] 21 | else: 22 | out["b"] = "" 23 | return out 24 | -------------------------------------------------------------------------------- /src/nodemaker.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | 3 | # TODO: node maker is just a test tool which does not support port type yet, all default to 'String' 4 | def create_node_spec(node_def): 5 | output_keys = None 6 | # TODO : unsecure 7 | exec(node_def) 8 | 9 | argspec = inspect.getargspec(func) 10 | input_names = argspec.args 11 | input_defaults = argspec.defaults or [] 12 | input_ports = [] 13 | default_start = len(input_names) - len(input_defaults) 14 | for i in range(len(input_names)): 15 | input = { 16 | 'name': input_names[i], 17 | 'order': i, 18 | 'type' : "String" 19 | } 20 | if i >= default_start: 21 | input['default'] = input_defaults[i - default_start] 22 | input_ports.append(input) 23 | 24 | spec['port'] = {'input': input_ports} 25 | if output_keys: 26 | spec['port']['output'] = [] 27 | for name in output_keys: 28 | spec['port']['output'].append({'name': name,'type':'String'}) 29 | index = node_def.find('\ndef func') 30 | spec['func'] = node_def[index + 1:] 31 | return spec 32 | 33 | 34 | # print create_node_spec(open('node_specs/breaker.py').read()) 35 | -------------------------------------------------------------------------------- /src/repo.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/repo.db -------------------------------------------------------------------------------- /src/server.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import request 3 | from flask import jsonify 4 | 5 | import os 6 | import json 7 | 8 | import nodemaker 9 | import fbp 10 | 11 | from fbp.port import Port 12 | 13 | app = Flask(__name__, static_url_path="") 14 | 15 | 16 | @app.route("/") 17 | def index(): 18 | return app.send_static_file("index.html") 19 | 20 | 21 | @app.route("/nodestree", methods=['GET']) 22 | def nodestree(): 23 | tree = list() 24 | repository = fbp.repository() 25 | node_specs = repository.get("nodespec") 26 | 27 | for k, v in node_specs.iteritems(): 28 | _insert(tree, v) 29 | return jsonify(tree) 30 | 31 | 32 | def _insert(treeroot, node): 33 | id = node["id"] 34 | ids = id.split(".") 35 | found = False 36 | 37 | for n in treeroot: 38 | if n["id"] == ids[0]: 39 | found = True 40 | _inset_node(n, node, ids) 41 | 42 | if not found: 43 | item = dict() 44 | item["id"] = ids[0] 45 | item["title"] = ids[0] 46 | item["children"] = list() 47 | treeroot.append(item) 48 | _inset_node(item, node, ids) 49 | 50 | return 51 | 52 | 53 | def _inset_node(parent, node, path): 54 | if len(path) == 1: 55 | if path[0] == parent["id"]: 56 | parent["value"] = node 57 | else: 58 | if path[0] == parent["id"]: 59 | children = parent["children"] 60 | found = False 61 | for item in children: 62 | if item["id"] == path[1]: 63 | _inset_node(item, node, path[1:]) 64 | found = True 65 | 66 | if not found: 67 | item = dict() 68 | item["id"] = path[1] 69 | item["title"] = path[1] 70 | item["children"] = list() 71 | parent["children"].append(item) 72 | _inset_node(item, node, path[1:]) 73 | return 74 | 75 | 76 | @app.route("/nodes", methods=['GET', 'POST']) 77 | def nodes(): 78 | repository = fbp.repository() 79 | if request.method == 'POST': 80 | node = request.get_json() 81 | repository.register("nodespec", node["id"], node) 82 | return jsonify(node), 200, {'ContentType': 'application/json'} 83 | else: 84 | node_specs = repository.get("nodespec") 85 | 86 | if not node_specs: 87 | return jsonify({}), 200, {'ContentType': 'application/json'} 88 | 89 | # Adding default output when it is not there 90 | for k, v in node_specs.iteritems(): 91 | if not v["port"].has_key("output"): 92 | v["port"]["output"] = list() 93 | v["port"]["output"].append({"name": "out"}) 94 | 95 | return jsonify(node_specs), 200, {'ContentType': 'application/json'} 96 | 97 | 98 | @app.route("/nodes/", methods=['GET', 'DELETE', 'PUT']) 99 | def get_node(id): 100 | repository = fbp.repository() 101 | if request.method == 'GET': 102 | node = repository.get("nodespec", id) 103 | return jsonify(node), 200, {'ContentType': 'application/json'} 104 | elif request.method == 'DELETE': 105 | repository.unregister("nodespec", id) 106 | return json.dumps({'success': True}), 200, {'ContentType': 'application/json'} 107 | elif request.method == 'PUT': 108 | node = request.get_json() 109 | # TODO Valude the node here 110 | repository.register("nodespec", id, node) 111 | return jsonify(node), 200, {'ContentType': 'application/json'} 112 | 113 | return json.dumps({'success': False}), 400, {'ContentType': 'application/json'} 114 | 115 | 116 | @app.route("/flows", methods=['GET', 'POST']) 117 | def flows(): 118 | repository = fbp.repository() 119 | if request.method == 'POST': 120 | flow = request.get_json() 121 | repository.register("flow", flow["id"], flow) 122 | return jsonify(flow) 123 | else: 124 | flows = repository.get("flow") 125 | if flows is None: 126 | return jsonify({}) 127 | 128 | result = [v for k, v in flows.items()] 129 | return jsonify(result) 130 | 131 | 132 | @app.route("/flows/", methods=['GET']) 133 | def get_flow(id): 134 | repository = fbp.repository() 135 | node = repository.get("flow", id) 136 | return jsonify(node) 137 | 138 | 139 | @app.route("/runflow", methods=['POST']) 140 | def runflow(): 141 | try: 142 | data = request.get_json() 143 | print(json.dumps(data)) 144 | return jsonify(fbp.run_flow(data)) 145 | except Exception as e: 146 | return json.dumps({"error": str(e)}), 500 147 | 148 | 149 | @app.route("/dumprepo", methods=['POST']) 150 | def dumprepo(): 151 | try: 152 | data = request.get_json() 153 | repository = fbp.repository() 154 | repository.dumps(data["path"]) 155 | return jsonify(data) 156 | except Exception as e: 157 | return json.dumps({"error": str(e)}), 500 158 | 159 | 160 | @app.route("/loadrepo", methods=['POST']) 161 | def loadrepo(): 162 | try: 163 | data = request.get_json() 164 | repository = fbp.repository() 165 | repository.loads(data["path"]) 166 | return jsonify(data) 167 | except Exception as e: 168 | return json.dumps({"error": str(e)}), 500 169 | 170 | 171 | @app.route("/ports/types", methods=['GET']) 172 | def get_supported_port_types(): 173 | return jsonify(types=Port.support_types()) 174 | 175 | 176 | def load_node_spec(): 177 | records = [] 178 | for file in os.listdir('node_specs'): 179 | if file.endswith('.py') and file != '__init__.py': 180 | with open('node_specs' + os.path.sep + file)as f: 181 | spec = nodemaker.create_node_spec(f.read()) 182 | records.append(json.dumps(spec)) 183 | 184 | repository = fbp.repository() 185 | for r in records: 186 | node = json.loads(r) 187 | repository.register("nodespec", node["id"], node) 188 | 189 | 190 | def init(): 191 | # load node spec from spec folders 192 | # load_node_spec() 193 | 194 | # TODO 195 | # initialize flows 196 | pass 197 | 198 | 199 | if __name__ == "__main__": 200 | init() 201 | app.run(host="0.0.0.0", threaded=True) 202 | -------------------------------------------------------------------------------- /src/static/bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pyflow", 3 | "homepage": "https://github.com/gangtao/pyflow", 4 | "authors": [ 5 | "Gang Tao " 6 | ], 7 | "description": "", 8 | "main": "", 9 | "keywords": [ 10 | "pyflow" 11 | ], 12 | "license": "MIT", 13 | "ignore": [ 14 | "**/.*", 15 | "node_modules", 16 | "bower_components", 17 | "test", 18 | "tests" 19 | ], 20 | "dependencies": { 21 | "jquery": "^3.1.1", 22 | "codemirror": "^5.22.2", 23 | "bootstrap": "^3.3.7", 24 | "x-editable": "^1.5.1", 25 | "json2": "*", 26 | "d3": "*", 27 | "js-beautify": "*", 28 | "jsplumb": "^2.3.2", 29 | "jquery-ui": "^1.12.1", 30 | "requirejs": "~2.1.22" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/static/css/OpenSans-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-Bold.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-BoldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-BoldItalic.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-ExtraBold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-ExtraBold.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-ExtraBoldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-ExtraBoldItalic.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-Italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-Italic.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-Light.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-Light.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-LightItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-LightItalic.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-Regular.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-Semibold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-Semibold.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans-SemiboldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans-SemiboldItalic.ttf -------------------------------------------------------------------------------- /src/static/css/OpenSans.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/OpenSans.woff -------------------------------------------------------------------------------- /src/static/css/images/bg_hr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/images/bg_hr.png -------------------------------------------------------------------------------- /src/static/css/images/blacktocat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/images/blacktocat.png -------------------------------------------------------------------------------- /src/static/css/images/collapse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/images/collapse.png -------------------------------------------------------------------------------- /src/static/css/images/expand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/images/expand.png -------------------------------------------------------------------------------- /src/static/css/images/icon_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/images/icon_download.png -------------------------------------------------------------------------------- /src/static/css/images/sprite_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/css/images/sprite_download.png -------------------------------------------------------------------------------- /src/static/css/pyflow.css: -------------------------------------------------------------------------------- 1 | /************/ 2 | /*Tree Style*/ 3 | /************/ 4 | 5 | .tree li { 6 | margin: 0px 0; 7 | list-style-type: none; 8 | position: relative; 9 | padding: 18px 5px 0px 5px; 10 | } 11 | 12 | .tree li::before { 13 | content: ''; 14 | position: absolute; 15 | top: 0; 16 | width: 1px; 17 | height: 100%; 18 | right: auto; 19 | left: -20px; 20 | border-left: 1px solid #ccc; 21 | bottom: 50px; 22 | } 23 | 24 | .tree li::after { 25 | content: ''; 26 | position: absolute; 27 | top: 30px; 28 | width: 25px; 29 | height: 10px; 30 | right: auto; 31 | left: -20px; 32 | border-top: 1px solid #ccc; 33 | } 34 | 35 | .tree li a { 36 | display: inline-block; 37 | border: 1px solid #2e6da4; 38 | padding: 5px 10px; 39 | text-decoration: none; 40 | color: #fff; 41 | font-family: arial, verdana, tahoma; 42 | font-size: 11px; 43 | border-radius: 5px; 44 | -webkit-border-radius: 5px; 45 | -moz-border-radius: 5px; 46 | background: #337ab7; 47 | } 48 | 49 | 50 | /*Remove connectors before root*/ 51 | 52 | .tree > ul > li::before, 53 | .tree > ul > li::after { 54 | border: 0; 55 | } 56 | 57 | 58 | /*Remove connectors after last child*/ 59 | 60 | .tree li:last-child::before { 61 | height: 30px; 62 | } 63 | 64 | 65 | /*Time for some hover effects*/ 66 | /*We will apply the hover effect the the lineage of the element also*/ 67 | 68 | .tree li a:hover, 69 | .tree li a:hover+ul li a { 70 | background: #73A550; 71 | color: #fff; 72 | border: 1px solid #94a0b4; 73 | } 74 | 75 | 76 | /*Connector styles on hover*/ 77 | 78 | .tree li a:hover+ul li::after, 79 | .tree li a:hover+ul li::before, 80 | .tree li a:hover+ul::before, 81 | .tree li a:hover+ul ul::before { 82 | border-color: #94a0b4; 83 | } 84 | 85 | .pyflowtree > li > a { 86 | display: inline-block; 87 | border: 1px solid #2e6da4; 88 | padding: 5px 10px; 89 | text-decoration: none; 90 | color: #fff; 91 | font-family: arial, verdana, tahoma; 92 | font-size: 11px; 93 | border-radius: 5px; 94 | -webkit-border-radius: 5px; 95 | -moz-border-radius: 5px; 96 | background: #337ab7; 97 | } 98 | 99 | 100 | /************/ 101 | /*Node Style*/ 102 | /************/ 103 | 104 | .pyflownode { 105 | border: 1px solid #346789; 106 | box-shadow: 2px 2px 19px #aaa; 107 | -o-box-shadow: 2px 2px 19px #aaa; 108 | -webkit-box-shadow: 2px 2px 19px #aaa; 109 | -moz-box-shadow: 2px 2px 19px #aaa; 110 | -moz-border-radius: 0.5em; 111 | border-radius: 0.5em; 112 | opacity: 0.8; 113 | filter: alpha(opacity=80); 114 | width: 8em; 115 | height: 4em; 116 | line-height: 2em; 117 | text-align: center; 118 | z-index: 20; 119 | position: absolute; 120 | background-color: #eeeeee; 121 | color: black; 122 | font-family: helvetica; 123 | padding: 0.5em; 124 | font-size: 1em; 125 | } 126 | 127 | .pyflownode:hover { 128 | box-shadow: 2px 2px 19px #444; 129 | -o-box-shadow: 2px 2px 19px #444; 130 | -webkit-box-shadow: 2px 2px 19px #444; 131 | -moz-box-shadow: 2px 2px 19px #444; 132 | opacity: 0.8; 133 | filter: alpha(opacity=80); 134 | } 135 | 136 | ._jsPlumb_connector { 137 | z-index: 4; 138 | } 139 | 140 | ._jsPlumb_endpoint { 141 | z-index: 21; 142 | cursor: pointer; 143 | } 144 | 145 | ._jsPlumb_dragging { 146 | z-index: 4000; 147 | } 148 | 149 | .flowbutton { 150 | margin-left: 3px; 151 | } 152 | 153 | .flowbody { 154 | height: 500px; 155 | } 156 | 157 | #flow-panel { 158 | height: 500px; 159 | width: 100%; 160 | position: absolute; 161 | } 162 | 163 | .node-fail { 164 | background-color: OrangeRed 165 | } 166 | 167 | .node-skip { 168 | background-color: Silver 169 | } 170 | -------------------------------------------------------------------------------- /src/static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /src/static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /src/static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /src/static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /src/static/img/animated-progress.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/src/static/img/animated-progress.gif -------------------------------------------------------------------------------- /src/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | PyFlow 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 24 | 25 | 26 | 29 | 30 | 31 | 32 | 33 | 66 | 67 |
68 |
69 | 70 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /src/static/js/app.js: -------------------------------------------------------------------------------- 1 | require(["flow", "node", "model/repo", "util"], function(Flow, Node, Repo, Util) { 2 | 3 | jsPlumb.ready(function() { 4 | console.log("jsPlumb is ready to use"); 5 | 6 | $.fn.editable.defaults.mode = 'inline'; 7 | 8 | $("#menuFlow").click(function() { 9 | toggleMenu($(this)); 10 | Flow.render(); 11 | }); 12 | 13 | $("#menuNode").click(function() { 14 | toggleMenu($(this)); 15 | Node.render(); 16 | }); 17 | 18 | $("#load-action-menu").click(function() { 19 | loadRepo(); 20 | }); 21 | 22 | $("#dump-action-menu").click(function() { 23 | dumpRepo(); 24 | }); 25 | 26 | Flow.render(); 27 | }); 28 | 29 | function toggleMenu(me) { 30 | if (!me.parent().hasClass("active")) { 31 | $("#menuNode").parent().toggleClass("active"); 32 | $("#menuFlow").parent().toggleClass("active"); 33 | } 34 | } 35 | 36 | function loadRepo() { 37 | var modal_id = "repo_load_modal"; 38 | var load_button_id = "repo_load_action"; 39 | var file_path_id = "repo_load_file_path"; 40 | 41 | var modal = Util.getModal(modal_id, "Load Repo", function(modal) { 42 | var body = modal.select(".modal-body"); 43 | var form = body.append("form"); 44 | var group = form.append("div").classed("form-group", true); 45 | group.append("label").text("Full File Path").style("margin-right", "5px"); 46 | group.append("a").attr("href", "#").attr("id", file_path_id); 47 | 48 | $("#" + file_path_id).text("/tmp/repo.json").editable(); 49 | 50 | var footer = modal.select(".modal-footer"); 51 | footer.append("button").attr("type", "button").classed("btn btn-default", true).attr("id", load_button_id).text("Load"); 52 | }); 53 | 54 | $("#" + load_button_id).click(function() { 55 | //Do load here 56 | var file = d3.select("#" + file_path_id).text(); 57 | console.log(file); 58 | var repo = new Repo(); 59 | repo.load(file, function(){ 60 | Flow.render(); 61 | }); 62 | $("#" + modal_id).modal('hide'); 63 | }); 64 | 65 | $("#" + modal_id).modal('show'); 66 | } 67 | 68 | function dumpRepo() { 69 | var modal_id = "repo_dump_modal"; 70 | var dump_button_id = "repo_dump_action"; 71 | var file_path_id = "repo_dump_file_path"; 72 | 73 | var modal = Util.getModal(modal_id, "Dump Repo", function(modal) { 74 | var body = modal.select(".modal-body"); 75 | var footer = modal.select(".modal-footer"); 76 | 77 | var form = body.append("form"); 78 | 79 | var group = form.append("div").classed("form-group", true); 80 | group.append("label").text("Full File Path").style("margin-right", "5px"); 81 | group.append("a").attr("href", "#").attr("id", file_path_id); 82 | 83 | $("#" + file_path_id).text("/tmp/repo.json").editable(); 84 | 85 | footer.append("button").attr("type", "button").classed("btn btn-default", true).attr("id", dump_button_id).text("Dump"); 86 | }); 87 | 88 | $("#" + dump_button_id).click(function() { 89 | //DO Dumps here 90 | var file = d3.select("#" + file_path_id).text(); 91 | console.log(file); 92 | var repo = new Repo(); 93 | repo.dump(file); 94 | $("#" + modal_id).modal('hide'); 95 | }); 96 | 97 | $("#" + modal_id).modal('show'); 98 | } 99 | 100 | }); -------------------------------------------------------------------------------- /src/static/js/components/flowCanvas.js: -------------------------------------------------------------------------------- 1 | define(["model/flow", "util"], function(Flow, Util) { 2 | //Global Paint Styles 3 | var connectorPaintStyle = { 4 | strokeWidth: 2, 5 | stroke: "#61B7CF", 6 | joinstyle: "round", 7 | outlineStroke: "white", 8 | outlineWidth: 2 9 | }, 10 | connectorHoverStyle = { 11 | strokeWidth: 3, 12 | stroke: "#216477", 13 | outlineWidth: 5, 14 | outlineStroke: "white" 15 | }, 16 | endpointHoverStyle = { 17 | fill: "#216477", 18 | stroke: "#216477" 19 | }, 20 | sourceEndpoint = { 21 | endpoint: "Dot", 22 | paintStyle: { 23 | stroke: "#7AB02C", 24 | fill: "transparent", 25 | radius: 7, 26 | strokeWidth: 1 27 | }, 28 | isSource: true, 29 | maxConnections: -1, 30 | connector: ["Flowchart", { stub: [40, 60], gap: 10, cornerRadius: 5, alwaysRespectStubs: true }], 31 | connectorStyle: connectorPaintStyle, 32 | hoverPaintStyle: endpointHoverStyle, 33 | connectorHoverStyle: connectorHoverStyle, 34 | dragOptions: {}, 35 | overlays: [ 36 | ["Label", { 37 | location: [0.5, 1.5], 38 | label: "Drag", 39 | cssClass: "endpointSourceLabel", 40 | visible: false 41 | }] 42 | ] 43 | }, 44 | 45 | // the definition of target endpoints (will appear when the user drags a connection) 46 | targetEndpoint = { 47 | endpoint: "Dot", 48 | paintStyle: { fill: "#7AB02C", radius: 7 }, 49 | hoverPaintStyle: endpointHoverStyle, 50 | maxConnections: -1, 51 | dropOptions: { hoverClass: "hover", activeClass: "active" }, 52 | isTarget: true, 53 | overlays: [ 54 | ["Label", { location: [0.5, -0.5], label: "Drop", cssClass: "endpointTargetLabel", visible: false }] 55 | ] 56 | }; 57 | 58 | var FLOW_PANEL_ID = "flow-panel"; 59 | var DEFAULT_FLOW_NAME = "untitled"; 60 | var DEFAULT_FLOW_ID = "pyflow.sample" 61 | 62 | var Canvas = function Canvas(rootId, nodeSpec, nodeInspector) { 63 | this._instance = jsPlumb.getInstance({ 64 | Connector: ["Flowchart", { stub: [40, 60], gap: 10, cornerRadius: 5, alwaysRespectStubs: true }], 65 | DragOptions: { cursor: "pointer", zIndex: 2000 }, 66 | Container: FLOW_PANEL_ID 67 | }); 68 | this._rootId = rootId; 69 | this._nodeSpec = nodeSpec; 70 | this._inspector = nodeInspector; 71 | this._selectedNode = undefined; 72 | this._heading = undefined; 73 | this._currentFlow = new Flow(DEFAULT_FLOW_ID, DEFAULT_FLOW_NAME); 74 | this._inspector.onNotify(this._update, this); 75 | this._panel = undefined; 76 | this._titleSpan = undefined; 77 | }; 78 | 79 | Canvas.prototype.getNodeSpecById = function(id) { 80 | if (this._nodeSpec === undefined) { 81 | return undefined; 82 | } 83 | return this._nodeSpec[id]; 84 | }; 85 | 86 | Canvas.prototype.render = function() { 87 | this._clear(); 88 | var root = d3.select("#" + this._rootId); 89 | this._panel = Util.addPanel(root, "Flow"); 90 | var me = this; 91 | 92 | this._heading = root.select(".panel-heading"); 93 | this._titleSpan = this._heading.append("span").classed("label label-primary", true).style("margin-left", "5px"); 94 | this._titleSpan.text(this._currentFlow.flow().name); 95 | this._heading.append("br"); 96 | 97 | this._heading.append("button").classed("glyphicon glyphicon-plus-sign flowbutton", true).on("click", function() { 98 | me._newflow(); 99 | }); 100 | 101 | this._heading.append("button").classed("glyphicon glyphicon-floppy-open flowbutton", true).on("click", function() { 102 | me._load(); 103 | }); 104 | 105 | this._heading.append("button").classed("glyphicon glyphicon-floppy-save flowbutton", true).on("click", function() { 106 | me._save(); 107 | }); 108 | 109 | this._heading.append("button").classed("glyphicon glyphicon-trash flowbutton", true).on("click", function() { 110 | me._clear(); 111 | }); 112 | 113 | this._heading.append("button").classed("glyphicon glyphicon-search flowbutton", true).on("click", function() { 114 | me._showFlowSource(); 115 | }); 116 | 117 | this._heading.append("button").classed("glyphicon glyphicon-remove-circle flowbutton", true).on("click", function() { 118 | me._removeNode(); 119 | }).style("visibility", "hidden"); 120 | 121 | this._panel.select(".panel-body").classed("flowbody", true).append("div").attr("id", FLOW_PANEL_ID); 122 | 123 | //Add node on drag & drop 124 | $("#" + FLOW_PANEL_ID).on("drop", function(ev) { 125 | //avoid event conlict for jsPlumb 126 | if (ev.target.className.indexOf("_jsPlumb") >= 0) { 127 | return; 128 | } 129 | ev.preventDefault(); 130 | 131 | var mx = "" + ev.originalEvent.offsetX + "px"; 132 | var my = "" + ev.originalEvent.offsetY + "px"; 133 | 134 | var nodeSpecId = ev.originalEvent.dataTransfer.getData("text"); 135 | var nodeSpec = me.getNodeSpecById(nodeSpecId); 136 | if (nodeSpec === undefined) { 137 | return; 138 | } 139 | var uid = "node" + (new Date().getTime()); 140 | 141 | //Update Flow Specification 142 | var node_def = {} 143 | node_def.id = uid; 144 | node_def.spec_id = nodeSpecId; 145 | node_def.name = nodeSpec.title; 146 | node_def.ports = []; // ports values 147 | node_def.ui = {}; 148 | node_def.ui.x = mx; 149 | node_def.ui.y = my; 150 | 151 | me._currentFlow.addnode(node_def); 152 | me._drawNode(node_def); 153 | 154 | }).on("dragover", function(ev) { 155 | ev.preventDefault(); 156 | }); 157 | 158 | this._initInstance(); 159 | 160 | //drawSampleFlow(instance); 161 | jsPlumb.fire("jsFlowLoaded", this._instance); 162 | }; 163 | 164 | Canvas.prototype._initInstance = function() { 165 | var me = this; 166 | this._instance.bind("connection", function(info, originalEvent) { 167 | var sourceId = info.sourceId; 168 | var targetId = info.targetId; 169 | var sourcePort = info.sourceEndpoint.getLabel(); 170 | var targetPort = info.targetEndpoint.getLabel(); 171 | 172 | me._currentFlow.connect(sourceId, targetId, sourcePort, targetPort); 173 | }); 174 | 175 | this._instance.bind("connectionDetached", function(info, originalEvent) { 176 | var sourceId = info.sourceId; 177 | var targetId = info.targetId; 178 | var sourcePort = info.sourceEndpoint.getLabel(); 179 | var targetPort = info.targetEndpoint.getLabel(); 180 | 181 | me._currentFlow.disconnect(sourceId, targetId, sourcePort, targetPort); 182 | }); 183 | 184 | this._instance.bind("connectionMoved", function(info, originalEvent) { 185 | var osourceId = info.originalSourceId; 186 | var otargetId = info.originalTargetId; 187 | var osourcePort = info.originalSourceEndpoint.getLabel(); 188 | var otargetPort = info.originalTargetEndpoint.getLabel(); 189 | 190 | var nsourceId = info.newSourceId; 191 | var ntargetId = info.newTargetId; 192 | var nsourcePort = info.newSourceEndpoint.getLabel(); 193 | var ntargetPort = info.newTargetEndpoint.getLabel(); 194 | 195 | me._currentFlow.disconnect(osourceId, otargetId, osourcePort, otargetPort); 196 | me._currentFlow.connect(nsourceId, ntargetId, nsourcePort, ntargetPort); 197 | }); 198 | } 199 | 200 | Canvas.prototype._drawSampleFlow = function() { 201 | //two sample nodes with one default connection 202 | var node1 = this._addNode(FLOW_PANEL_ID, "node1", { "title": "node1" }, { x: "80px", y: "120px" }); 203 | var node2 = this._addNode(FLOW_PANEL_ID, "node2", { "title": "node2" }, { x: "380px", y: "120px" }); 204 | 205 | this._addPorts(node1, ["out1", "out2"], "output"); 206 | this._addPorts(node2, ["in", "in1", "in2"], "input"); 207 | 208 | this._connectPorts(node1, "out2", node2, "in"); 209 | this._connectPorts(node1, "out2", node2, "in1"); 210 | 211 | this._instance.draggable($(node1)); 212 | this._instance.draggable($(node2)); 213 | }; 214 | 215 | Canvas.prototype._drawNode = function(nodedef) { 216 | var nodeSpec = this.getNodeSpecById(nodedef.spec_id); 217 | if (nodeSpec === undefined) { 218 | console.log("Now such spec : " + nodedef.spec_id); 219 | return; 220 | } 221 | 222 | //TODO: handle the case where nodedef has no ui information. 223 | // DO auto layout, or put everything at 0.0 224 | var node = this._addNode(FLOW_PANEL_ID, nodedef.id, nodeSpec, { x: nodedef.ui.x, y: nodedef.ui.y }); 225 | var i = 0, 226 | length = nodeSpec.port.input.length; 227 | var input_port_name = []; 228 | for (; i < length; i++) { 229 | input_port_name.push(nodeSpec.port.input[i].name); 230 | //TODO : sort by order 231 | } 232 | this._addPorts(node, input_port_name, "input"); 233 | 234 | if (nodeSpec.port.output === undefined) { 235 | this._addPorts(node, ["out"], "output"); 236 | } else { 237 | i = 0, length = nodeSpec.port.output.length; 238 | var output_port_name = []; 239 | for (; i < length; i++) { 240 | output_port_name.push(nodeSpec.port.output[i].name); 241 | } 242 | this._addPorts(node, output_port_name, "output"); 243 | } 244 | 245 | this._instance.draggable($(node)); 246 | return node; 247 | } 248 | 249 | //Flow UI control logic 250 | //UI Code to create node and port 251 | Canvas.prototype._addNode = function(parentId, nodeId, nodeSpec, position) { 252 | var me = this; 253 | var panel = d3.select("#" + parentId); 254 | //construct the node data copied from the nodeSpec 255 | var data = {}; 256 | $.extend(data, nodeSpec, { nodeId: nodeId }); 257 | 258 | panel.append("div").datum(data) 259 | .style("top", position.y) 260 | .style("left", position.x) 261 | .classed("pyflownode", true) 262 | .attr("id", function(d) { 263 | return d.nodeId; 264 | }) 265 | .text(function(d) { 266 | return d.title; 267 | }) 268 | .on("click", function(d) { 269 | me._inspector.showNodeDetails(d, me._currentFlow); 270 | me._selectedNode = d; 271 | d3.select(".glyphicon-remove-circle").style("visibility", "visible"); 272 | }) 273 | .on("mouseover", function(d) { 274 | //TODO : handling hover style here 275 | //d3.select(this).style("border", "3px #000 solid"); 276 | }) 277 | .on("mouseout", function(d) { 278 | //d3.select(this).style("border", "2px #000 solid"); 279 | }); 280 | 281 | return jsPlumb.getSelector("#" + nodeId)[0]; 282 | }; 283 | 284 | Canvas.prototype._addPorts = function(node, ports, type) { 285 | //Assume horizental layout 286 | var number_of_ports = ports.length; 287 | var i = 0; 288 | var height = $(node).height(); //Note, jquery does not include border for height 289 | var y_offset = 1 / (number_of_ports + 1); 290 | var y = 0; 291 | 292 | for (; i < number_of_ports; i++) { 293 | var anchor = [0, 0, 0, 0]; 294 | var isSource = false, 295 | isTarget = false; 296 | if (type === "output") { 297 | anchor[0] = 1; 298 | isSource = true; 299 | } else { 300 | isTarget = true; 301 | } 302 | 303 | anchor[1] = y + y_offset; 304 | y = anchor[1]; 305 | 306 | var endpoint = undefined; 307 | 308 | if (isSource) { 309 | endpoint = this._instance.addEndpoint(node, sourceEndpoint, { 310 | anchor: anchor, 311 | uuid: node.getAttribute("id") + "-" + ports[i] 312 | }); 313 | } else { 314 | endpoint = this._instance.addEndpoint(node, targetEndpoint, { 315 | anchor: anchor, 316 | uuid: node.getAttribute("id") + "-" + ports[i] 317 | }); 318 | } 319 | 320 | var labelAnchor = [-1.5, -0.1]; 321 | if (isSource) { 322 | labelAnchor = [1.5, -0.1]; 323 | } 324 | endpoint.setLabel({ location: labelAnchor, label: ports[i], cssClass: "endpointLabel" }); 325 | 326 | // Only show port lable on mouse over 327 | /* 328 | d3.selectAll(".endpointLabel").style("visibility", "hidden"); 329 | endpoint.bind("mouseover", function(source) { 330 | var label = source.getLabel(); 331 | $(source.canvas).next().css("visibility", "visible"); 332 | }); 333 | endpoint.bind("mouseout", function(source) { 334 | d3.selectAll(".endpointLabel").style("visibility", "hidden"); 335 | }); 336 | */ 337 | } 338 | }; 339 | 340 | Canvas.prototype._connectPorts = function(node1, port1, node2, port2) { 341 | var uuid_source = node1.getAttribute("id") + "-" + port1; 342 | var uuid_target = node2.getAttribute("id") + "-" + port2; 343 | this._instance.connect({ uuids: [uuid_source, uuid_target] }); 344 | }; 345 | 346 | // update the port values according to the run flow results 347 | Canvas.prototype._update = function() { 348 | var instance = this._instance; 349 | this._currentFlow._result.map(function(r){ 350 | var node = $("#"+ r.id); 351 | node.removeClass("node-fail"); 352 | node.removeClass("node-skip"); 353 | 354 | if ( r.status == "fail" ) { 355 | node.addClass("node-fail"); 356 | } 357 | 358 | if ( r.status == "skip" ) { 359 | node.addClass("node-skip"); 360 | } 361 | 362 | r.inputs.map(function(input){ 363 | var endpoint = instance.getEndpoint(r.id + "-" + input.name); 364 | endpoint.setLabel("" + input.value) 365 | }) 366 | r.outputs.map(function(output){ 367 | var endpoint = instance.getEndpoint(r.id + "-" + output.name); 368 | endpoint.setLabel("" + output.value) 369 | }) 370 | }) 371 | }; 372 | 373 | Canvas.prototype._showFlowSource = function() { 374 | var modal_id = "flow_source_modal"; 375 | var container_id = "flow_source_container"; 376 | var showSourceModal = Util.getModal(modal_id, "Flow Description", function(modal) { 377 | var body = modal.select(".modal-body"); 378 | body.attr("id", container_id).style("overflow", "auto"); 379 | }); 380 | 381 | $("#" + container_id).empty(); 382 | d3.select("#" + container_id).append("pre").attr("id", "flow_source_text"); 383 | var value = js_beautify(JSON.stringify(this._currentFlow.flow())); 384 | $("#flow_source_text").text(value); 385 | $("#" + modal_id).modal("show"); 386 | }; 387 | 388 | Canvas.prototype._clear = function() { 389 | this._instance.reset(); 390 | $("#" + FLOW_PANEL_ID).empty(); 391 | this._currentFlow.clear(); 392 | this._initInstance(); 393 | }; 394 | 395 | Canvas.prototype._newflow = function () { 396 | var me = this; 397 | this._clear(); 398 | 399 | var modal_id = "flow_new_modal"; 400 | var flowNewModal = Util.getModal(modal_id, "New Flow", function(modal) { 401 | var body = modal.select(".modal-body"); 402 | body.style("overflow", "auto"); 403 | var form = body.append("form"); 404 | var group1 = form.append("div").classed("form-group", true); 405 | group1.append("label").attr("for", "flowid").text("Flow ID"); 406 | group1.append("a").attr("href", "#").attr("id", "flowid").text("sampleId"); 407 | 408 | var group2 = form.append("div").classed("form-group", true); 409 | group2.append("label").attr("for", "flowname").text("Flow Name"); 410 | group2.append("a").attr("href", "#").attr("id", "flowname").text("sampleName"); 411 | 412 | var footer = modal.select(".modal-footer"); 413 | footer.append("button").attr("type", "button").classed("btn btn-default", true).attr("id", "new_flow_btn").text("New"); 414 | 415 | $("#flowid").text("xxx.xxx.xxx").editable(); 416 | $("#flowname").text("untitled").editable(); 417 | }); 418 | 419 | $("#new_flow_btn").click(function() { 420 | $("#flow_new_modal").modal("hide"); 421 | me._currentFlow = new Flow($("#flowid").text(), $("#flowname").text()); 422 | me._titleSpan.text(me._currentFlow.flow().name); 423 | }); 424 | $("#flow_new_modal").modal("show"); 425 | }; 426 | 427 | Canvas.prototype._save = function() { 428 | this._currentFlow.save(); 429 | }; 430 | 431 | Canvas.prototype._load = function() { 432 | var me = this; 433 | //load existing flows 434 | $.get("/flows", function(data) { 435 | //console.log(data); 436 | var modal_id = "flow_load_modal"; 437 | var container_id = "flow_load_container"; 438 | var flowLoadModal = Util.getModal(modal_id, "Load Flow", function(modal) { 439 | var body = modal.select(".modal-body"); 440 | body.attr("id", container_id).style("overflow", "auto"); 441 | }); 442 | 443 | $("#" + container_id).empty(); 444 | var flowItems = d3.select("#" + container_id).append("ul").selectAll("li").data(data).enter().append("li").append("a").text(function(d) { 445 | return d.id + ":" + d.name; 446 | }).on("click", function(d) { 447 | $("#" + modal_id).modal("hide"); 448 | me._loadflow(d); 449 | }); 450 | 451 | $("#" + modal_id).modal("show"); 452 | }); 453 | }; 454 | 455 | Canvas.prototype._loadflow = function(flow) { 456 | var me = this; 457 | this._clear(); 458 | this._currentFlow = new Flow(flow.id, flow.name); 459 | me._titleSpan.text(me._currentFlow.flow().name); 460 | 461 | flow.nodes.map(function(node) { 462 | me._currentFlow.addnode(node); 463 | var anode = me._drawNode(node); 464 | }); 465 | 466 | flow.links.map(function(link) { 467 | source = link.source.split(":"); 468 | target = link.target.split(":"); 469 | me._currentFlow.connect(source[0], target[0], source[1], target[1]); 470 | me._connectPorts($("#" + [source[0]])[0], source[1], $("#" + [target[0]])[0], target[1]); 471 | }); 472 | }; 473 | 474 | Canvas.prototype._removeNode = function(){ 475 | this._instance.remove(this._selectedNode.nodeId); 476 | d3.select(".glyphicon-remove-circle").style("visibility", "hidden"); 477 | //TODO : remove the node and links from current flow 478 | this._currentFlow.removenode(this._selectedNode); 479 | //TODO : clear the inspector as well 480 | }; 481 | 482 | return Canvas; 483 | }); -------------------------------------------------------------------------------- /src/static/js/components/flowInspector.js: -------------------------------------------------------------------------------- 1 | define(["util"], function(Util) { 2 | var Inspector = function(rootId) { 3 | this._rootId = rootId; 4 | this._update = undefined; 5 | this._updateOb = undefined; 6 | }; 7 | 8 | Inspector.prototype.render = function() { 9 | var root = d3.select("#" + this._rootId); 10 | var panel = Util.addPanel(root, "Inspector"); 11 | this._body = panel.select(".panel-body").attr("id", "InspectorBody"); 12 | } 13 | 14 | Inspector.prototype.onNotify = function(callback, observer) { 15 | this._update = callback; 16 | this._updateOb = observer; 17 | } 18 | 19 | Inspector.prototype.showNodeDetails = function(node, flow) { 20 | $("#InspectorBody").empty(); 21 | var inspector = this; 22 | 23 | var table = this._body.append("table").classed("table table-bordered table-condensed", true); 24 | var tbody = table.append("tbody"); 25 | 26 | var row_id = tbody.append("tr"); 27 | row_id.append("th").text("ID"); 28 | row_id.append("td").text(node.nodeId); 29 | 30 | var row_spec_id = tbody.append("tr"); 31 | row_spec_id.append("th").text("Specification ID"); 32 | row_spec_id.append("td").text(node.id); 33 | 34 | var row_spec_title = tbody.append("tr"); 35 | row_spec_title.append("th").text("Title"); 36 | row_spec_title.append("td").text(node.title); 37 | 38 | //Input Ports 39 | var i = 0, 40 | length = node.port.input.length; 41 | for (; i < length; i++) { 42 | var portName = node.port.input[i].name; 43 | var row_input_port = tbody.append("tr"); 44 | row_input_port.append("th").text("In Port : " + portName); 45 | var data = {}; 46 | data.id = node.nodeId; 47 | data.port = portName; 48 | 49 | var sourcePort = flow.findSourcePort(node.nodeId, portName); 50 | 51 | if (sourcePort) { 52 | var source_port_value = flow.getRunResult(sourcePort.id, sourcePort.port); 53 | var soruce_port_result_cell = row_input_port.append("td").append("div"); 54 | 55 | soruce_port_result_cell.append("p").style("margin", "0px").text("From:" + sourcePort.id + ":" + sourcePort.port); 56 | 57 | if (source_port_value !== undefined) { 58 | soruce_port_result_cell.append("br").style("margin", "0px"); 59 | var value = source_port_value; 60 | soruce_port_result_cell.append("pre").style("margin", "0px").text("Value : \n" + value); 61 | } 62 | 63 | } else { 64 | var port_input = row_input_port.append("td").append("input").datum(data) 65 | .on("change", function(d) { 66 | flow.setPortValue(d.id, d.port, d3.select(this).property("value")); 67 | }); 68 | 69 | //TODO: get node Spec 70 | var port_value = flow.getPortValue(data.id, data.port, undefined); 71 | if (port_value) { 72 | port_input.property("value", port_value); 73 | } 74 | } 75 | } 76 | 77 | //Output Ports 78 | i = 0, length = node.port.output.length; 79 | for (; i < length; i++) { 80 | var portName = node.port.output[i].name; 81 | var result = flow.getRunResult(node.nodeId, portName); 82 | var row_output_port = tbody.append("tr"); 83 | 84 | row_output_port.append("th").text("Out Port : " + portName); 85 | 86 | var targetPort = flow.findTargetPort(node.nodeId, portName); 87 | 88 | var out_result_cell = row_output_port.append("td").style("max-width", "200px").append("div"); 89 | if (targetPort) { 90 | out_result_cell.append("p").style("margin", "0px").text("To:" + targetPort.id + ":" + targetPort.port); 91 | } 92 | 93 | if (result !== undefined) { 94 | out_result_cell.append("br").style("margin", "0px"); 95 | 96 | var value = result; 97 | out_result_cell.append("pre").style("margin", "0px").text("Value : \n" + value); 98 | } 99 | } 100 | 101 | var row_flow_status = tbody.append("tr"); 102 | row_flow_status.append("th").text("Status"); 103 | row_flow_status.append("td").text(flow.status(node.nodeId)); 104 | 105 | var row_flow_error = tbody.append("tr"); 106 | row_flow_error.append("th").text("Error"); 107 | row_flow_error.append("td").text(flow.error(node.nodeId)); 108 | 109 | 110 | var row_action = tbody.append("tr"); 111 | row_action.append("th").text("Action"); 112 | var action_content = row_action.append("td"); 113 | action_content.append("button").datum({ id: node.nodeId }).text("Run").classed("btn btn-xs btn-primary", true) 114 | .on("click", function(d) { 115 | var progressBar = action_content.append('img').attr("src", "img/animated-progress.gif") 116 | .attr("height", "30").attr("width", "30"); 117 | function handleFlowRunResult(data) { 118 | inspector.showNodeDetails(node, flow); 119 | inspector.notify(); 120 | // TODO : update the flow to show the running result on the flow 121 | } 122 | 123 | function handleFlowRunFailure(data) { 124 | // TODO : handler flow failure 125 | inspector.showNodeDetails(node, flow); 126 | inspector.notify(); 127 | } 128 | 129 | flow.setEndNode(d.id); 130 | flow.run(handleFlowRunResult,handleFlowRunFailure); 131 | }); 132 | }; 133 | 134 | Inspector.prototype.notify = function() { 135 | this._update.apply(this._updateOb); 136 | } 137 | 138 | return Inspector; 139 | }); 140 | -------------------------------------------------------------------------------- /src/static/js/components/nodeCodePanel.js: -------------------------------------------------------------------------------- 1 | define(["util", "model/flow"], function(Util, Flow) { 2 | var Panel = function(rootId, propertyPanel) { 3 | this._rootId = rootId; 4 | this._editor = undefined; 5 | this._propertyPanel = propertyPanel; 6 | this._currentNode = undefined; 7 | this._titleSpan = undefined; 8 | }; 9 | 10 | Panel.prototype.connectListPanel = function(listPanel) { 11 | this._listPanel = listPanel; 12 | }; 13 | 14 | Panel.prototype.render = function() { 15 | $("#" + this._rootId).empty(); 16 | var root = d3.select("#" + this._rootId); 17 | var panel = Util.addPanel(root, "Node Code"); 18 | var me = this; 19 | this._body = panel.select(".panel-body").attr("id", "NodeCodeBody"); 20 | this._body.append("textarea").classed("form-control", true).attr("id", "NodeCodeEditor"); 21 | 22 | var heading = root.select(".panel-heading"); 23 | this._titleSpan = heading.append("span").classed("label label-primary", true).style("margin-left", "5px"); 24 | heading.append("br"); 25 | 26 | heading.append("button").classed("glyphicon glyphicon-plus-sign flowbutton", true).on("click", function() { 27 | me._listPanel.addNode(); 28 | }); 29 | 30 | heading.append("button").classed("glyphicon glyphicon-floppy-save flowbutton", true).on("click", function() { 31 | me._save(); 32 | }); 33 | 34 | heading.append("button").classed("glyphicon glyphicon-trash flowbutton", true).on("click", function() { 35 | me._listPanel.deleteNode(me._currentNode); 36 | }); 37 | 38 | heading.append("button").classed("glyphicon glyphicon-glass flowbutton", true).on("click", function() { 39 | me._test(); 40 | }); 41 | 42 | this._editor = CodeMirror.fromTextArea(document.getElementById("NodeCodeEditor"), { 43 | lineNumbers: true, 44 | indentUnit: 4, 45 | mode: "python", 46 | theme: "icecoder" 47 | }); 48 | 49 | this._editor.on("change", function(cm, change) { 50 | var code = cm.getValue(); 51 | me._validate(code); 52 | }); 53 | }; 54 | 55 | Panel.prototype.text = function(data) { 56 | if (data == undefined) { 57 | return this._editor.getValue(); 58 | } 59 | this._editor.setValue(data); 60 | }; 61 | 62 | Panel.prototype.update = function(node) { 63 | this._currentNode = node; 64 | this._titleSpan.text(node.id() + " : " + node.title()); 65 | this.text(node.func()); 66 | }; 67 | 68 | Panel.prototype._save = function() { 69 | this._currentNode.func(this._editor.getValue()); 70 | this._currentNode.save(); 71 | }; 72 | 73 | Panel.prototype._validate = function(code) { 74 | var lines = code.trim().split("\n"); 75 | var func_declare_line = lines[0]; 76 | var func_pattern = /def(\s)+func\(([\w|\W]*,?)*\):/g; 77 | var is_valid_function = func_pattern.test(func_declare_line); 78 | var node = this._currentNode; 79 | 80 | // TODO: show error in case the function is not a valid python function 81 | // Consider http://esprima.org/index.html 82 | 83 | if (is_valid_function) { 84 | //TODO : validate singature 85 | var start_pos = func_declare_line.indexOf("("); 86 | var end_pos = func_declare_line.indexOf(")"); 87 | var parameter = func_declare_line.substr(start_pos + 1, end_pos - start_pos - 1); 88 | parameters = parameter.split(","); 89 | this._currentNode.port().input = []; 90 | 91 | if (node.port().input.length == 0) { 92 | for (i = 0; i < parameters.length; i++) { 93 | var port = { "name": parameters[i].trim(), "type": "String", "order": i }; 94 | node.port().input.push(port); 95 | } 96 | } else { 97 | for (i = 0; i < node.port().input.length; i++) { 98 | if (parameters[i]) { 99 | node.port().input[i].name = parameters[i]; 100 | } 101 | } 102 | } 103 | 104 | for (var i = 0; i < lines.length; i++) { 105 | var line = lines[i].trim(); 106 | if (line.startsWith(":params:")) { 107 | // do nothing now, use parameter in func def 108 | } else if (line.startsWith(":ptypes:")) { 109 | var types = line.substring(8).split(","); 110 | for (var j = 0; j < node.port().input.length; j++) { 111 | if (types.length > j) { 112 | node.port().input[j].type = types[j].trim(); 113 | } 114 | } 115 | 116 | } else if (line.startsWith(":returns:")) { 117 | var rets = line.substring(9).split(","); 118 | //Update existing output name 119 | for (var j = 0; j < node.port().output.length; j++) { 120 | if (rets.length > j) { 121 | node.port().output[j].name = rets[j].trim(); 122 | } 123 | } 124 | 125 | // handling newly added outputs 126 | if (node.port().output.length < rets.length) { 127 | for (var j = rets.length - node.port().output.length; j < rets.length; j++) { 128 | var port = { "name": rets[j].trim(), "type": "String" }; 129 | node.port().output.push(port); 130 | } 131 | } 132 | } else if (line.startsWith(":rtype:")) { 133 | var rtypes = line.substring(7).split(","); 134 | for (var j = 0; j < node.port().output.length; j++) { 135 | if (rtypes.length > j) { 136 | node.port().output[j].type = rtypes[j].trim(); 137 | } 138 | } 139 | } 140 | } 141 | 142 | this._propertyPanel.update(node); 143 | } 144 | 145 | /* 146 | for (var i =0;i< lines.length;i++) { 147 | var line = lines[i].trim(); 148 | if ( line.startsWith(":params:") ) { 149 | // do nothing now, use parameter in func def 150 | } else if ( line.startsWith(":ptypes:") ) { 151 | var types = line.substring(7).split(","); 152 | for (j = 0; j < node.port().input.length; i++) { 153 | if ( types.length > j ) { 154 | node.port().input[j].type = types[j].trim(); 155 | } 156 | } 157 | 158 | } else if ( line.startsWith(":returns:") ) { 159 | 160 | } else if ( line.startsWith(":rtype:") ) { 161 | 162 | } 163 | } 164 | */ 165 | }; 166 | 167 | Panel.prototype._test = function() { 168 | var uuid = new Date().getTime(); 169 | var node_test_modal_id = "node-test-modal"; 170 | var node_test_modal_title = "Test Node"; 171 | var node_test_button = "node-test-button" 172 | var me = this; 173 | var node = this._currentNode; 174 | 175 | var testModal = Util.getModal(node_test_modal_id, node_test_modal_title, function(modal) { 176 | var footer = modal.select(".modal-footer"); 177 | 178 | footer.append("button").attr("type", "button").classed("btn btn-default", true).attr("id", node_test_button).text("Test"); 179 | }); 180 | 181 | var body = testModal.select(".modal-body"); 182 | // Reinit the body every time to make a clean test 183 | // refer to http://collaboradev.com/2014/03/18/d3-and-jquery-interoperability/ 184 | $(body.node()).empty(); 185 | 186 | var form = body.append("form"); 187 | var group = form.selectAll("div").data(node.port().input).enter().append("div").classed("form-group", true); 188 | group.append("label").text(function(d) { 189 | return d.name; 190 | }).style("margin-right", "5px"); 191 | inputs = group.append("input").classed("node_inputs", true); 192 | var outputs = body.append("div"); 193 | output_result = outputs.append("div").attr("visibility", "hidden").attr("id", "node_output_result").style("overflow", "auto"); 194 | 195 | // deregister existing handlers 196 | $("#" + node_test_button).unbind("click"); 197 | $("#" + node_test_button).click(function() { 198 | var inputs = d3.selectAll(".node_inputs"); 199 | var output_result = d3.select("#node_output_result"); 200 | 201 | // RUN Test 202 | var ports = []; 203 | inputs.each(function(d) { 204 | var o = {}; 205 | o.name = d.name.trim(); 206 | o.value = d3.select(this).property("value"); 207 | ports.push(o); 208 | }) 209 | var test_flow = new Flow("nodetestflow" + uuid, "nodetestflow"); 210 | var test_node = {}; 211 | test_node.id = "testnodeid" + uuid; 212 | test_node.spec_id = node.id(); 213 | test_node.name = node.title(); 214 | test_node.ports = ports; 215 | test_node.is_end = 1; 216 | test_flow.addnode(test_node); 217 | test_flow.run(function(data) { 218 | if (data[0].status == "fail") { 219 | output_result.text(data[0].error).attr("visibility", "visible").classed("alert alert-warning", true).classed("alert-success", false); 220 | } else { 221 | //Handling result here 222 | output_result.text(js_beautify(JSON.stringify(data[0].outputs))).attr("visibility", "visible").classed("alert alert-success", true).classed("alert-warning", false); 223 | } 224 | }, function(data) { 225 | //handling error here 226 | output_result.text(js_beautify(JSON.stringify(data))).attr("visibility", "visible").classed("alert alert-warning", true).classed("alert-success", false); 227 | }); 228 | }); 229 | 230 | $("#" + node_test_modal_id).modal('show'); 231 | }; 232 | 233 | return Panel; 234 | }); -------------------------------------------------------------------------------- /src/static/js/components/nodeListPanel.js: -------------------------------------------------------------------------------- 1 | define(["model/node","util"], function(Node, Util) { 2 | var Panel = function(rootId, data, codePanel, propertyPanel) { 3 | this._rootId = rootId; 4 | this._nodes = []; 5 | for ( d in data ) { 6 | this._nodes.push(new Node(data[d])); 7 | } 8 | 9 | this._codePanel = codePanel; 10 | this._propertyPanel = propertyPanel; 11 | }; 12 | 13 | Panel.prototype.render = function() { 14 | $("#" + this._rootId).empty(); 15 | 16 | var root = d3.select("#" + this._rootId); 17 | var panel = Util.addPanel(root, "Nodes"); 18 | this._body = panel.select(".panel-body").attr("id", "NodeListBody"); 19 | var me = this; 20 | 21 | var nodes = this._nodes; 22 | 23 | var nodeList = this._body.append("ul"); 24 | var nodeItems = nodeList.selectAll("li").data(nodes).enter().append("li").append("div"); 25 | 26 | nodeItems.append("a").text(function(d) { 27 | return d.id(); 28 | }).on("click", function(d) { 29 | me._loadNode(d); 30 | }); 31 | }; 32 | 33 | Panel.prototype._loadNode = function(node) { 34 | this._codePanel.update(node); 35 | // remove this as the property panel render will be triggerd by code panel 36 | //this._propertyPanel.update(node); 37 | }; 38 | 39 | Panel.prototype.addNode = function() { 40 | var add_modal_id = "add-node-modal"; 41 | var add_modal_titel = "Add New Node"; 42 | var add_node_id = "add_node_id"; 43 | var add_node_name = "add_node_name"; 44 | var add_node_button = "add_node_button"; 45 | var me = this; 46 | 47 | var add_modal = Util.getModal(add_modal_id, add_modal_titel, function(modal) { 48 | var body = modal.select(".modal-body"); 49 | var footer = modal.select(".modal-footer"); 50 | 51 | var form = body.append("form"); 52 | var group1 = form.append("div").classed("form-group",true); 53 | group1.append("label").attr("for",add_node_id).text("Node ID").style("margin-right","5px");; 54 | group1.append("a").attr("href","#").attr("id",add_node_id); 55 | 56 | var group2 = form.append("div").classed("form-group",true); 57 | group2.append("label").attr("for",add_node_name).text("Node Title").style("margin-right","5px"); 58 | group2.append("a").attr("href","#").attr("id",add_node_name); 59 | 60 | $("#" + add_node_id).text("xxx.xxx.xxx").editable(); 61 | $("#" + add_node_name).text("untitled").editable(); 62 | 63 | footer.append("button").attr("type","button").classed("btn btn-default",true).attr("id",add_node_button).text("New"); 64 | }); 65 | 66 | $("#" + add_node_button).unbind("click"); 67 | $("#" + add_node_button).click(function() { 68 | $("#" + add_modal_id).modal("hide"); 69 | var node = new Node(); 70 | node.id($("#" + add_node_id).text()); 71 | node.title($("#" + add_node_name).text()); 72 | me._nodes.push(node); 73 | me.render(); 74 | me._loadNode(node); 75 | }); 76 | 77 | $("#"+ add_modal_id ).modal('show'); 78 | }; 79 | 80 | Panel.prototype.deleteNode = function(node) { 81 | var delete_modal_id = "delete-node-modal"; 82 | var delete_modal_titel = "Delete Node"; 83 | var delete_node_button = "delete-node-button" 84 | var me = this; 85 | 86 | var deletModal = Util.getModal(delete_modal_id, delete_modal_titel, function(modal) { 87 | var body = modal.select(".modal-body"); 88 | var footer = modal.select(".modal-footer"); 89 | 90 | body.text("Do you want to delete this node " + node.id() + " ?"); 91 | 92 | footer.append("button").attr("type","button").classed("btn btn-default",true).attr("id",delete_node_button).text("Delete"); 93 | }); 94 | 95 | $("#" + delete_node_button).click(function() { 96 | $("#" + delete_modal_id).modal("hide"); 97 | // TODO : delete the node; 98 | node.delete(); 99 | 100 | me._nodes = jQuery.grep(me._nodes, function(value) { 101 | return value !== node; 102 | }); 103 | me.render(); 104 | me._loadNode(me._nodes[0]); 105 | }); 106 | 107 | $("#"+ delete_modal_id ).modal('show'); 108 | }; 109 | 110 | return Panel; 111 | }); 112 | -------------------------------------------------------------------------------- /src/static/js/components/nodePropertyPanel.js: -------------------------------------------------------------------------------- 1 | define(["util"], function(Util) { 2 | var Panel = function(rootId) { 3 | this._rootId = rootId; 4 | }; 5 | 6 | Panel.prototype.render = function() { 7 | $("#" + this._rootId).empty(); 8 | var root = d3.select("#" + this._rootId); 9 | var panel = Util.addPanel(root, "Node Property"); 10 | this._body = panel.select(".panel-body").attr("id", "NodePropertyBody"); 11 | }; 12 | 13 | Panel.prototype.update = function(node) { 14 | $("#NodePropertyBody").empty(); 15 | var me = this; 16 | 17 | var table = this._body.append("table").classed("table table-bordered table-condensed", true); 18 | var tbody = table.append("tbody"); 19 | 20 | var row_id = tbody.append("tr"); 21 | row_id.append("td").text("ID"); 22 | row_id.append("td").text(node.id()); 23 | 24 | var row_title = tbody.append("tr"); 25 | row_title.append("td").text("Title"); 26 | row_title.append("td").text(node.title()); 27 | 28 | var row_port = tbody.append("tr"); 29 | row_port.append("td").text("Ports"); 30 | var portDiv = row_port.append("td").append("div"); 31 | this._updatePorts(node.port(), portDiv); 32 | }; 33 | 34 | Panel.prototype._updatePorts = function(ports, div) { 35 | var table = div.append("table").classed("table table-bordered table-condensed", true); 36 | var header = ["in/out","type","name","default"]; 37 | table.append("thead").selectAll("tr").data(header).enter().append("th").text(function(d){ 38 | return d; 39 | }); 40 | 41 | var tbody = table.append("tbody"); 42 | 43 | var content = []; 44 | 45 | ports.input.map(function(p) { 46 | var item = {}; 47 | item.direction = "input"; 48 | $.extend(item,p); 49 | content.push(item); 50 | }); 51 | 52 | ports.output.map(function(p) { 53 | var item = {}; 54 | item.direction = "output"; 55 | $.extend(item,p); 56 | content.push(item); 57 | }); 58 | 59 | var port_row = tbody.selectAll("tr").data(content).enter().append("tr"); 60 | 61 | port_row.append("td").text(function(d){ 62 | return d.direction; 63 | }); 64 | 65 | port_row.append("td").text(function(d){ 66 | return d.type; 67 | }); 68 | port_row.append("td").text(function(d){ 69 | return d.name; 70 | }); 71 | port_row.append("td").text(function(d){ 72 | if ( d.default == undefined ) { 73 | return ""; 74 | } 75 | return d.default; 76 | }); 77 | }; 78 | 79 | return Panel; 80 | }); 81 | -------------------------------------------------------------------------------- /src/static/js/components/treeview.js: -------------------------------------------------------------------------------- 1 | define(["util"], function(Util) { 2 | var TreeView = function(rootId, nodeSpec) { 3 | this._rootId = rootId; 4 | this._nodeSpec = nodeSpec; 5 | }; 6 | 7 | TreeView.prototype.render = function() { 8 | var root = d3.select("#" + this._rootId); 9 | var panel = Util.addPanel(root, "Nodes"); 10 | 11 | panel.select(".panel-body").append("div").attr("id", "tree"); 12 | var tree = _list2tree(this._nodeSpec); 13 | _rendor("tree", tree); 14 | } 15 | 16 | function _list2tree(nodes) { 17 | var tree = [{ 18 | "title": "pyflow", 19 | "id": "pyflow", 20 | "children": [] 21 | }]; 22 | 23 | for (var node in nodes) { 24 | _insertNode(tree[0], nodes[node]); 25 | } 26 | 27 | return tree; 28 | } 29 | 30 | function _insertNode(node, item) { 31 | if (item === undefined) { 32 | return; 33 | } 34 | var id = item.id; 35 | var ids = id.split("."); 36 | 37 | var i = 0, 38 | length = ids.length; 39 | var checkId = ids[0]; 40 | 41 | for (; i < length; i++) { 42 | if (checkId === node.id) { 43 | if (i === length - 1) { //Is last Layer /Leaf 44 | node.content = item; //override duplicated node, should never run into this now 45 | return; 46 | 47 | } else if (i === length - 2) { 48 | if (node.children === undefined) { 49 | node.children = []; 50 | } 51 | var newChild = {}; 52 | newChild.id = checkId + "." + ids[i + 1]; 53 | newChild.title = ids[i + 1]; 54 | newChild.content = item; 55 | node.children.push(newChild); 56 | return; 57 | 58 | } else { 59 | if (node.children === undefined) { 60 | node.children = []; 61 | } 62 | var newChild = {}; 63 | newChild.id = checkId + "." + ids[i + 1]; 64 | newChild.title = ids[i + 1]; 65 | _appendChild(node, newChild); 66 | node.children.forEach(function(entry) { 67 | _insertNode(entry, item); 68 | }) 69 | } 70 | } 71 | checkId = checkId + "." + ids[i + 1]; 72 | } 73 | 74 | } 75 | 76 | function _appendChild(tree, child) { 77 | var i = 0, 78 | length = tree.children.length; 79 | for (; i < length; i++) { 80 | if (tree.children[i].id === child.id) { 81 | return; 82 | } 83 | } 84 | tree.children.push(child); 85 | } 86 | 87 | function _rendor(rootId, data) { 88 | var root = d3.select("#" + rootId); 89 | 90 | var level1 = root.append("ul").classed("pyflowtree", true).selectAll("li").data(data).enter().append("li").classed("branch", true); 91 | level1.append("a").attr("href", "#").text(function(d) { 92 | return d.title; 93 | }); 94 | 95 | var depth = 4, 96 | i = 0, 97 | previousLevel = level1; 98 | 99 | for (; i < depth; i++) { 100 | var currentLevel = previousLevel.append("ul").classed("tree", true).selectAll("li").classed("level-" + i, true).data(function(d) { 101 | if (d.children !== undefined) { 102 | return d.children; 103 | } else { 104 | return [] 105 | } 106 | }).enter().append("li").style("display", function() { 107 | if (i > 0) { 108 | return "None"; 109 | } else { 110 | return "list-item"; 111 | } 112 | }); 113 | 114 | currentLevel.each(function(d) { 115 | if (d.children == undefined || d.children.length == 0) { 116 | d3.select(this).classed("leaf", tree); 117 | } else { 118 | d3.select(this).classed("branch", tree); 119 | } 120 | }) 121 | 122 | currentLevel.append("a").attr("href", "#").text(function(d) { 123 | if (d.children && d.children.length > 0) { 124 | return d.title + " +"; 125 | } 126 | return d.title; 127 | }).attr("specId", function(d) { 128 | if (d.id) { 129 | return d.id; 130 | } 131 | }); 132 | 133 | previousLevel = currentLevel; 134 | } 135 | 136 | $('.tree li').on('click', function(e) { 137 | var children = $(this).find('> ul > li'); 138 | if (children.is(":visible")) 139 | children.hide('fast'); 140 | else children.show('fast'); 141 | e.stopPropagation(); 142 | }); 143 | 144 | //Disable drag on non-leaf node 145 | $('.branch').attr('draggable', 'false').on('dragstart', function(ev) { 146 | if (ev.target.parentNode.className != "leaf") { 147 | ev.stopPropagation(); 148 | return false; 149 | } 150 | }); 151 | 152 | //Handle drag and drop 153 | $('.leaf').attr('draggable', 'true').on('dragstart', function(ev) { 154 | ev.originalEvent.dataTransfer.setData('text', $(ev.target).attr("specId")); 155 | }); 156 | }; 157 | 158 | return TreeView; 159 | }); 160 | -------------------------------------------------------------------------------- /src/static/js/flow.js: -------------------------------------------------------------------------------- 1 | define(["comp/treeview", "comp/flowCanvas", "comp/flowInspector", "util"], function(TreeView, FlowCanvas, FlowInspector, Util) { 2 | var Flow = {}; 3 | var canvas = undefined; 4 | 5 | Flow.render = function() { 6 | $("#mainUI").empty(); 7 | 8 | var rootUI = d3.select("#mainUI").append("div").classed("row", true); 9 | var nodeTree = rootUI.append("div").classed("col-md-3", true).attr("id", "flowTree"); 10 | 11 | var flowUI = rootUI.append("div").classed("col-md-8", true).attr("id", "flowUI"); 12 | var flowCanvas = flowUI.append("div").classed("row", true).attr("id", "flowCanvas"); 13 | var flowInspector = flowUI.append("div").classed("row", true).attr("id", "flowInspector"); 14 | 15 | // Init Tree 16 | $.get("/nodes", function(data) { 17 | var nodeTreeSpecification = data; 18 | var treeView = new TreeView("flowTree", nodeTreeSpecification) 19 | treeView.render(); 20 | 21 | // Init Flow Inspector 22 | var inspector = new FlowInspector("flowInspector"); 23 | inspector.render(); 24 | 25 | canvas = new FlowCanvas("flowCanvas", nodeTreeSpecification, inspector ); 26 | canvas.render(); 27 | }); 28 | } 29 | 30 | return Flow; 31 | }); 32 | -------------------------------------------------------------------------------- /src/static/js/model/flow.js: -------------------------------------------------------------------------------- 1 | define([], function() { 2 | var Flow = function(id, name) { 3 | this._flow = {}; 4 | this._flow.id = id; 5 | this._flow.name = name; 6 | this._flow.nodes = []; 7 | this._flow.links = []; 8 | this._result = undefined; // store flow running results 9 | }; 10 | 11 | Flow.prototype.flow = function() { 12 | return this._flow; 13 | }; 14 | 15 | Flow.prototype.nodes = function() { 16 | return this._flow.nodes; 17 | }; 18 | 19 | Flow.prototype.addnode = function(node) { 20 | this._flow.nodes.push(node); 21 | }; 22 | 23 | Flow.prototype.removenode= function(node) { 24 | this._flow.nodes = this._flow.nodes.filter(function(e){ 25 | return e.id !== node.nodeId; 26 | }); 27 | this._flow.links = this._flow.links.filter(function(e){ 28 | if ( e.source.includes(node.id) || e.target.includes(node.id) ) { 29 | return false; 30 | } 31 | return true; 32 | }); 33 | }; 34 | 35 | Flow.prototype.connections = function() { 36 | var connections = []; 37 | this._flow.links.map(function(link) { 38 | source = link.source.split(":"); 39 | target = link.target.split(":"); 40 | connections.push({ 41 | "sourceId": source[0], 42 | "targetId": target[0], 43 | "sourcePort": source[1], 44 | "targetPort": target[1] 45 | }); 46 | }) 47 | 48 | return connections; 49 | }; 50 | 51 | 52 | 53 | Flow.prototype._findConnection = function(sourceId, targetId, sourcePort, targetPort) { 54 | var i = 0, 55 | index = -1, 56 | length = this._flow.links.length; 57 | 58 | for (; i < length; i++) { 59 | var link = this._flow.links[i]; 60 | if (link.source == sourceId + ":" + sourcePort && 61 | link.target == targetId + ":" + targetPort) { 62 | index = i; 63 | } 64 | } 65 | 66 | return index; 67 | }; 68 | 69 | Flow.prototype.link = function(link) { 70 | source = link.source.split(":"); 71 | target = link.target.split(":"); 72 | this.connect(source[0], target[0], source[1], target[1]); 73 | }; 74 | 75 | Flow.prototype.connect = function(sourceId, targetId, sourcePort, targetPort) { 76 | var index = this._findConnection(sourceId, targetId, sourcePort, targetPort); 77 | if (index == -1) { 78 | this._flow.links.push({ 79 | "source": sourceId + ":" + sourcePort, 80 | "target": targetId + ":" + targetPort 81 | }); 82 | } 83 | 84 | // TODO: in case it is move from one target to another, need remove the previous link 85 | }; 86 | 87 | Flow.prototype.disconnect = function(sourceId, targetId, sourcePort, targetPort) { 88 | var index = this._findConnection(sourceId, targetId, sourcePort, targetPort); 89 | 90 | if (index > -1) { 91 | this._flow.links.splice(index, 1); 92 | } 93 | }; 94 | 95 | Flow.prototype.clear = function() { 96 | this._flow.nodes = []; 97 | this._flow.links = []; 98 | this._result = undefined; 99 | }; 100 | 101 | Flow.prototype.findSourcePort = function(nodeId, port) { 102 | var connections = this.connections(); 103 | var i = 0, 104 | length = connections.length; 105 | 106 | for (; i < length; i++) { 107 | if (connections[i].targetId === nodeId && connections[i].targetPort === port) { 108 | return { 109 | "id": connections[i].sourceId, 110 | "port": connections[i].sourcePort 111 | } 112 | } 113 | } 114 | }; 115 | 116 | Flow.prototype.findTargetPort = function(nodeId, port) { 117 | var connections = this.connections(); 118 | var i = 0, 119 | length = connections.length; 120 | 121 | for (; i < length; i++) { 122 | if (connections[i].sourceId === nodeId && connections[i].sourcePort === port) { 123 | return { 124 | "id": connections[i].targetId, 125 | "port": connections[i].targetPort 126 | } 127 | } 128 | } 129 | }; 130 | 131 | Flow.prototype.setPortValue = function(nodeId, portName, value) { 132 | var nodes = this._flow.nodes; 133 | var i = 0, 134 | length = nodes.length; 135 | var node = undefined, 136 | port = undefined; 137 | for (; i < length; i++) { 138 | if (nodes[i].id === nodeId) { 139 | node = nodes[i]; 140 | break; 141 | } 142 | } 143 | 144 | if (node === undefined) { 145 | return; 146 | } 147 | 148 | i = 0, length = node.ports.length; 149 | for (; i < length; i++) { 150 | if (node.ports[i].name === portName) { 151 | port = node.ports[i]; 152 | break; 153 | } 154 | } 155 | 156 | if (port === undefined) { 157 | node.ports.push({ 158 | "name": portName, 159 | "value": value 160 | }) 161 | } else { 162 | port.value = value; 163 | } 164 | }; 165 | 166 | Flow.prototype.getPortValue = function(nodeId, portName, nodeSpec) { 167 | var nodes = this._flow.nodes; 168 | var i = 0, 169 | length = nodes.length; 170 | var node = undefined, 171 | port = undefined; 172 | for (; i < length; i++) { 173 | if (nodes[i].id === nodeId) { 174 | node = nodes[i]; 175 | break; 176 | } 177 | } 178 | 179 | if (node === undefined) { 180 | return; 181 | } 182 | 183 | i = 0, length = node.ports.length; 184 | for (; i < length; i++) { 185 | if (node.ports[i].name === portName) { 186 | port = node.ports[i]; 187 | break; 188 | } 189 | } 190 | 191 | if (port) { 192 | return port.value; 193 | } 194 | 195 | //Port value not set, use default one when the nodeSpec is provided 196 | if (nodeSpec) { 197 | i = 0, length = nodeSpec.port.input.length; 198 | for (; i < length; i++) { 199 | if (nodeSpec.port.input[i].name === portName && nodeSpec.port.input[i].default !== undefined) { 200 | var default_var = nodeSpec.port.input[i].default; 201 | if (default_var === "\n") { 202 | default_var = "\\n"; //TODO : add generict logic for this. 203 | } 204 | return default_var; 205 | } 206 | } 207 | } 208 | }; 209 | 210 | Flow.prototype.setEndNode = function(nodeId) { 211 | var nodes = this._flow.nodes; 212 | var i = 0, 213 | length = nodes.length; 214 | for (; i < length; i++) { 215 | if (nodes[i].id === nodeId) { 216 | nodes[i].is_end = 1; 217 | } else { 218 | delete nodes[i]["is_end"]; 219 | } 220 | } 221 | }; 222 | 223 | Flow.prototype.run = function(cb, fail) { 224 | var me = this; 225 | 226 | $.ajax({ 227 | url: '/runflow', 228 | contentType: 'application/json', 229 | type: 'POST', 230 | data: JSON.stringify(this._flow), 231 | dataType: 'json' 232 | }).done(function(data) { 233 | me._result = data; 234 | cb(data); 235 | }).fail(function(data) { 236 | // TODO : error handling here 237 | console.log(data); 238 | fail(data); 239 | }); 240 | }; 241 | 242 | Flow.prototype.set = function(id, name) { 243 | this._flow.id = id; 244 | this._flow.name = name; 245 | }; 246 | 247 | Flow.prototype._update = function() { 248 | this._flow.nodes.map(function(node) { 249 | var el = $("#" + node.id); 250 | var position = el.position(); 251 | node.ui.x = position.left + "px"; 252 | node.ui.y = position.top + "px"; 253 | }); 254 | }; 255 | 256 | Flow.prototype.save = function() { 257 | this._update(); 258 | 259 | $.ajax({ 260 | url: '/flows', 261 | contentType: 'application/json', 262 | type: 'POST', 263 | data: JSON.stringify(this._flow), 264 | dataType: 'json' 265 | }).done(function(data) { 266 | console.log(data); 267 | }); 268 | }; 269 | 270 | Flow.prototype.status = function(nodeId) { 271 | if (this._result === undefined) { 272 | return ""; 273 | } 274 | 275 | var i = 0, 276 | length = this._result.length; 277 | for (; i < length; i++) { 278 | if (this._result[i].id === nodeId) { 279 | return this._result[i].status 280 | } 281 | } 282 | 283 | return ""; 284 | }; 285 | 286 | Flow.prototype.error = function(nodeId) { 287 | if (this._result === undefined) { 288 | return ""; 289 | } 290 | 291 | var i = 0, 292 | length = this._result.length; 293 | for (; i < length; i++) { 294 | if (this._result[i].id === nodeId) { 295 | return this._result[i].error 296 | } 297 | } 298 | 299 | return ""; 300 | }; 301 | 302 | Flow.prototype.getRunResult = function(nodeId, port) { 303 | if (this._result === undefined) { 304 | return undefined; 305 | } 306 | 307 | var i = 0, 308 | j = 0, 309 | length = this._result.length; 310 | for (; i < length; i++) { 311 | if (this._result[i].id === nodeId) { 312 | for (; j < this._result[i].outputs.length; j++) { 313 | var aport = this._result[i].outputs[j]; 314 | if (aport.name === port) { 315 | return aport.value; 316 | } 317 | } 318 | } 319 | } 320 | 321 | return undefined; 322 | }; 323 | 324 | return Flow; 325 | }); -------------------------------------------------------------------------------- /src/static/js/model/node.js: -------------------------------------------------------------------------------- 1 | define([], function() { 2 | var default_func = "def func():\n\t return None"; 3 | 4 | var Node = function(node) { 5 | if (node === undefined) { 6 | this._node = {}; 7 | this._node.id = undefined; 8 | this._node.func = default_func; 9 | // Default port is empty input 10 | // and default out for output 11 | this._node.port = {}; 12 | this._node.port.input = []; 13 | this._node.port.output = []; 14 | this._node.port.output.push({"name":"out","type":"String"}); 15 | this._node.title = ""; 16 | } else { 17 | this._node = node 18 | } 19 | }; 20 | 21 | Node.prototype.node = function(node) { 22 | if (node === undefined) { 23 | return this._node; 24 | } 25 | this._node = node 26 | }; 27 | 28 | Node.prototype.id = function(id) { 29 | if (id === undefined) { 30 | return this._node.id; 31 | } 32 | this._node.id = id 33 | }; 34 | 35 | Node.prototype.port = function(port) { 36 | if (port === undefined) { 37 | return this._node.port; 38 | } 39 | this._node.port = port 40 | }; 41 | 42 | Node.prototype.title = function(title) { 43 | if (title === undefined) { 44 | return this._node.title; 45 | } 46 | this._node.title = title 47 | }; 48 | 49 | Node.prototype.func = function(func) { 50 | if (func === undefined) { 51 | return this._node.func; 52 | } 53 | this._node.func = func 54 | }; 55 | 56 | Node.prototype.save = function() { 57 | $.ajax({ 58 | url: '/nodes', 59 | contentType: 'application/json', 60 | type: 'POST', 61 | data: JSON.stringify(this._node), 62 | dataType: 'json' 63 | }).done(function(data) { 64 | console.log(data); 65 | }); 66 | }; 67 | 68 | Node.prototype.delete = function() { 69 | $.ajax({ 70 | url: '/nodes/' + this.id(), 71 | contentType: 'application/json', 72 | type: 'DELETE', 73 | dataType: 'json' 74 | }).done(function(data) { 75 | console.log(data); 76 | }); 77 | }; 78 | 79 | return Node; 80 | }); -------------------------------------------------------------------------------- /src/static/js/model/repo.js: -------------------------------------------------------------------------------- 1 | define([], function() { 2 | var Repo = function() { 3 | }; 4 | 5 | Repo.prototype.load = function(path, onComplete) { 6 | var data = {}; 7 | data.path = path; 8 | $.ajax({ 9 | url: '/loadrepo', 10 | contentType: 'application/json', 11 | type: 'POST', 12 | data: JSON.stringify(data), 13 | dataType: 'json' 14 | }).done(function(data) { 15 | console.log(data); 16 | onComplete.apply(); 17 | }); 18 | } 19 | 20 | Repo.prototype.dump = function(path) { 21 | var data = {}; 22 | data.path = path; 23 | $.ajax({ 24 | url: '/dumprepo', 25 | contentType: 'application/json', 26 | type: 'POST', 27 | data: JSON.stringify(data), 28 | dataType: 'json' 29 | }).done(function(data) { 30 | console.log(data); 31 | }); 32 | } 33 | 34 | return Repo; 35 | }); -------------------------------------------------------------------------------- /src/static/js/node.js: -------------------------------------------------------------------------------- 1 | define(["comp/nodeListPanel","comp/nodeCodePanel","comp/nodePropertyPanel", "util"], function(NodeListPanel,NodeCodePanel,NodePropertyPanel, Util) { 2 | var Node = {}; 3 | Node.render = function() { 4 | $("#mainUI").empty(); 5 | 6 | var rootUI = d3.select("#mainUI").append("div").classed("row", true); 7 | var nodeListPanel = rootUI.append("div").classed("col-md-3", true).attr("id", "nodeListPanel"); 8 | 9 | var nodeEditorPanel = rootUI.append("div").classed("col-md-8", true).attr("id", "nodeEditorPanel"); 10 | var nodeCodePanel = nodeEditorPanel.append("div").classed("row", true).attr("id", "nodeCodePanel"); 11 | var nodePropertyPanel = nodeEditorPanel.append("div").classed("row", true).attr("id", "nodePropertyPanel"); 12 | 13 | var nodeProperty = new NodePropertyPanel("nodePropertyPanel"); 14 | nodeProperty.render(); 15 | 16 | var nodeCode = new NodeCodePanel("nodeCodePanel", nodeProperty); 17 | nodeCode.render(); 18 | 19 | 20 | $.get("/nodes", function(data) { 21 | var nodeList = new NodeListPanel("nodeListPanel",data,nodeCode,nodeProperty); 22 | nodeList.render(); 23 | nodeCode.connectListPanel(nodeList); 24 | }); 25 | }; 26 | return Node; 27 | }); -------------------------------------------------------------------------------- /src/static/js/util.js: -------------------------------------------------------------------------------- 1 | define([], function() { 2 | var Util = {}; 3 | 4 | Util.addPanel = function(parent, title) { 5 | var panel = parent.append("div").classed("panel panel-default", true); 6 | if (title != undefined) { 7 | panel.append("div").classed("panel-heading", true).text(title); 8 | } 9 | panel.append("div").classed("panel-body", true); 10 | 11 | return panel 12 | }; 13 | 14 | Util.getModal = function(id, title, init) { 15 | if ( Util[id] !== undefined ) { 16 | return Util[id]; 17 | } 18 | 19 | var modal = d3.select("body").append("div"); 20 | Util[id] = modal; 21 | modal.attr("id",id); 22 | 23 | modal.classed("modal fade",true).attr("tableindex","-1").attr("role","dialog"); 24 | var dialog = modal.append("div").classed("modal-dialog",true).attr("role","document"); 25 | var content = dialog.append("div").classed("modal-content",true); 26 | var header = content.append("div").classed("modal-header",true); 27 | header.append("button").classed("close",true).attr("data-dismiss","modal").attr("aria-label","Close").append("span").attr("aria-hidden",true).text("x"); 28 | header.append("h4").classed("modal-title",true).text(title); 29 | var body = content.append("div").classed("modal-body",true); 30 | var footer = content.append("div").classed("modal-footer",true); 31 | footer.append("button").attr("type","button").classed("btn btn-default",true).attr("data-dismiss","modal").text("Close"); 32 | 33 | // call init function to initialize the modal dialog 34 | init.call(Util, modal); 35 | 36 | return modal; 37 | }; 38 | 39 | return Util; 40 | }); 41 | -------------------------------------------------------------------------------- /src/static/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pyflow", 3 | "version": "1.0.0", 4 | "description": "a python application of flow base programing concept implementation", 5 | "main": "app.js", 6 | "repository": "https://github.com/gangtao/pyflow", 7 | "author": "Gang Tao ", 8 | "license": "MIT", 9 | "dependencies": { 10 | "jquery": "^3.1.1", 11 | "codemirror": "^5.22.2", 12 | "bootstrap": "^3.3.7", 13 | "x-editable": "^1.5.1", 14 | "d3": "^4.12.2", 15 | "js-beautify": "*", 16 | "jsplumb": "^2.3.2", 17 | "requirejs": "~2.1.22" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/static/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | abbrev@1: 6 | version "1.1.1" 7 | resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" 8 | 9 | bluebird@^3.0.5: 10 | version "3.5.1" 11 | resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" 12 | 13 | bootstrap@^3.3.7: 14 | version "3.3.7" 15 | resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.3.7.tgz#5a389394549f23330875a3b150656574f8a9eb71" 16 | 17 | codemirror@^5.22.2: 18 | version "5.38.0" 19 | resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-5.38.0.tgz#26a9551446e51dbdde36aabe60f72469724fd332" 20 | 21 | commander@2, commander@^2.9.0: 22 | version "2.15.1" 23 | resolved "https://registry.yarnpkg.com/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f" 24 | 25 | config-chain@~1.1.5: 26 | version "1.1.11" 27 | resolved "https://registry.yarnpkg.com/config-chain/-/config-chain-1.1.11.tgz#aba09747dfbe4c3e70e766a6e41586e1859fc6f2" 28 | dependencies: 29 | ini "^1.3.4" 30 | proto-list "~1.2.1" 31 | 32 | d3-array@1, d3-array@1.2.1, d3-array@^1.2.0: 33 | version "1.2.1" 34 | resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.1.tgz#d1ca33de2f6ac31efadb8e050a021d7e2396d5dc" 35 | 36 | d3-axis@1.0.8: 37 | version "1.0.8" 38 | resolved "https://registry.yarnpkg.com/d3-axis/-/d3-axis-1.0.8.tgz#31a705a0b535e65759de14173a31933137f18efa" 39 | 40 | d3-brush@1.0.4: 41 | version "1.0.4" 42 | resolved "https://registry.yarnpkg.com/d3-brush/-/d3-brush-1.0.4.tgz#00c2f238019f24f6c0a194a26d41a1530ffe7bc4" 43 | dependencies: 44 | d3-dispatch "1" 45 | d3-drag "1" 46 | d3-interpolate "1" 47 | d3-selection "1" 48 | d3-transition "1" 49 | 50 | d3-chord@1.0.4: 51 | version "1.0.4" 52 | resolved "https://registry.yarnpkg.com/d3-chord/-/d3-chord-1.0.4.tgz#7dec4f0ba886f713fe111c45f763414f6f74ca2c" 53 | dependencies: 54 | d3-array "1" 55 | d3-path "1" 56 | 57 | d3-collection@1, d3-collection@1.0.4: 58 | version "1.0.4" 59 | resolved "https://registry.yarnpkg.com/d3-collection/-/d3-collection-1.0.4.tgz#342dfd12837c90974f33f1cc0a785aea570dcdc2" 60 | 61 | d3-color@1: 62 | version "1.2.0" 63 | resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-1.2.0.tgz#d1ea19db5859c86854586276ec892cf93148459a" 64 | 65 | d3-color@1.0.3: 66 | version "1.0.3" 67 | resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-1.0.3.tgz#bc7643fca8e53a8347e2fbdaffa236796b58509b" 68 | 69 | d3-dispatch@1, d3-dispatch@1.0.3: 70 | version "1.0.3" 71 | resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-1.0.3.tgz#46e1491eaa9b58c358fce5be4e8bed626e7871f8" 72 | 73 | d3-drag@1, d3-drag@1.2.1: 74 | version "1.2.1" 75 | resolved "https://registry.yarnpkg.com/d3-drag/-/d3-drag-1.2.1.tgz#df8dd4c502fb490fc7462046a8ad98a5c479282d" 76 | dependencies: 77 | d3-dispatch "1" 78 | d3-selection "1" 79 | 80 | d3-dsv@1, d3-dsv@1.0.8: 81 | version "1.0.8" 82 | resolved "https://registry.yarnpkg.com/d3-dsv/-/d3-dsv-1.0.8.tgz#907e240d57b386618dc56468bacfe76bf19764ae" 83 | dependencies: 84 | commander "2" 85 | iconv-lite "0.4" 86 | rw "1" 87 | 88 | d3-ease@1, d3-ease@1.0.3: 89 | version "1.0.3" 90 | resolved "https://registry.yarnpkg.com/d3-ease/-/d3-ease-1.0.3.tgz#68bfbc349338a380c44d8acc4fbc3304aa2d8c0e" 91 | 92 | d3-force@1.1.0: 93 | version "1.1.0" 94 | resolved "https://registry.yarnpkg.com/d3-force/-/d3-force-1.1.0.tgz#cebf3c694f1078fcc3d4daf8e567b2fbd70d4ea3" 95 | dependencies: 96 | d3-collection "1" 97 | d3-dispatch "1" 98 | d3-quadtree "1" 99 | d3-timer "1" 100 | 101 | d3-format@1: 102 | version "1.3.0" 103 | resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-1.3.0.tgz#a3ac44269a2011cdb87c7b5693040c18cddfff11" 104 | 105 | d3-format@1.2.2: 106 | version "1.2.2" 107 | resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-1.2.2.tgz#1a39c479c8a57fe5051b2e67a3bee27061a74e7a" 108 | 109 | d3-geo@1.9.1: 110 | version "1.9.1" 111 | resolved "https://registry.yarnpkg.com/d3-geo/-/d3-geo-1.9.1.tgz#157e3b0f917379d0f73bebfff3be537f49fa7356" 112 | dependencies: 113 | d3-array "1" 114 | 115 | d3-hierarchy@1.1.5: 116 | version "1.1.5" 117 | resolved "https://registry.yarnpkg.com/d3-hierarchy/-/d3-hierarchy-1.1.5.tgz#a1c845c42f84a206bcf1c01c01098ea4ddaa7a26" 118 | 119 | d3-interpolate@1: 120 | version "1.2.0" 121 | resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-1.2.0.tgz#40d81bd8e959ff021c5ea7545bc79b8d22331c41" 122 | dependencies: 123 | d3-color "1" 124 | 125 | d3-interpolate@1.1.6: 126 | version "1.1.6" 127 | resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-1.1.6.tgz#2cf395ae2381804df08aa1bf766b7f97b5f68fb6" 128 | dependencies: 129 | d3-color "1" 130 | 131 | d3-path@1, d3-path@1.0.5: 132 | version "1.0.5" 133 | resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-1.0.5.tgz#241eb1849bd9e9e8021c0d0a799f8a0e8e441764" 134 | 135 | d3-polygon@1.0.3: 136 | version "1.0.3" 137 | resolved "https://registry.yarnpkg.com/d3-polygon/-/d3-polygon-1.0.3.tgz#16888e9026460933f2b179652ad378224d382c62" 138 | 139 | d3-quadtree@1, d3-quadtree@1.0.3: 140 | version "1.0.3" 141 | resolved "https://registry.yarnpkg.com/d3-quadtree/-/d3-quadtree-1.0.3.tgz#ac7987e3e23fe805a990f28e1b50d38fcb822438" 142 | 143 | d3-queue@3.0.7: 144 | version "3.0.7" 145 | resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618" 146 | 147 | d3-random@1.1.0: 148 | version "1.1.0" 149 | resolved "https://registry.yarnpkg.com/d3-random/-/d3-random-1.1.0.tgz#6642e506c6fa3a648595d2b2469788a8d12529d3" 150 | 151 | d3-request@1.0.6: 152 | version "1.0.6" 153 | resolved "https://registry.yarnpkg.com/d3-request/-/d3-request-1.0.6.tgz#a1044a9ef4ec28c824171c9379fae6d79474b19f" 154 | dependencies: 155 | d3-collection "1" 156 | d3-dispatch "1" 157 | d3-dsv "1" 158 | xmlhttprequest "1" 159 | 160 | d3-scale@1.0.7: 161 | version "1.0.7" 162 | resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-1.0.7.tgz#fa90324b3ea8a776422bd0472afab0b252a0945d" 163 | dependencies: 164 | d3-array "^1.2.0" 165 | d3-collection "1" 166 | d3-color "1" 167 | d3-format "1" 168 | d3-interpolate "1" 169 | d3-time "1" 170 | d3-time-format "2" 171 | 172 | d3-selection@1, d3-selection@1.3.0, d3-selection@^1.1.0: 173 | version "1.3.0" 174 | resolved "https://registry.yarnpkg.com/d3-selection/-/d3-selection-1.3.0.tgz#d53772382d3dc4f7507bfb28bcd2d6aed2a0ad6d" 175 | 176 | d3-shape@1.2.0: 177 | version "1.2.0" 178 | resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-1.2.0.tgz#45d01538f064bafd05ea3d6d2cb748fd8c41f777" 179 | dependencies: 180 | d3-path "1" 181 | 182 | d3-time-format@2, d3-time-format@2.1.1: 183 | version "2.1.1" 184 | resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-2.1.1.tgz#85b7cdfbc9ffca187f14d3c456ffda268081bb31" 185 | dependencies: 186 | d3-time "1" 187 | 188 | d3-time@1, d3-time@1.0.8: 189 | version "1.0.8" 190 | resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-1.0.8.tgz#dbd2d6007bf416fe67a76d17947b784bffea1e84" 191 | 192 | d3-timer@1, d3-timer@1.0.7: 193 | version "1.0.7" 194 | resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-1.0.7.tgz#df9650ca587f6c96607ff4e60cc38229e8dd8531" 195 | 196 | d3-transition@1, d3-transition@1.1.1: 197 | version "1.1.1" 198 | resolved "https://registry.yarnpkg.com/d3-transition/-/d3-transition-1.1.1.tgz#d8ef89c3b848735b060e54a39b32aaebaa421039" 199 | dependencies: 200 | d3-color "1" 201 | d3-dispatch "1" 202 | d3-ease "1" 203 | d3-interpolate "1" 204 | d3-selection "^1.1.0" 205 | d3-timer "1" 206 | 207 | d3-voronoi@1.1.2: 208 | version "1.1.2" 209 | resolved "https://registry.yarnpkg.com/d3-voronoi/-/d3-voronoi-1.1.2.tgz#1687667e8f13a2d158c80c1480c5a29cb0d8973c" 210 | 211 | d3-zoom@1.7.1: 212 | version "1.7.1" 213 | resolved "https://registry.yarnpkg.com/d3-zoom/-/d3-zoom-1.7.1.tgz#02f43b3c3e2db54f364582d7e4a236ccc5506b63" 214 | dependencies: 215 | d3-dispatch "1" 216 | d3-drag "1" 217 | d3-interpolate "1" 218 | d3-selection "1" 219 | d3-transition "1" 220 | 221 | d3@^4.12.2: 222 | version "4.13.0" 223 | resolved "https://registry.yarnpkg.com/d3/-/d3-4.13.0.tgz#ab236ff8cf0cfc27a81e69bf2fb7518bc9b4f33d" 224 | dependencies: 225 | d3-array "1.2.1" 226 | d3-axis "1.0.8" 227 | d3-brush "1.0.4" 228 | d3-chord "1.0.4" 229 | d3-collection "1.0.4" 230 | d3-color "1.0.3" 231 | d3-dispatch "1.0.3" 232 | d3-drag "1.2.1" 233 | d3-dsv "1.0.8" 234 | d3-ease "1.0.3" 235 | d3-force "1.1.0" 236 | d3-format "1.2.2" 237 | d3-geo "1.9.1" 238 | d3-hierarchy "1.1.5" 239 | d3-interpolate "1.1.6" 240 | d3-path "1.0.5" 241 | d3-polygon "1.0.3" 242 | d3-quadtree "1.0.3" 243 | d3-queue "3.0.7" 244 | d3-random "1.1.0" 245 | d3-request "1.0.6" 246 | d3-scale "1.0.7" 247 | d3-selection "1.3.0" 248 | d3-shape "1.2.0" 249 | d3-time "1.0.8" 250 | d3-time-format "2.1.1" 251 | d3-timer "1.0.7" 252 | d3-transition "1.1.1" 253 | d3-voronoi "1.1.2" 254 | d3-zoom "1.7.1" 255 | 256 | editorconfig@^0.13.2: 257 | version "0.13.3" 258 | resolved "https://registry.yarnpkg.com/editorconfig/-/editorconfig-0.13.3.tgz#e5219e587951d60958fd94ea9a9a008cdeff1b34" 259 | dependencies: 260 | bluebird "^3.0.5" 261 | commander "^2.9.0" 262 | lru-cache "^3.2.0" 263 | semver "^5.1.0" 264 | sigmund "^1.0.1" 265 | 266 | iconv-lite@0.4: 267 | version "0.4.23" 268 | resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" 269 | dependencies: 270 | safer-buffer ">= 2.1.2 < 3" 271 | 272 | ini@^1.3.4: 273 | version "1.3.5" 274 | resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" 275 | 276 | jquery@^3.1.1: 277 | version "3.3.1" 278 | resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.3.1.tgz#958ce29e81c9790f31be7792df5d4d95fc57fbca" 279 | 280 | js-beautify@*: 281 | version "1.7.5" 282 | resolved "https://registry.yarnpkg.com/js-beautify/-/js-beautify-1.7.5.tgz#69d9651ef60dbb649f65527b53674950138a7919" 283 | dependencies: 284 | config-chain "~1.1.5" 285 | editorconfig "^0.13.2" 286 | mkdirp "~0.5.0" 287 | nopt "~3.0.1" 288 | 289 | jsplumb@^2.3.2: 290 | version "2.7.1" 291 | resolved "https://registry.yarnpkg.com/jsplumb/-/jsplumb-2.7.1.tgz#d9c46ad660252d22960954e1e9e8f03c17fe1267" 292 | 293 | lru-cache@^3.2.0: 294 | version "3.2.0" 295 | resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-3.2.0.tgz#71789b3b7f5399bec8565dda38aa30d2a097efee" 296 | dependencies: 297 | pseudomap "^1.0.1" 298 | 299 | minimist@0.0.8: 300 | version "0.0.8" 301 | resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" 302 | 303 | mkdirp@~0.5.0: 304 | version "0.5.1" 305 | resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" 306 | dependencies: 307 | minimist "0.0.8" 308 | 309 | nopt@~3.0.1: 310 | version "3.0.6" 311 | resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" 312 | dependencies: 313 | abbrev "1" 314 | 315 | proto-list@~1.2.1: 316 | version "1.2.4" 317 | resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" 318 | 319 | pseudomap@^1.0.1: 320 | version "1.0.2" 321 | resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" 322 | 323 | requirejs@~2.1.22: 324 | version "2.1.22" 325 | resolved "https://registry.yarnpkg.com/requirejs/-/requirejs-2.1.22.tgz#dd78fd2d34180c0d62c724b5b8aebc0664e0366f" 326 | 327 | rw@1: 328 | version "1.3.3" 329 | resolved "https://registry.yarnpkg.com/rw/-/rw-1.3.3.tgz#3f862dfa91ab766b14885ef4d01124bfda074fb4" 330 | 331 | "safer-buffer@>= 2.1.2 < 3": 332 | version "2.1.2" 333 | resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" 334 | 335 | semver@^5.1.0: 336 | version "5.5.0" 337 | resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" 338 | 339 | sigmund@^1.0.1: 340 | version "1.0.1" 341 | resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590" 342 | 343 | x-editable@^1.5.1: 344 | version "1.5.1" 345 | resolved "https://registry.yarnpkg.com/x-editable/-/x-editable-1.5.1.tgz#2edbb8911ef2c5d61f63f06b0cf020be0fcc5849" 346 | 347 | xmlhttprequest@1: 348 | version "1.8.0" 349 | resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc" 350 | -------------------------------------------------------------------------------- /tests/rest/test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - config: 3 | - testset: "Pyflow API Test" 4 | 5 | - test: 6 | - name: "List nodes as a tree" 7 | - url: "/nodestree" 8 | 9 | - test: 10 | - name: "List nodes" 11 | - url: "/nodes" 12 | 13 | - test: 14 | - name: "List node by id" 15 | - url: "/nodes/pyflow.source.cli" 16 | 17 | - test: 18 | - name: "Add node" 19 | - url: "/nodes" 20 | - method: "POST" 21 | - body: '{"title":"node_plus","id":"my.test.node.plus","port":{"input":[{"name":"port1","order":0},{"name":"port2","order":1}]},"func":"def func(x,y): return x + y "}' 22 | - headers: {"Content-Type": "application/json", "charset" : "UTF-8"} 23 | 24 | - test: 25 | - name: "Delete node by id" 26 | - url: "/nodes/my.test.node.plus" 27 | - method: "DELETE" 28 | 29 | - test: 30 | - name: "List flows" 31 | - url: "/flows" 32 | 33 | - test: 34 | - name: "Create a flow" 35 | - url: "/flows" 36 | - method: "POST" 37 | - body: '{"id":"flowbuilder.gen.dir","name":"BuilderSample","nodes":[{"id":"node1419317316499","spec_id":"pyflow.source.cli","name":"cli","ports":[{"name":"command","value":"dir"}],"is_end":1}],"links":[]}' 38 | - headers: {"Content-Type": "application/json", "charset" : "UTF-8"} 39 | 40 | - test: 41 | - name: "Run a flow" 42 | - url: "/runflow" 43 | - method: "POST" 44 | - body: '{"id":"flowbuilder.gen.dir","name":"BuilderSample","nodes":[{"id":"node1419317316499","spec_id":"pyflow.source.cli","name":"cli","ports":[{"name":"command","value":"dir"}],"is_end":1}],"links":[]}' 45 | - headers: {"Content-Type": "application/json", "charset" : "UTF-8"} 46 | 47 | - test: 48 | - name: "Dumps repository" 49 | - url: "/dumprepo" 50 | - method: "POST" 51 | - body: '{"path":"./repo.json"}' 52 | - headers: {"Content-Type": "application/json", "charset" : "UTF-8"} 53 | 54 | - test: 55 | - name: "Load repository" 56 | - url: "/loadrepo" 57 | - method: "POST" 58 | - body: '{"path":"./repo.json"}' 59 | - headers: {"Content-Type": "application/json", "charset" : "UTF-8"} 60 | 61 | - test: 62 | - name: "Get supported types" 63 | - url: "/ports/types" 64 | - headers: {"Content-Type": "application/json", "charset" : "UTF-8"} -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangtao/pyflow/e491254dce9caec493b44b17c4873cae1eece694/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/test_flow.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sys 4 | import json 5 | import time 6 | 7 | sys.path.append('../../src') 8 | 9 | import fbp 10 | from fbp.port import Inport, Outport 11 | from fbp.node import Node 12 | from fbp.flow import Flow 13 | 14 | 15 | class TestFBPPort(unittest.TestCase): 16 | 17 | @classmethod 18 | def setUpClass(cls): 19 | pass 20 | 21 | @classmethod 22 | def tearDownClass(cls): 23 | pass 24 | 25 | def setUp(self): 26 | pass 27 | 28 | def tearDown(self): 29 | pass 30 | 31 | def test_flow_2nodes(self): 32 | aflow = Flow("my.test.aflow", "A flow test") 33 | 34 | spec = ''' 35 | { 36 | "title" : "node_plus", 37 | "id" : "my.test.node.plus", 38 | "port" : { 39 | "input" : [ 40 | {"name" : "port1", "order" : 0}, 41 | {"name" : "port2", "order" : 1} 42 | ] 43 | }, 44 | "func" : "def func(x,y): 45 | return x + y" 46 | } 47 | ''' 48 | 49 | spec_obj = json.loads(spec, strict=False) 50 | 51 | node1 = Node("my.test.node1", "node1", spec_obj) 52 | node2 = Node("my.test.node2", "node2", spec_obj) 53 | 54 | aflow.add_node(node1) 55 | aflow.add_node(node2) 56 | 57 | aflow.link(node1.id, "out", node2.id, "port2") 58 | 59 | node1.set_inport_value("port1", "A") 60 | node1.set_inport_value("port2", "B") 61 | node2.set_inport_value("port1", "C") 62 | 63 | stats = aflow.run(node2) 64 | 65 | while not stats.check_stat(): 66 | time.sleep(0.1) 67 | 68 | result = stats.get_result_by_id("my.test.node2") 69 | self.assertEqual(result["outputs"][0]["value"], "CAB") 70 | 71 | def test_flow_3nodes(self): 72 | aflow = Flow("my.test.aflow", "A flow test") 73 | 74 | spec = ''' 75 | { 76 | "title" : "node_plus", 77 | "id" : "my.test.node.plus", 78 | "port" : { 79 | "input" : [ 80 | {"name" : "port1", "order" : 0}, 81 | {"name" : "port2", "order" : 1} 82 | ] 83 | }, 84 | "func" : "def func(x,y): 85 | return x + y" 86 | } 87 | ''' 88 | 89 | spec_obj = json.loads(spec, strict=False) 90 | 91 | node1 = Node("my.test.node1", "node1", spec_obj) 92 | node2 = Node("my.test.node2", "node1", spec_obj) 93 | node3 = Node("my.test.node3", "node3", spec_obj) 94 | 95 | aflow.add_node(node1) 96 | aflow.add_node(node2) 97 | aflow.add_node(node3) 98 | 99 | aflow.link(node1.id, "out", node3.id, "port1") 100 | aflow.link(node2.id, "out", node3.id, "port2") 101 | 102 | node1.set_inport_value("port1", "A") 103 | node1.set_inport_value("port2", "B") 104 | node2.set_inport_value("port1", "C") 105 | node2.set_inport_value("port2", "D") 106 | 107 | stats = aflow.run(node3) 108 | 109 | while not stats.check_stat(): 110 | time.sleep(0.1) 111 | 112 | result = stats.get_result_by_id("my.test.node3") 113 | self.assertEqual(result["outputs"][0]["value"], "ABCD") 114 | 115 | def test_flow_4nodes(self): 116 | aflow = Flow("my.test.aflow", "A flow test") 117 | 118 | spec = ''' 119 | { 120 | "title" : "node_plus", 121 | "id" : "my.test.node.plus", 122 | "port" : { 123 | "input" : [ 124 | {"name" : "port1", "order" : 0}, 125 | {"name" : "port2", "order" : 1} 126 | ] 127 | }, 128 | "func" : "def func(x,y): 129 | return x + y" 130 | } 131 | ''' 132 | 133 | spec_obj = json.loads(spec, strict=False) 134 | 135 | node1 = Node("my.test.node1", "node1", spec_obj) 136 | node2 = Node("my.test.node2", "node1", spec_obj) 137 | node3 = Node("my.test.node3", "node3", spec_obj) 138 | node4 = Node("my.test.node4", "node4", spec_obj) 139 | 140 | aflow.add_node(node1) 141 | aflow.add_node(node2) 142 | aflow.add_node(node3) 143 | aflow.add_node(node4) 144 | 145 | aflow.link(node1.id, "out", node3.id, "port1") 146 | aflow.link(node2.id, "out", node3.id, "port2") 147 | aflow.link(node2.id, "out", node4.id, "port1") 148 | 149 | node1.set_inport_value("port1", "A") 150 | node1.set_inport_value("port2", "B") 151 | node2.set_inport_value("port1", "C") 152 | node2.set_inport_value("port2", "D") 153 | node4.set_inport_value("port2", "E") 154 | 155 | stats3 = aflow.run(node3) 156 | stats4 = aflow.run(node4) 157 | 158 | while not stats3.check_stat() or not stats4.check_stat(): 159 | time.sleep(0.1) 160 | 161 | result3 = stats3.get_result_by_id("my.test.node3") 162 | self.assertEqual(result3["outputs"][0]["value"], "ABCD") 163 | result4 = stats4.get_result_by_id("my.test.node4") 164 | self.assertEqual(result4["outputs"][0]["value"], "CDE") 165 | 166 | 167 | if __name__ == '__main__': 168 | unittest.main() 169 | -------------------------------------------------------------------------------- /tests/unit/test_message.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sys,time 4 | 5 | sys.path.append('../../src') 6 | from message import Bus,Subscriber 7 | 8 | class TestFBPMessage(unittest.TestCase): 9 | 10 | @classmethod 11 | def setUpClass(cls): 12 | pass 13 | 14 | @classmethod 15 | def tearDownClass(cls): 16 | pass 17 | 18 | def setUp(self): 19 | pass 20 | 21 | def tearDown(self): 22 | pass 23 | 24 | def test_message(self): 25 | channel = Bus("message broker") 26 | suba = Subscriber("A") 27 | subb = Subscriber("B") 28 | subc = Subscriber("C") 29 | 30 | channel.subcribe(suba) 31 | channel.subcribe(subb) 32 | channel.subcribe(subc) 33 | 34 | channel.start() 35 | channel.publish("first message") 36 | channel.publish("second message") 37 | time.sleep(1) 38 | channel.unsubscribe(subc) 39 | time.sleep(1) 40 | channel.publish("third message") 41 | 42 | time.sleep(1) 43 | channel.stop() 44 | time.sleep(1) 45 | 46 | 47 | if __name__ == '__main__': 48 | unittest.main() 49 | -------------------------------------------------------------------------------- /tests/unit/test_node.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sys 4 | import json 5 | 6 | sys.path.append('../../src') 7 | 8 | from fbp.node import Node 9 | 10 | 11 | class TestFBPNode(unittest.TestCase): 12 | 13 | @classmethod 14 | def setUpClass(cls): 15 | pass 16 | 17 | @classmethod 18 | def tearDownClass(cls): 19 | pass 20 | 21 | def setUp(self): 22 | pass 23 | 24 | def tearDown(self): 25 | pass 26 | 27 | def test_node(self): 28 | spec = ''' 29 | { 30 | "title" : "node_plus", 31 | "id" : "my.test.node.plus", 32 | "port" : { 33 | "input" : [ 34 | {"name" : "port1", "order" : 0}, 35 | {"name" : "port2", "order" : 1} 36 | ] 37 | }, 38 | "func" : "def func(x,y): 39 | return x + y" 40 | } 41 | ''' 42 | 43 | spec_obj = json.loads(spec, strict=False) 44 | 45 | anode = Node("my.test.node1", "node1", spec_obj) 46 | 47 | anode.set_inport_value("port1", "x") 48 | anode.set_inport_value("port2", "y") 49 | anode.run() 50 | 51 | self.assertEqual(anode.get_outport_value(), "xy") 52 | 53 | anode.set_inport_value("port1", 1) 54 | anode.set_inport_value("port2", 2) 55 | anode.run() 56 | 57 | self.assertEqual(anode.get_outport_value(), 3) 58 | 59 | def test_node_default_value(self): 60 | spec = ''' 61 | { 62 | "title" : "node_plus", 63 | "id" : "my.test.node.plus", 64 | "port" : { 65 | "input" : [ 66 | {"name" : "port1", "order" : 0, "default" : "xyz"}, 67 | {"name" : "port2", "order" : 1} 68 | ] 69 | }, 70 | "func" : "def func(x,y): 71 | return x + y" 72 | } 73 | ''' 74 | 75 | spec_obj = json.loads(spec, strict=False) 76 | 77 | anode = Node("my.test.node1", "node1", spec_obj) 78 | 79 | anode.set_inport_value("port2", "y") 80 | anode.run() 81 | 82 | self.assertEqual(anode.get_outport_value(), "xyzy") 83 | 84 | def test_node_multiple_output(self): 85 | spec = ''' 86 | { 87 | "title" : "node_exchange", 88 | "id" : "my.test.node.exchange", 89 | "port" : { 90 | "input" : [ 91 | {"name" : "port1", "order" : 0}, 92 | {"name" : "port2", "order" : 1} 93 | ], 94 | "output" : [ 95 | {"name" : "out1" }, 96 | {"name" : "out2" } 97 | ] 98 | }, 99 | "func" : "def func(x,y): 100 | ret = {} 101 | ret[\\"out2\\"] = x ## Using ' or \\" here to avoid JSON decoding error 102 | ret[\\"out1\\"] = y 103 | return ret" 104 | } 105 | ''' 106 | 107 | spec_obj = json.loads(spec, strict=False) 108 | 109 | anode = Node("my.test.node2", "node2", spec_obj) 110 | 111 | anode.set_inport_value("port1", "goto2") 112 | anode.set_inport_value("port2", "goto1") 113 | anode.run() 114 | 115 | self.assertEqual(anode.get_outport_value("out1"), "goto1") 116 | self.assertEqual(anode.get_outport_value("out2"), "goto2") 117 | 118 | 119 | if __name__ == '__main__': 120 | unittest.main() 121 | -------------------------------------------------------------------------------- /tests/unit/test_port.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sys 4 | 5 | sys.path.append('../../src') 6 | 7 | from fbp.port import Port, Inport, Outport 8 | 9 | 10 | class TestFBPPort(unittest.TestCase): 11 | 12 | @classmethod 13 | def setUpClass(cls): 14 | pass 15 | 16 | @classmethod 17 | def tearDownClass(cls): 18 | pass 19 | 20 | def setUp(self): 21 | pass 22 | 23 | def tearDown(self): 24 | pass 25 | 26 | def test_port(self): 27 | print "test port" 28 | aport = Port("aport") 29 | self.assertEqual(aport.name, "aport") 30 | self.assertEqual(aport.type, "String") 31 | 32 | aport.value = "new value" 33 | self.assertEqual(aport.value, "new value") 34 | 35 | print aport 36 | 37 | def test_in_port(self): 38 | print "test in port" 39 | aport = Inport("aport") 40 | self.assertEqual(aport.name, "aport") 41 | self.assertEqual(aport.type, "String") 42 | self.assertEqual(aport.value, None) 43 | self.assertEqual(aport.is_required, False) 44 | self.assertEqual(aport.order, 0) 45 | 46 | aport.value = "new value" 47 | self.assertEqual(aport.value, "new value") 48 | 49 | print aport 50 | 51 | def test_port_link(self): 52 | print "test port link" 53 | aport = Inport("in_port") 54 | bport = Outport("out_port") 55 | 56 | bport.point_to(aport) 57 | bport.value = "new value" 58 | 59 | self.assertEqual(aport.value, "new value") 60 | self.assertEqual(bport.value, "new value") 61 | 62 | print aport 63 | print bport 64 | 65 | 66 | if __name__ == '__main__': 67 | unittest.main() 68 | -------------------------------------------------------------------------------- /tests/unit/test_repo.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sys 4 | import time 5 | 6 | sys.path.append('../../src') 7 | 8 | from fbp import create_node 9 | from fbp.flow import Flow 10 | import fbp.repository 11 | 12 | 13 | # CLI Node Definition 14 | cli_spec = ''' 15 | { 16 | "title" : "cli", 17 | "id" : "flow.cli", 18 | "port" : { 19 | "input" : [ 20 | {"name" : "command", "order" : 0} 21 | ] 22 | }, 23 | "func" : "def func(command):\n import shlex, subprocess\n ## This cli cannot hand code that refresh the screen like top\n args = shlex.split(command)\n p = subprocess.Popen(args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n outdata, errdata = p.communicate()\n if len(errdata) != 0:\n raise Exception('Failed to execut command % , error is {}'.format(command, errdata))\n return outdata" 24 | } 25 | ''' 26 | 27 | # Rest Http Get 28 | rest_spec = ''' 29 | { 30 | "title" : "rest", 31 | "id" : "flow.rest", 32 | "port" : { 33 | "input" : [ 34 | { 35 | "name" : "url", 36 | "order" : 0 37 | }, 38 | { 39 | "name" : "parameter", 40 | "order" : 1 41 | }, 42 | { 43 | "name" : "header", 44 | "order" : 2 45 | } 46 | ], 47 | "output" : [ 48 | {"name" : "text" }, 49 | {"name" : "json" }, 50 | {"name" : "encoding" } 51 | ] 52 | }, 53 | "func" : "def func(url, parameter=None, header=None):\n import requests\n paras = {}\n if parameter is not None:\n paras['params'] = parameter\n\n if header is not None:\n paras['headers'] = header\n\n r = requests.get(url, **paras)\n\n ret = {}\n ret['text'] = r.text\n ret['json'] = r.json()\n ret['encoding'] = r.encoding\n\n return ret" 54 | } 55 | ''' 56 | 57 | # Data Handling and Trasformation 58 | 59 | # Ln Breaker Node Definition 60 | 61 | line_breaker_spec = ''' 62 | { 63 | "title" : "line_breaker", 64 | "id" : "flow.line_breaker", 65 | "port" : { 66 | "input": [ 67 | { 68 | "name": "input", 69 | "order": 0 70 | }, 71 | { 72 | "name": "breaker", 73 | "order": 1, 74 | "default": "\\n" 75 | } 76 | ] 77 | }, 78 | "func" : "def func(input, breaker):\n import re\n print 'breaker is {}'.format(breaker)\n print 'input is {}'.format(input)\n lines = re.split(breaker, input)\n return lines" 79 | } 80 | ''' 81 | 82 | 83 | class TestFBPRepository(unittest.TestCase): 84 | 85 | @classmethod 86 | def setUpClass(cls): 87 | repository = fbp.repository() 88 | repository.register("nodespec", "flow.cli", cli_spec) 89 | repository.register("nodespec", "flow.rest", rest_spec) 90 | repository.register( 91 | "nodespec", "flow.line_breaker", line_breaker_spec) 92 | 93 | @classmethod 94 | def tearDownClass(cls): 95 | pass 96 | 97 | def setUp(self): 98 | pass 99 | 100 | def tearDown(self): 101 | pass 102 | 103 | #@unittest.skip("") 104 | def test_respository1(self): 105 | repo = fbp.repository() 106 | 107 | anode = create_node("flow.cli", "myflow.command", "command") 108 | bnode = create_node("flow.line_breaker", 109 | "myflow.breaker", "breaker") 110 | 111 | aflow = Flow("my.test.aflow", "A flow test") 112 | aflow.add_node(anode) 113 | aflow.add_node(bnode) 114 | 115 | aflow.link(anode.id, "out", bnode.id, "input") 116 | anode.set_inport_value("command", "iostat") 117 | 118 | stats = aflow.run(bnode) 119 | while not stats.check_stat(): 120 | time.sleep(0.1) 121 | 122 | # print bnode.get_outport_value() 123 | print repo.domains() 124 | 125 | def test_respository2(self): 126 | repo = fbp.repository() 127 | 128 | anode = create_node("flow.rest", "myflow.rest", "rest") 129 | 130 | aflow = Flow("my.test.aflow", "A flow test") 131 | aflow.add_node(anode) 132 | 133 | anode.set_inport_value("url", "https://api.github.com/events") 134 | 135 | stats = aflow.run(anode) 136 | while not stats.check_stat(): 137 | time.sleep(0.1) 138 | 139 | # print anode.get_outport_value('json') 140 | print repo.domains() 141 | 142 | #@unittest.skip("") 143 | def test_respository3(self): 144 | repo = fbp.repository() 145 | print repo.dumps("./repo.json") 146 | 147 | #@unittest.skip("") 148 | def test_respository4(self): 149 | repo = fbp.repository() 150 | repo.loads("./repo.json") 151 | 152 | anode = create_node("flow.rest", "myflow.rest", "rest") 153 | 154 | aflow = Flow("my.test.aflow", "A flow test") 155 | aflow.add_node(anode) 156 | 157 | anode.set_inport_value("url", "https://api.github.com/events") 158 | 159 | stats = aflow.run(anode) 160 | while not stats.check_stat(): 161 | time.sleep(0.1) 162 | 163 | # print anode.get_outport_value('json') 164 | 165 | 166 | if __name__ == '__main__': 167 | unittest.main() 168 | -------------------------------------------------------------------------------- /tests/unit/test_run.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | import json 4 | 5 | sys.path.append('../../src') 6 | 7 | from fbp import create_node, run_flow 8 | import fbp.repository 9 | 10 | # CLI Node Definition 11 | cli_spec = ''' 12 | { 13 | "title" : "cli", 14 | "id" : "flow.cli", 15 | "port" : { 16 | "input" : [ 17 | {"name" : "command", "order" : 0} 18 | ] 19 | }, 20 | "func" : "def func(command):\n import shlex, subprocess\n ## This cli cannot hand code that refresh the screen like top\n args = shlex.split(command)\n p = subprocess.Popen(args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n outdata, errdata = p.communicate()\n if len(errdata) != 0:\n raise Exception('Failed to execut command % , error is {}'.format(command, errdata))\n return outdata" 21 | } 22 | ''' 23 | 24 | # Rest Http Get 25 | rest_spec = ''' 26 | { 27 | "title" : "rest", 28 | "id" : "flow.rest", 29 | "port" : { 30 | "input" : [ 31 | { 32 | "name" : "url", 33 | "order" : 0 34 | }, 35 | { 36 | "name" : "parameter", 37 | "order" : 1 38 | }, 39 | { 40 | "name" : "header", 41 | "order" : 2 42 | } 43 | ], 44 | "output" : [ 45 | {"name" : "text" }, 46 | {"name" : "json" }, 47 | {"name" : "encoding" } 48 | ] 49 | }, 50 | "func" : "def func(url, parameter=None, header=None):\n import requests\n paras = {}\n if parameter is not None:\n paras['params'] = parameter\n\n if header is not None:\n paras['headers'] = header\n\n r = requests.get(url, **paras)\n\n ret = {}\n ret['text'] = r.text\n ret['json'] = r.json()\n ret['encoding'] = r.encoding\n\n return ret" 51 | } 52 | ''' 53 | 54 | # Data Handling and Trasformation 55 | 56 | # Ln Breaker Node Definition 57 | 58 | line_breaker_spec = ''' 59 | { 60 | "title" : "line_breaker", 61 | "id" : "flow.line_breaker", 62 | "port" : { 63 | "input": [ 64 | { 65 | "name": "input", 66 | "order": 0 67 | }, 68 | { 69 | "name": "breaker", 70 | "order": 1, 71 | "default": "\\n" 72 | } 73 | ] 74 | }, 75 | "func" : "def func(input, breaker):\n import re\n print 'breaker is {}'.format(breaker)\n print 'input is {}'.format(input)\n lines = re.split(breaker, input)\n return lines" 76 | } 77 | ''' 78 | 79 | 80 | class TestFBPRunner(unittest.TestCase): 81 | 82 | @classmethod 83 | def setUpClass(cls): 84 | repository = fbp.repository() 85 | repository.register("nodespec", "flow.cli", cli_spec) 86 | repository.register("nodespec", "flow.rest", rest_spec) 87 | repository.register( 88 | "nodespec", "flow.line_breaker", line_breaker_spec) 89 | 90 | @classmethod 91 | def tearDownClass(cls): 92 | pass 93 | 94 | def setUp(self): 95 | pass 96 | 97 | def tearDown(self): 98 | pass 99 | 100 | def test_run1(self): 101 | # A sample flow definition 102 | flow_spec = ''' 103 | { 104 | "id": "my.test.spec", 105 | "name": "test flow", 106 | "nodes": [ 107 | { 108 | "spec_id": "flow.cli", 109 | "id": "myflow.cli", 110 | "name": "cli", 111 | "ports": [ 112 | { 113 | "name": "command", 114 | "value": "iostat" 115 | } 116 | ] 117 | }, 118 | { 119 | "spec_id": "flow.line_breaker", 120 | "id": "myflow.line_breaker", 121 | "name": "line_breaker", 122 | "ports": [], 123 | "is_end": 1 124 | } 125 | ], 126 | "links": [ 127 | { 128 | "source": "myflow.cli:out", 129 | "target": "myflow.line_breaker:input" 130 | } 131 | ] 132 | } 133 | ''' 134 | 135 | print json.dumps(run_flow(flow_spec)) 136 | print "\n" 137 | 138 | def test_run2(self): 139 | # A sample flow definition 140 | flow_spec = '''{"id":"flowbuilder.gen","name":"BuilderSample","nodes":[{"id":"node1419317316499","spec_id":"flow.cli","name":"cli","ports":[{"name":"command","value":"iostat"}],"is_end":1}],"links":[]} 141 | ''' 142 | 143 | print json.dumps(run_flow(flow_spec)) 144 | print "\n" 145 | 146 | def test_run3(self): 147 | # A sample flow definition with failure command 148 | flow_spec = '''{"id":"flowbuilder.gen","name":"BuilderSample","nodes":[{"id":"node1419317316499","spec_id":"flow.cli","name":"cli","ports":[{"name":"command","value":"ls"}],"is_end":1}],"links":[]} 149 | ''' 150 | 151 | print json.dumps(run_flow(flow_spec)) 152 | print "\n" 153 | 154 | 155 | if __name__ == '__main__': 156 | unittest.main() 157 | --------------------------------------------------------------------------------