├── .editorconfig ├── .gitignore ├── .gitlab-ci.yml ├── .pre-commit-config.yaml ├── .python-version ├── AUTHORS ├── LICENSE ├── Makefile ├── Procfile ├── README.md ├── api_plugins ├── doc ├── api.md ├── images │ ├── general_arch.jpeg │ └── globomap_api.jpeg ├── perm.md └── swagger.json ├── docker-compose.yml ├── examples ├── collections_start.json ├── document.json ├── edge.json └── graphs_start.json ├── globomap_api ├── __init__.py ├── api │ ├── __init__.py │ └── v2 │ │ ├── __init__.py │ │ ├── api.py │ │ ├── auth │ │ ├── __init__.py │ │ ├── decorators.py │ │ ├── exceptions.py │ │ ├── facade.py │ │ └── permissions.py │ │ ├── endpoints │ │ ├── __init__.py │ │ ├── auth.py │ │ ├── collections.py │ │ ├── edges.py │ │ ├── graphs.py │ │ ├── healthcheck.py │ │ ├── plugins.py │ │ └── queries.py │ │ ├── facade.py │ │ ├── parsers │ │ ├── __init__.py │ │ ├── base.py │ │ ├── collections.py │ │ ├── edges.py │ │ ├── graphs.py │ │ └── queries.py │ │ └── util.py ├── api_plugins │ ├── __init__.py │ ├── abstract_plugin.py │ └── plugin_loader.py ├── app.py ├── config.py ├── errors.py ├── exceptions.py ├── models │ ├── __init__.py │ ├── constructor.py │ ├── db.py │ └── document.py ├── run.py ├── scripts.py ├── specs │ ├── auth.json │ ├── clear.json │ ├── collections.json │ ├── documents.json │ ├── documents_partial.json │ ├── edges.json │ ├── edges_partial.json │ ├── graphs.json │ ├── queries.json │ └── search.json ├── util.py └── wsgi.py ├── requirements.txt ├── requirements_test.txt ├── scripts ├── arango │ ├── arango-create.py │ └── arango-remove.py └── docker │ ├── api │ ├── Dockerfile │ ├── meta_collections.sh │ └── wait_keystone_start.sh │ ├── ci.Dockerfile │ ├── expose_ports.sh │ ├── globomap.env │ └── keystone │ ├── Dockerfile │ ├── keystone.sh │ └── setup.sh ├── test_config.py ├── tests ├── __init__.py ├── config.py └── integration │ ├── __init__.py │ ├── api │ ├── __init__.py │ ├── collections_test.py │ └── edges_test.py │ ├── api_plugins │ ├── __init__.py │ └── plugin_data_test.py │ ├── cleanup.py │ └── models │ ├── __init__.py │ ├── collection_test.py │ ├── db_test.py │ ├── document_test.py │ └── graph_test.py └── tsuru.yml /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | indent_size = 4 8 | end_of_line = lf 9 | indent_style = space 10 | insert_final_newline = true 11 | trim_trailing_whitespace = true 12 | 13 | [*.md] 14 | trim_trailing_whitespace = false 15 | 16 | [Makefile] 17 | indent_style = tab -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .coverage 3 | .env 4 | *.log 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | .hypothesis/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # dotenv 85 | .env 86 | 87 | # virtualenv 88 | .venv 89 | venv/ 90 | ENV/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | .spyproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | 99 | # mkdocs documentation 100 | /site 101 | 102 | # mypy 103 | .mypy_cache/ 104 | 105 | # VSCode 106 | .vscode/ 107 | 108 | docker-compose-temp.yml 109 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | test: 2 | image: docker:stable 3 | before_script: 4 | - apk add make 5 | - apk add bash 6 | - apk add python3 7 | - apk --update add 'py-pip' 8 | - pip install 'docker-compose==1.23.2' 9 | script: 10 | - make dynamic_ports 11 | - make containers_clean 12 | - make containers_build 13 | - make containers_start 14 | - make tests_ci 15 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: https://github.com/pre-commit/pre-commit-hooks 2 | sha: v1.2.3 3 | hooks: 4 | - id: check-json 5 | - id: check-added-large-files 6 | - id: check-yaml 7 | - id: requirements-txt-fixer 8 | - id: trailing-whitespace 9 | language_version: python3.6 10 | - id: end-of-file-fixer 11 | language_version: python3.6 12 | - id: autopep8-wrapper 13 | language_version: python3.6 14 | args: 15 | - -i 16 | - --ignore=E501 17 | - id: check-docstring-first 18 | language_version: python3.6 19 | - id: debug-statements 20 | language_version: python3.6 21 | - id: double-quote-string-fixer 22 | language_version: python3.6 23 | - id: check-merge-conflict 24 | language_version: python3.6 25 | - id: check-ast 26 | language_version: python3.6 27 | - id: check-builtin-literals 28 | language_version: python3.6 29 | - id: check-byte-order-marker 30 | language_version: python3.6 31 | - id: name-tests-test 32 | language_version: python3.6 33 | - id: fix-encoding-pragma 34 | language_version: python3.6 35 | args: 36 | - --remove 37 | - id: flake8 38 | language_version: python3.6 39 | args: 40 | - '-' 41 | - --ignore=E501,E402,E722, F405 42 | - repo: https://github.com/pre-commit/pre-commit 43 | sha: 41dcaff3fb53fb7819a1d783d67a9ccb42464c1d 44 | hooks: 45 | - id: validate_config 46 | - id: validate_manifest 47 | - repo: https://github.com/asottile/reorder_python_imports 48 | sha: 9aa4d08f9a28d3defc5e4db3c3b77d1a9980fd1a 49 | hooks: 50 | - id: reorder-python-imports 51 | language_version: python3.6 52 | - repo: https://github.com/pre-commit/mirrors-autopep8 53 | sha: a59112ac7558236e411fd83a26551584ef8b928f 54 | hooks: 55 | - id: autopep8 56 | language_version: python3.6 57 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.6.5 2 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Alan 2 | Ederson Brilhante 3 | Sergio Saules 4 | Victor Hugo 5 | Victor Mendes 6 | juanaugusto 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 Globo.com 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT_HOME = "`pwd`" 2 | 3 | help: 4 | @echo 5 | @echo "Please use 'make ' where is one of" 6 | @echo 7 | 8 | @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 9 | 10 | setup: ## Install project dependencies 11 | @pip install -r $(PROJECT_HOME)/requirements_test.txt 12 | 13 | clean: ## Clear *.pyc files, etc 14 | @rm -rf build dist *.egg-info 15 | @find . \( -name '*.pyc' -o -name '__pycache__' -o -name '**/*.pyc' -o -name '*~' \) -delete 16 | 17 | exec_tests: ## Make tests 18 | @nosetests --verbose --rednose --nocapture --cover-package=globomap_api --with-coverage; coverage report -m 19 | 20 | tests: ## Run tests 21 | @docker exec -it globomap_api make exec_tests || true 22 | 23 | tests_ci: ## Run tests 24 | @docker exec globomap_api make exec_tests || true 25 | 26 | run: ## Run a development web server 27 | @PYTHONPATH=`pwd`:$PYTHONPATH python3.6 globomap_api/run.py 28 | 29 | containers_start: dynamic_ports ## Start containers 30 | docker-compose up -d 31 | ./scripts/docker/keystone/setup.sh 32 | 33 | containers_build: dynamic_ports ## Build containers 34 | docker-compose build --no-cache 35 | 36 | containers_stop: ## Stop containers 37 | docker-compose stop 38 | 39 | containers_clean: ## Destroy containers 40 | docker-compose rm -s -v -f 41 | 42 | dynamic_ports: ## Set ports to services 43 | ./scripts/docker/expose_ports.sh 44 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: gunicorn --timeout $TIMEOUT -b 0.0.0.0:$PORT -w $WORKERS globomap_api.wsgi --log-level $LOG_LEVEL 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Globo Map API 2 | 3 | Application responsible for reading and writing in ARANGODB. This application has a RESTFul API. 4 | 5 | ## Starting Project: 6 | 7 | ` make dynamic_ports`
8 | ` make containers_build ` (Build images.)
9 | ` make containers_start ` (Up containers)
10 | 11 | ## Running local with docker: 12 | 13 | ` make dynamic_ports`
14 | ` make containers_build ` (When project not started yet.)
15 | ` make containers_start ` (When project not started yet.)
16 | 17 | ## Running Tests: 18 | 19 | ` make containers_build ` (When project not started yet.)
20 | ` make containers_start ` (When project not started yet.)
21 | ` make tests ` 22 | 23 | ## Plugin environment variables configuration 24 | 25 | All of the environment variables below must be set for the api to work properly. 26 | 27 | | Variable | Description | Example | 28 | |------------------------------------|-----------------------------------------------------------------------------|----------------------------| 29 | | ARANGO_DB | Database name | globomap | 30 | | ARANGO_USERNAME | Database user | user | 31 | | ARANGO_PASSWORD | Database password | password | 32 | | ARANGO_PROTOCOL | Database protocol | https | 33 | | ARANGO_PORT | Database port | 8529 | 34 | | ARANGO_HOST | Database host | arangodb.domain.com | 35 | | VARIABLES of globomap-auth-manager | [globomap-auth-manager](https://github.com/globocom/globomap-auth-manager) | -- | 36 | 37 | Environment variables needed for the Zabbix plugin to work properly 38 | 39 | | Variable | Description | Example | 40 | |-----------------------------|-------------------------|----------------------------| 41 | | ZABBIX_API_URL | Zabbix API endpoint | https://ro.api.zabbix.com | 42 | | ZABBIX_API_USER | Zabbix username | username | 43 | | ZABBIX_API_PASSWORD | Zabbix password | xyz | 44 | | ZABBIX_UI_URL | Zabbix endpoint | https://zabbix.com | 45 | 46 | 47 | ### Environment variables configuration from external libs 48 | All of the environment variables below must be set for the application to work properly. 49 | 50 | [globomap-auth-manager](https://github.com/globocom/globomap-auth-manager) 51 | 52 | ### Requirements: 53 | #### Collections in ArangoDB: 54 | meta_collection
55 | meta_graph
56 | meta_query
57 | internal_metadata
58 | 59 | #### Roles in Keystone: 60 | globomap_admin
61 | globomap_read
62 | globomap_write
63 | 64 | ## Simple Example 65 | Complete flow with [python](https://github.com/edersonbrilhante/globomap-hands-on) 66 | 67 | ## Licensing 68 | Globo Map API is under [Apache 2 License](./LICENSE) 69 | -------------------------------------------------------------------------------- /api_plugins: -------------------------------------------------------------------------------- 1 | [zabbix-triggers] 2 | types=['comp_unit'] 3 | parameters=['ips'] 4 | description=Zabbix API trigger plugin 5 | module=globomap_plugin_zabbix.zabbix.ZabbixPlugin 6 | 7 | [zabbix-graph] 8 | types=['zabbix_graph'] 9 | parameters=['graphid'] 10 | description=Zabbix API graph plugin 11 | module=globomap_plugin_zabbix.zabbix.ZabbixPlugin 12 | 13 | [healthcheck] 14 | types=['comp_unit'] 15 | parameters=['ips'] 16 | description=Healthcheck plugin 17 | module=globomap_plugin_healthcheck.healthcheck.HealthcheckPlugin 18 | -------------------------------------------------------------------------------- /doc/api.md: -------------------------------------------------------------------------------- 1 | # GloboMap API 2 | 3 | API responsible for storing and retrieving information in [ArangoDB](https://arangodb.com/), the Globomap database. 4 | As can be seen in the [general architecture](images/general_arch.jpeg), it is used by [WEB UI](https://github.com/globocom/globomap-ui), [LOADER](https://github.com/globocom/globomap-loader) and any application that wishes to consume information from the database. 5 | 6 | 7 | ![alt text](images/globomap_api.jpeg) 8 | -------------------------------------------------------------------------------- /doc/images/general_arch.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/globocom/globomap-api/2fbbe96db9f6a1d34a6c886662383a42534a6a1a/doc/images/general_arch.jpeg -------------------------------------------------------------------------------- /doc/images/globomap_api.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/globocom/globomap-api/2fbbe96db9f6a1d34a6c886662383a42534a6a1a/doc/images/globomap_api.jpeg -------------------------------------------------------------------------------- /doc/perm.md: -------------------------------------------------------------------------------- 1 | # Permissions 2 | 3 | ### List of Roles 4 | | ROLES | Description | 5 | | ------ | ------ | 6 | | globomap_admin | Administrative Permission | 7 | | globomap_write | Write Permission | 8 | | globomap_read | Read Permission | 9 | 10 | ### Data Struct 11 | | ACTION | ROLES | 12 | | ------ | ------ | 13 | | **Create Collection** | *globomap_admin* | 14 | | **Create Edge** | *globomap_admin* | 15 | | **Create Graph** | *globomap_admin* | 16 | 17 | ### Collection Document 18 | | ACTION | ROLES | 19 | | ------ | ------ | 20 | | **Create** | *globomap_write* | 21 | | **Change** | *globomap_write* | 22 | | **Delete** | *globomap_write* | 23 | | **Read** | *globomap_read* | 24 | | **Search** | *globomap_read* | 25 | 26 | ### Edge Document 27 | | ACTION | ROLES | 28 | | ------ | ------ | 29 | | **Create** | *globomap_write* | 30 | | **Change** | *globomap_write* | 31 | | **Delete** | *globomap_write* | 32 | | **Read** | *globomap_read* | 33 | | **Search** | *globomap_read* | 34 | 35 | ### Queries 36 | | ACTION | ROLES | 37 | | ------ | ------ | 38 | | **Create** | *globomap_write* | 39 | | **Read** | *globomap_read* | 40 | | **Search** | *globomap_read* | 41 | | **Execute** | *globomap_read* | 42 | 43 | ### Graphs 44 | | ACTION | ROLES | 45 | | ------ | ------ | 46 | | **Read** | *globomap_read* | 47 | | **Search** | *globomap_read* | 48 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | 4 | db: 5 | container_name: globomap_db 6 | image: arangodb:3.3.15 7 | ports: 8 | - "${GMAP_DB_PORT}:8529" 9 | environment: 10 | - ARANGO_NO_AUTH=1 11 | volumes: 12 | - globomap_db_vol:/var/lib/arangodb-apps/_db 13 | networks: 14 | - globomap_net 15 | 16 | api: 17 | container_name: globomap_api 18 | build: 19 | context: ./ 20 | dockerfile: scripts/docker/api/Dockerfile 21 | ports: 22 | - "${GMAP_API_PORT}:5000" 23 | - "${GMAP_API_DEBUG_PORT}:5001" 24 | env_file: 25 | - scripts/docker/globomap.env 26 | command: bash -c "/bin/bash /home/scripts/docker/api/wait_keystone_start.sh" 27 | networks: 28 | - globomap_net 29 | external_links: 30 | - db:globomap_db 31 | - keystone:globomap_keystone 32 | 33 | keystone: 34 | container_name: globomap_keystone 35 | build: 36 | context: ./ 37 | dockerfile: scripts/docker/keystone/Dockerfile 38 | ports: 39 | - "${GMAP_KS_ADM_PORT}:35357" 40 | - "${GMAP_KS_PORT}:5000" 41 | restart: always 42 | environment: 43 | OS_TENANT_NAME: admin 44 | OS_USERNAME: admin 45 | networks: 46 | - globomap_net 47 | 48 | redis: 49 | container_name: globomap_redis 50 | image: redis:4.0.5-alpine 51 | command: ["redis-server", "--appendonly", "yes", "--requirepass", "password"] 52 | ports: 53 | - "${GMAP_REDIS_PORT}:6379" 54 | networks: 55 | - globomap_net 56 | volumes: 57 | - globomap_redis_vol:/data 58 | 59 | volumes: 60 | globomap_db_vol: 61 | globomap_redis_vol: 62 | 63 | networks: 64 | globomap_net: 65 | -------------------------------------------------------------------------------- /examples/collections_start.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "coll1" 3 | } 4 | -------------------------------------------------------------------------------- /examples/document.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "1", 3 | "name": "yyy", 4 | "provider": "xxx", 5 | "timestamp": 1501543772, 6 | "properties":{ 7 | "key": "value" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /examples/edge.json: -------------------------------------------------------------------------------- 1 | { 2 | "from": "coll1/xxx_1", 3 | "to": "coll2/xxx_1", 4 | "id": "1", 5 | "name": "yyy", 6 | "provider": "xxx", 7 | "timestamp": 1501543772, 8 | "properties": { 9 | "key": "value" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /examples/graphs_start.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "abc", 3 | "links": [ 4 | { 5 | "edge": "edge1", 6 | "from_collections": [ 7 | "coll1" 8 | ], 9 | "to_collections": [ 10 | "coll2" 11 | ] 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /globomap_api/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /globomap_api/api/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /globomap_api/api/v2/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from flask import Blueprint 17 | from flask_restplus import Api 18 | 19 | 20 | blueprint = Blueprint('APIv2', __name__, url_prefix='/v2') 21 | api = Api(blueprint, version='2.0', title='GloboMap API', 22 | description='GloboMap API') 23 | -------------------------------------------------------------------------------- /globomap_api/api/v2/api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | __all__ = ['api', 'blueprint'] 17 | from globomap_api.api.v2 import api 18 | from globomap_api.api.v2 import blueprint 19 | from globomap_api.api.v2.endpoints.auth import ns as auth_ns 20 | from globomap_api.api.v2.endpoints.collections import ns as coll_ns 21 | from globomap_api.api.v2.endpoints.edges import ns as edges_ns 22 | from globomap_api.api.v2.endpoints.graphs import ns as graphs_ns 23 | from globomap_api.api.v2.endpoints.queries import ns as query_ns 24 | from globomap_api.api.v2.endpoints.healthcheck import ns as healthcheck_ns 25 | from globomap_api.api.v2.endpoints.plugins import ns as plugin_ns 26 | 27 | api.add_namespace(auth_ns) 28 | api.add_namespace(coll_ns) 29 | api.add_namespace(edges_ns) 30 | api.add_namespace(graphs_ns) 31 | api.add_namespace(healthcheck_ns) 32 | api.add_namespace(plugin_ns) 33 | api.add_namespace(query_ns) 34 | -------------------------------------------------------------------------------- /globomap_api/api/v2/auth/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /globomap_api/api/v2/auth/decorators.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import functools 17 | 18 | from flask import request 19 | 20 | from globomap_api.api.v2.auth.facade import validate_token 21 | 22 | 23 | def permission_classes(permission_classes): 24 | def outer(func): 25 | @functools.wraps(func) 26 | def inner(self, *args, **kwargs): 27 | token = request.headers.get('Authorization') 28 | auth_inst = validate_token(token) 29 | if auth_inst: 30 | for permission_class in permission_classes: 31 | permission_class(auth_inst) 32 | return func(self, *args, **kwargs) 33 | return inner 34 | return outer 35 | -------------------------------------------------------------------------------- /globomap_api/api/v2/auth/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | 18 | class AuthException(Exception): 19 | 20 | def __init__(self, message): 21 | super(AuthException, self).__init__(message) 22 | 23 | self.message = message 24 | -------------------------------------------------------------------------------- /globomap_api/api/v2/auth/facade.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from flask import current_app as app 17 | from globomap_auth_manager import exceptions 18 | from globomap_auth_manager.auth import Auth 19 | 20 | from globomap_api.api.v2 import api 21 | 22 | 23 | def create_token(username=None, password=None): 24 | 25 | auth_inst = Auth() 26 | auth_inst.set_credentials(username, password) 27 | token = auth_inst.get_token_data() 28 | 29 | return token 30 | 31 | 32 | def get_roles(token): 33 | auth_inst = Auth() 34 | try: 35 | auth_inst.set_token(token) 36 | auth_inst.validate_token() 37 | token_data = auth_inst.get_token_data_details() 38 | 39 | except exceptions.InvalidToken: 40 | app.logger.warning('Invalid Token') 41 | api.abort(401, errors='Invalid Token') 42 | 43 | except exceptions.AuthException: 44 | err_msg = 'Error to validate token' 45 | app.logger.exception(err_msg) 46 | api.abort(503) 47 | 48 | return token_data 49 | 50 | 51 | def validate_token(token): 52 | auth_inst = Auth() 53 | 54 | try: 55 | auth_inst.set_token(token) 56 | auth_inst.validate_token() 57 | 58 | return auth_inst 59 | 60 | except exceptions.InvalidToken: 61 | app.logger.warning('Invalid Token') 62 | api.abort(401, errors='Invalid Token') 63 | 64 | except exceptions.AuthException: 65 | err_msg = 'Error to validate token' 66 | app.logger.exception(err_msg) 67 | api.abort(503) 68 | -------------------------------------------------------------------------------- /globomap_api/api/v2/auth/permissions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from flask import current_app as app 17 | 18 | from globomap_api.api.v2 import api 19 | 20 | 21 | class BasePermission(object): 22 | 23 | def __init__(self, auth): 24 | self.auth = auth 25 | self.validate_token() 26 | 27 | def has_permission(self, token=None): 28 | return True 29 | 30 | def has_perm(self, token, role): 31 | roles = [usr_role['name'] for usr_role in token['roles']] 32 | if role in roles: 33 | return True 34 | return False 35 | 36 | def validate_token(self): 37 | token_data = self.auth.get_token_data_details() 38 | if not self.has_permission(token_data): 39 | api.abort(403, 'User have not permission to this action') 40 | app.logger.error('User have not permission to this action') 41 | 42 | 43 | class Admin(BasePermission): 44 | 45 | def has_permission(self, token): 46 | return self.has_perm(token, app.config['ADMIN']) 47 | 48 | 49 | class Read(BasePermission): 50 | 51 | def has_permission(self, token): 52 | return self.has_perm(token, app.config['READ']) 53 | 54 | 55 | class Write(BasePermission): 56 | 57 | def has_permission(self, token): 58 | return self.has_perm(token, app.config['WRITE']) 59 | -------------------------------------------------------------------------------- /globomap_api/api/v2/endpoints/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /globomap_api/api/v2/endpoints/auth.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from flask import current_app as app 17 | from flask import request 18 | from flask_restplus import Resource 19 | from globomap_auth_manager import exceptions 20 | 21 | from globomap_api.api.v2 import api 22 | from globomap_api.api.v2.auth import facade 23 | from globomap_api.api.v2.auth.exceptions import AuthException 24 | from globomap_api.util import get_dict 25 | 26 | ns = api.namespace( 27 | 'auth', description='Operations related to auth') 28 | 29 | specs = app.config['SPECS'] 30 | 31 | 32 | @ns.route('/') 33 | class CreateAuth(Resource): 34 | 35 | @api.doc(responses={ 36 | 200: 'Success', 37 | 401: 'Unauthorized', 38 | }) 39 | @api.expect(api.schema_model('Auth', get_dict(specs.get('auth')))) 40 | def post(self): 41 | """Create token""" 42 | 43 | try: 44 | data = request.get_json() 45 | if type(data) is dict: 46 | username = data.get('username') 47 | password = data.get('password') 48 | else: 49 | username = None 50 | password = None 51 | if not username or not password: 52 | app.logger.error('Username and Password is required.') 53 | api.abort(401, errors='Username and Password is required.') 54 | 55 | token = facade.create_token(username, password) 56 | return token, 200 57 | 58 | except exceptions.Unauthorized: 59 | app.logger.error('User %s not authorized.', username) 60 | api.abort(401, 'User not authorized.') 61 | 62 | except exceptions.AuthException: 63 | app.logger.error('Auth Unavailable.') 64 | api.abort(503, 'Auth Unavailable.') 65 | 66 | except AuthException: 67 | app.logger.error('Auth Unavailable.') 68 | api.abort(503, 'Auth Unavailable.') 69 | 70 | 71 | @ns.route('/roles/') 72 | @api.header( 73 | 'Authorization', 74 | 'Token Authorization', 75 | required=True, 76 | default='Token token=' 77 | ) 78 | class Roles(Resource): 79 | 80 | @api.doc(responses={ 81 | 200: 'Success', 82 | 401: 'Unauthorized', 83 | }) 84 | def get(self): 85 | """Create token""" 86 | token = request.headers.get('Authorization') 87 | token_data = facade.get_roles(token) 88 | return token_data, 200 89 | -------------------------------------------------------------------------------- /globomap_api/api/v2/endpoints/collections.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import json 17 | from json.decoder import JSONDecodeError 18 | 19 | from flask import current_app as app 20 | from flask import request 21 | from flask_restplus import Resource 22 | from jsonspec.validators.exceptions import ValidationError 23 | 24 | from globomap_api import exceptions as gmap_exc 25 | from globomap_api.api.v2 import api 26 | from globomap_api.api.v2 import facade 27 | from globomap_api.api.v2.auth import permissions 28 | from globomap_api.api.v2.auth.decorators import permission_classes 29 | from globomap_api.api.v2.parsers import collections as coll_parsers 30 | from globomap_api.api.v2.parsers.base import search_parser 31 | from globomap_api.api.v2.util import get_dict 32 | from globomap_api.api.v2.util import validate 33 | 34 | ns = api.namespace( 35 | 'collections', description='Operations related to collections') 36 | 37 | specs = app.config['SPECS'] 38 | 39 | 40 | @ns.route('/') 41 | @api.header( 42 | 'Authorization', 43 | 'Token Authorization', 44 | required=True, 45 | default='Token token=' 46 | ) 47 | class Collections(Resource): 48 | 49 | @api.doc(responses={ 50 | 200: 'Success', 51 | 401: 'Unauthorized', 52 | 403: 'Forbidden', 53 | }) 54 | @api.expect(search_parser) 55 | @permission_classes((permissions.Read,)) 56 | def get(self): 57 | """List all collections of kind document from DB.""" 58 | 59 | args = search_parser.parse_args(request) 60 | 61 | try: 62 | query = args.get('query') or '[]' 63 | page = args.get('page') 64 | per_page = args.get('per_page') 65 | data = json.loads(query) 66 | res = facade.list_collections('document', data, page, per_page) 67 | except JSONDecodeError: 68 | raise gmap_exc.SearchException('Parameter query is invalid') 69 | else: 70 | return res, 200 71 | 72 | @api.doc(responses={ 73 | 200: 'Success', 74 | 400: 'Validation Error', 75 | 401: 'Unauthorized', 76 | 403: 'Forbidden', 77 | 404: 'Not Found' 78 | }) 79 | @permission_classes((permissions.Admin,)) 80 | @api.expect(api.schema_model('Collections', 81 | get_dict(specs.get('collections')))) 82 | def post(self): 83 | """Create collection of kind document in DB.""" 84 | 85 | try: 86 | data = request.get_json() 87 | 88 | app.logger.debug('Receive Data: %s', data) 89 | facade.create_collection_document(data) 90 | return {}, 200 91 | 92 | except ValidationError as error: 93 | res = validate(error) 94 | app.logger.error(res) 95 | api.abort(400, errors=res) 96 | 97 | except gmap_exc.CollectionNotExist as err: 98 | app.logger.error(err.message) 99 | api.abort(404, errors=err.message) 100 | 101 | 102 | @ns.route('/search/') 103 | @api.header( 104 | 'Authorization', 105 | 'Token Authorization', 106 | required=True, 107 | default='Token token=' 108 | ) 109 | class Search(Resource): 110 | 111 | @api.doc(responses={ 112 | 200: 'Success', 113 | 400: 'Validation Error', 114 | 401: 'Unauthorized', 115 | 403: 'Forbidden', 116 | 404: 'Not Found' 117 | }) 118 | @api.expect(coll_parsers.search_all_parser) 119 | @permission_classes((permissions.Read,)) 120 | def get(self): 121 | """Search document in collections of kind document from DB.""" 122 | 123 | args = coll_parsers.search_all_parser.parse_args(request) 124 | 125 | try: 126 | try: 127 | page = args.get('page') 128 | query = args.get('query') or '[]' 129 | per_page = args.get('per_page') 130 | collections = args.get('collections').split(',') 131 | data = json.loads(query) 132 | app.logger.debug('Receive Data: %s', data) 133 | except JSONDecodeError: 134 | raise gmap_exc.SearchException('Parameter query is invalid') 135 | else: 136 | res = facade.search_collections( 137 | collections, data, page, per_page) 138 | return res, 200 139 | 140 | except gmap_exc.CollectionNotExist as err: 141 | app.logger.error(err.message) 142 | api.abort(404, errors=err.message) 143 | 144 | except ValidationError as error: 145 | res = validate(error) 146 | app.logger.error(res) 147 | api.abort(400, errors=res) 148 | 149 | 150 | @ns.route('//') 151 | @api.doc(params={'collection': 'Name Of Collection'}) 152 | @api.header( 153 | 'Authorization', 154 | 'Token Authorization', 155 | required=True, 156 | default='Token token=' 157 | ) 158 | class Collection(Resource): 159 | 160 | @api.doc(responses={ 161 | 200: 'Success', 162 | 400: 'Validation Error', 163 | 401: 'Unauthorized', 164 | 403: 'Forbidden', 165 | 404: 'Not Found', 166 | 409: 'Document Already Exists' 167 | }) 168 | @api.expect(api.schema_model('DocumentPost', 169 | get_dict(specs.get('documents')))) 170 | @permission_classes((permissions.Write,)) 171 | def post(self, collection): 172 | """Insert document in DB.""" 173 | 174 | try: 175 | data = request.get_json() 176 | app.logger.debug('Receive Data: %s', data) 177 | res = facade.create_document(collection, data) 178 | return res, 200 179 | 180 | except ValidationError as error: 181 | res = validate(error) 182 | app.logger.error(res) 183 | api.abort(400, errors=res) 184 | 185 | except gmap_exc.CollectionNotExist as err: 186 | app.logger.error(err.message) 187 | api.abort(404, errors=err.message) 188 | 189 | except gmap_exc.DocumentAlreadyExist as err: 190 | app.logger.warning(err.message) 191 | api.abort(409, errors=err.message) 192 | 193 | except gmap_exc.DocumentException as err: 194 | app.logger.error(err.message) 195 | api.abort(400, errors=err.message) 196 | 197 | @api.doc(responses={ 198 | 200: 'Success', 199 | 400: 'Validation Error', 200 | 401: 'Unauthorized', 201 | 403: 'Forbidden', 202 | 404: 'Not Found', 203 | 409: 'Document Already Exists' 204 | }) 205 | @api.expect(coll_parsers.search_parser) 206 | @permission_classes((permissions.Read,)) 207 | def get(self, collection): 208 | """Search documents from collection.""" 209 | 210 | args = coll_parsers.search_parser.parse_args(request) 211 | 212 | try: 213 | try: 214 | page = args.get('page') 215 | query = args.get('query') or '[]' 216 | per_page = args.get('per_page') 217 | data = json.loads(query) 218 | except JSONDecodeError: 219 | raise gmap_exc.SearchException('Parameter query is invalid') 220 | else: 221 | res = facade.search(collection, data, page, per_page) 222 | return res, 200 223 | 224 | except gmap_exc.CollectionNotExist as err: 225 | app.logger.error(err.message) 226 | api.abort(404, errors=err.message) 227 | 228 | except ValidationError as error: 229 | res = validate(error) 230 | app.logger.error(res) 231 | api.abort(400, errors=res) 232 | 233 | 234 | @ns.route('//clear/') 235 | @api.doc(params={'collection': 'Name Of Collection'}) 236 | @api.header( 237 | 'Authorization', 238 | 'Token Authorization', 239 | required=True, 240 | default='Token token=' 241 | ) 242 | class CollectionClear(Resource): 243 | 244 | @api.doc(responses={ 245 | 200: 'Success', 246 | 400: 'Validation Error', 247 | 401: 'Unauthorized', 248 | 403: 'Forbidden', 249 | 404: 'Not Found' 250 | }) 251 | @api.expect(api.schema_model('DocumentClear', 252 | get_dict(specs.get('clear')))) 253 | @permission_classes((permissions.Write,)) 254 | def post(self, collection): 255 | """Clear documents in collection.""" 256 | 257 | try: 258 | data = request.get_json() 259 | app.logger.debug('Receive Data: %s', data) 260 | res = facade.clear_collection(collection, data) 261 | return res, 200 262 | 263 | except gmap_exc.CollectionNotExist as err: 264 | app.logger.error(err.message) 265 | api.abort(404, errors=err.message) 266 | 267 | except ValidationError as error: 268 | res = validate(error) 269 | app.logger.error(res) 270 | api.abort(400, errors=res) 271 | 272 | 273 | @ns.route('///') 274 | @api.doc(params={ 275 | 'collection': 'Name Of Collection', 276 | 'key': 'Key Of Document' 277 | }) 278 | @api.header( 279 | 'Authorization', 280 | 'Token Authorization', 281 | required=True, 282 | default='Token token=' 283 | ) 284 | class Document(Resource): 285 | 286 | @api.doc(responses={ 287 | 200: 'Success', 288 | 400: 'Validation Error', 289 | 401: 'Unauthorized', 290 | 403: 'Forbidden', 291 | 404: 'Not Found' 292 | }) 293 | @api.expect(api.schema_model('DocumentPut', 294 | get_dict(specs.get('documents')))) 295 | @permission_classes((permissions.Write,)) 296 | def put(self, collection, key): 297 | """Update document.""" 298 | 299 | try: 300 | data = request.get_json() 301 | app.logger.debug('Receive Data: %s', data) 302 | res = facade.update_document(collection, key, data) 303 | return res, 200 304 | 305 | except ValidationError as error: 306 | res = validate(error) 307 | app.logger.error(res) 308 | api.abort(400, errors=res) 309 | 310 | except gmap_exc.CollectionNotExist as err: 311 | app.logger.error(err.message) 312 | api.abort(404, errors=err.message) 313 | 314 | except gmap_exc.DocumentNotExist as err: 315 | app.logger.warning(err.message) 316 | api.abort(404, errors=err.message) 317 | 318 | @api.doc(responses={ 319 | 200: 'Success', 320 | 400: 'Validation Error', 321 | 401: 'Unauthorized', 322 | 403: 'Forbidden', 323 | 404: 'Not Found' 324 | }) 325 | @api.expect(api.schema_model('DocumentPatch', 326 | get_dict(specs.get('documents_partial')))) 327 | @permission_classes((permissions.Write,)) 328 | def patch(self, collection, key): 329 | """Partial update document.""" 330 | 331 | try: 332 | data = request.get_json() 333 | app.logger.debug('Receive Data: %s', data) 334 | res = facade.patch_document(collection, key, data) 335 | return res, 200 336 | 337 | except ValidationError as error: 338 | res = validate(error) 339 | app.logger.error(res) 340 | api.abort(400, errors=res) 341 | 342 | except gmap_exc.CollectionNotExist as err: 343 | app.logger.error(err.message) 344 | api.abort(404, errors=err.message) 345 | 346 | except gmap_exc.DocumentNotExist as err: 347 | app.logger.warning(err.message) 348 | api.abort(404, errors=err.message) 349 | 350 | @api.doc(responses={ 351 | 200: 'Success', 352 | 400: 'Validation Error', 353 | 401: 'Unauthorized', 354 | 403: 'Forbidden', 355 | 404: 'Not Found' 356 | }) 357 | @permission_classes((permissions.Read,)) 358 | def get(self, collection, key): 359 | """Get document by key.""" 360 | 361 | try: 362 | res = facade.get_document(collection, key) 363 | return res, 200 364 | 365 | except gmap_exc.CollectionNotExist as err: 366 | app.logger.error(err.message) 367 | api.abort(404, errors=err.message) 368 | 369 | except gmap_exc.DocumentNotExist as err: 370 | app.logger.warning(err.message) 371 | api.abort(404, errors=err.message) 372 | 373 | @api.doc(responses={ 374 | 200: 'Success', 375 | 401: 'Unauthorized', 376 | 403: 'Forbidden', 377 | 404: 'Not Found' 378 | }) 379 | @permission_classes((permissions.Write,)) 380 | def delete(self, collection, key): 381 | """Delete document by key.""" 382 | 383 | try: 384 | facade.delete_document(collection, key) 385 | return {}, 200 386 | 387 | except gmap_exc.CollectionNotExist as err: 388 | app.logger.error(err.message) 389 | api.abort(404, errors=err.message) 390 | 391 | except gmap_exc.DocumentNotExist as err: 392 | app.logger.warning(err.message) 393 | api.abort(404, errors=err.message) 394 | 395 | 396 | @ns.route('//count/') 397 | @api.doc(params={ 398 | 'collection': 'Name Of Collection' 399 | }) 400 | @api.header( 401 | 'Authorization', 402 | 'Token Authorization', 403 | required=True, 404 | default='Token token=' 405 | ) 406 | class CollectionsCount(Resource): 407 | 408 | @api.doc(responses={ 409 | 200: 'Success', 410 | 400: 'Validation Error', 411 | 401: 'Unauthorized', 412 | 403: 'Forbidden', 413 | 404: 'Not Found' 414 | }) 415 | @api.expect(search_parser) 416 | @permission_classes((permissions.Read,)) 417 | def get(self, collection): 418 | """Get count documents.""" 419 | 420 | args = search_parser.parse_args(request) 421 | 422 | try: 423 | query = args.get('query') or '[]' 424 | data = json.loads(query) 425 | res = facade.count_document(collection, data) 426 | return res, 200 427 | 428 | except gmap_exc.CollectionNotExist as err: 429 | app.logger.error(err.message) 430 | api.abort(404, errors=err.message) 431 | 432 | except gmap_exc.DocumentNotExist as err: 433 | app.logger.warning(err.message) 434 | api.abort(404, errors=err.message) 435 | -------------------------------------------------------------------------------- /globomap_api/api/v2/endpoints/edges.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import json 17 | from json.decoder import JSONDecodeError 18 | 19 | from flask import current_app as app 20 | from flask import request 21 | from flask_restplus import Resource 22 | from jsonspec.validators.exceptions import ValidationError 23 | 24 | from globomap_api import exceptions as gmap_exc 25 | from globomap_api.api.v2 import api 26 | from globomap_api.api.v2 import facade 27 | from globomap_api.api.v2.auth import permissions 28 | from globomap_api.api.v2.auth.decorators import permission_classes 29 | from globomap_api.api.v2.parsers import edges as edges_parsers 30 | from globomap_api.api.v2.parsers.base import search_parser 31 | from globomap_api.api.v2.util import get_dict 32 | from globomap_api.api.v2.util import validate 33 | 34 | ns = api.namespace('edges', description='Operations related to edges') 35 | 36 | specs = app.config['SPECS'] 37 | 38 | 39 | @ns.route('/') 40 | @api.header( 41 | 'Authorization', 42 | 'Token Authorization', 43 | required=True, 44 | default='Token token=' 45 | ) 46 | class Edges(Resource): 47 | 48 | @api.doc(responses={ 49 | 200: 'Success', 50 | 401: 'Unauthorized', 51 | 403: 'Forbidden', 52 | }) 53 | @api.expect(search_parser) 54 | @permission_classes((permissions.Read,)) 55 | def get(self): 56 | """List all collections of kind edge from DB.""" 57 | 58 | args = search_parser.parse_args(request) 59 | 60 | try: 61 | query = args.get('query') or '[]' 62 | page = args.get('page') 63 | per_page = args.get('per_page') 64 | data = json.loads(query) 65 | res = facade.list_collections('edge', data, page, per_page) 66 | except JSONDecodeError: 67 | raise gmap_exc.SearchException('Parameter query is invalid') 68 | else: 69 | return res, 200 70 | 71 | @api.doc(responses={ 72 | 200: 'Success', 73 | 400: 'Validation Error', 74 | 401: 'Unauthorized', 75 | 403: 'Forbidden', 76 | 404: 'Not Found' 77 | }) 78 | @api.expect(api.schema_model('Edge', get_dict(specs.get('collections')))) 79 | @permission_classes((permissions.Admin,)) 80 | def post(self): 81 | """Create collection of kind edge in DB.""" 82 | 83 | try: 84 | data = request.get_json() 85 | app.logger.debug('Receive Data: %s', data) 86 | facade.create_collection_edge(data) 87 | return {}, 200 88 | 89 | except ValidationError as error: 90 | res = validate(error) 91 | app.logger.error(res) 92 | api.abort(400, errors=res) 93 | 94 | except gmap_exc.CollectionNotExist as err: 95 | app.logger.error(err.message) 96 | api.abort(404, errors=err.message) 97 | 98 | 99 | @ns.route('/search/') 100 | @api.header( 101 | 'Authorization', 102 | 'Token Authorization', 103 | required=True, 104 | default='Token token=' 105 | ) 106 | class EdgeSearch(Resource): 107 | 108 | @api.doc(responses={ 109 | 200: 'Success', 110 | 400: 'Validation Error', 111 | 401: 'Unauthorized', 112 | 403: 'Forbidden', 113 | 404: 'Not Found' 114 | }) 115 | @api.expect(edges_parsers.search_all_parser) 116 | @permission_classes((permissions.Read,)) 117 | def get(self): 118 | """Search edge in collections of kind edge from DB.""" 119 | 120 | args = edges_parsers.search_all_parser.parse_args(request) 121 | 122 | try: 123 | try: 124 | page = args.get('page') 125 | query = args.get('query') or '[]' 126 | per_page = args.get('per_page') 127 | edges = args.get('edges').split(',') 128 | data = json.loads(query) 129 | except JSONDecodeError: 130 | raise gmap_exc.SearchException('Parameter query is invalid') 131 | else: 132 | res = facade.search_collections( 133 | edges, data, page, per_page) 134 | return res, 200 135 | 136 | except gmap_exc.CollectionNotExist as err: 137 | app.logger.error(err.message) 138 | api.abort(404, errors=err.message) 139 | 140 | except ValidationError as error: 141 | res = validate(error) 142 | app.logger.error(res) 143 | api.abort(400, errors=res) 144 | 145 | 146 | @ns.route('//clear/') 147 | @api.header( 148 | 'Authorization', 149 | 'Token Authorization', 150 | required=True, 151 | default='Token token=' 152 | ) 153 | @api.doc(params={'edge': 'Name Of Edge(Collection)'}) 154 | class EdgeClear(Resource): 155 | 156 | @api.doc(responses={ 157 | 200: 'Success', 158 | 400: 'Validation Error', 159 | 401: 'Unauthorized', 160 | 403: 'Forbidden', 161 | 404: 'Not Found' 162 | }) 163 | @api.expect(api.schema_model('EdgeClear', get_dict(specs.get('clear')))) 164 | @permission_classes((permissions.Write,)) 165 | def post(self, edge): 166 | """Clear documents in edge.""" 167 | 168 | try: 169 | data = request.get_json() 170 | app.logger.debug('Receive Data: %s', data) 171 | res = facade.clear_collection(edge, data) 172 | return res, 200 173 | 174 | except gmap_exc.CollectionNotExist as err: 175 | app.logger.error(err.message) 176 | api.abort(404, errors=err.message) 177 | 178 | except ValidationError as error: 179 | res = validate(error) 180 | app.logger.error(res) 181 | api.abort(400, errors=res) 182 | 183 | 184 | @ns.route('//') 185 | @api.header( 186 | 'Authorization', 187 | 'Token Authorization', 188 | required=True, 189 | default='Token token=' 190 | ) 191 | @api.doc(params={'edge': 'Name Of Edge(Collection)'}) 192 | class Edge(Resource): 193 | 194 | @api.doc(responses={ 195 | 200: 'Success', 196 | 400: 'Validation Error', 197 | 401: 'Unauthorized', 198 | 403: 'Forbidden', 199 | 404: 'Not Found', 200 | 409: 'Document Already Exists' 201 | }) 202 | @api.expect(api.schema_model('EdgePost', get_dict(specs.get('edges')))) 203 | @permission_classes((permissions.Write,)) 204 | def post(self, edge): 205 | """Insert edge in DB.""" 206 | 207 | try: 208 | data = request.get_json() 209 | app.logger.debug('Receive Data: %s', data) 210 | res = facade.create_edge(edge, data) 211 | return res, 200 212 | 213 | except gmap_exc.EdgeNotExist as err: 214 | app.logger.error(err.message) 215 | api.abort(404, errors=err.message) 216 | 217 | except gmap_exc.DocumentAlreadyExist as err: 218 | app.logger.warning(err.message) 219 | api.abort(409, errors=err.message) 220 | 221 | except ValidationError as error: 222 | res = validate(error) 223 | app.logger.error(res) 224 | api.abort(400, errors=res) 225 | 226 | except gmap_exc.DocumentException as err: 227 | api.abort(400, errors=err.message) 228 | 229 | @api.doc(responses={ 230 | 200: 'Success', 231 | 400: 'Validation Error', 232 | 401: 'Unauthorized', 233 | 403: 'Forbidden', 234 | 404: 'Not Found' 235 | }) 236 | @api.expect(edges_parsers.search_parser) 237 | @permission_classes((permissions.Read,)) 238 | def get(self, edge): 239 | """Search documents from collection.""" 240 | 241 | args = edges_parsers.search_parser.parse_args(request) 242 | 243 | try: 244 | try: 245 | page = args.get('page') 246 | query = args.get('query') or '[]' 247 | per_page = args.get('per_page') 248 | data = json.loads(query) 249 | except JSONDecodeError: 250 | api.abort(400, errors='Parameter query is invalid') 251 | else: 252 | res = facade.search(edge, data, page, per_page) 253 | return res, 200 254 | 255 | except gmap_exc.CollectionNotExist as err: 256 | app.logger.error(err.message) 257 | api.abort(404, errors=err.message) 258 | 259 | except ValidationError as error: 260 | res = validate(error) 261 | app.logger.error(res) 262 | api.abort(400, errors=res) 263 | 264 | 265 | @ns.route('///') 266 | @api.doc(params={ 267 | 'edge': 'Name Of Edge(Collection)', 268 | 'key': 'Key Of Document' 269 | }) 270 | @api.header( 271 | 'Authorization', 272 | 'Token Authorization', 273 | required=True, 274 | default='Token token=' 275 | ) 276 | class DocumentEdge(Resource): 277 | 278 | @api.doc(responses={ 279 | 200: 'Success', 280 | 400: 'Validation Error', 281 | 401: 'Unauthorized', 282 | 403: 'Forbidden', 283 | 404: 'Not Found' 284 | }) 285 | @api.expect(api.schema_model('EdgePut', get_dict(specs.get('edges')))) 286 | @permission_classes((permissions.Write,)) 287 | def put(self, edge, key): 288 | """Update edge.""" 289 | 290 | try: 291 | data = request.get_json() 292 | app.logger.debug('Receive Data: %s', data) 293 | res = facade.update_edge(edge, key, data) 294 | return res, 200 295 | 296 | except ValidationError as error: 297 | res = validate(error) 298 | app.logger.error(res) 299 | api.abort(400, errors=res) 300 | 301 | except gmap_exc.EdgeNotExist as err: 302 | app.logger.error(err.message) 303 | api.abort(404, errors=err.message) 304 | 305 | except gmap_exc.DocumentNotExist as err: 306 | app.logger.warning(err.message) 307 | api.abort(404, errors=err.message) 308 | 309 | @api.doc(responses={ 310 | 200: 'Success', 311 | 400: 'Validation Error', 312 | 401: 'Unauthorized', 313 | 403: 'Forbidden', 314 | 404: 'Not Found' 315 | }) 316 | @api.expect(api.schema_model('EdgePatch', 317 | get_dict(specs.get('edges_partial')))) 318 | @permission_classes((permissions.Write,)) 319 | def patch(self, edge, key): 320 | """Partial update edge.""" 321 | 322 | try: 323 | data = request.get_json() 324 | app.logger.debug('Receive Data: %s', data) 325 | res = facade.patch_edge(edge, key, data) 326 | return res, 200 327 | 328 | except ValidationError as error: 329 | res = validate(error) 330 | app.logger.error(res) 331 | api.abort(400, errors=res) 332 | 333 | except gmap_exc.EdgeNotExist as err: 334 | app.logger.error(err.message) 335 | api.abort(404, errors=err.message) 336 | 337 | except gmap_exc.DocumentNotExist as err: 338 | app.logger.warning(err.message) 339 | api.abort(404, errors=err.message) 340 | 341 | @api.doc(responses={ 342 | 200: 'Success', 343 | 400: 'Validation Error', 344 | 401: 'Unauthorized', 345 | 403: 'Forbidden', 346 | 404: 'Not Found' 347 | }) 348 | @permission_classes((permissions.Read,)) 349 | def get(self, edge, key): 350 | """Get edge by key.""" 351 | 352 | try: 353 | res = facade.get_edge(edge, key) 354 | return res, 200 355 | 356 | except gmap_exc.EdgeNotExist as err: 357 | app.logger.error(err.message) 358 | api.abort(404, errors=err.message) 359 | 360 | except gmap_exc.DocumentNotExist as err: 361 | app.logger.warning(err.message) 362 | api.abort(404, errors=err.message) 363 | 364 | @api.doc(responses={ 365 | 200: 'Success', 366 | 401: 'Unauthorized', 367 | 403: 'Forbidden', 368 | 404: 'Not Found' 369 | }) 370 | @permission_classes((permissions.Write,)) 371 | def delete(self, edge, key): 372 | """Delete edge by key.""" 373 | 374 | try: 375 | facade.delete_edge(edge, key) 376 | return {}, 200 377 | 378 | except gmap_exc.EdgeNotExist as err: 379 | app.logger.error(err.message) 380 | api.abort(404, errors=err.message) 381 | 382 | except gmap_exc.DocumentNotExist as err: 383 | app.logger.warning(err.message) 384 | api.abort(404, errors=err.message) 385 | 386 | 387 | @ns.route('//count/') 388 | @api.doc(params={ 389 | 'collection': 'Name Of Collection' 390 | }) 391 | @api.header( 392 | 'Authorization', 393 | 'Token Authorization', 394 | required=True, 395 | default='Token token=' 396 | ) 397 | class EdgesCount(Resource): 398 | 399 | @api.doc(responses={ 400 | 200: 'Success', 401 | 400: 'Validation Error', 402 | 401: 'Unauthorized', 403 | 403: 'Forbidden', 404 | 404: 'Not Found' 405 | }) 406 | @api.expect(search_parser) 407 | @permission_classes((permissions.Read,)) 408 | def get(self, collection): 409 | """Get count documents.""" 410 | 411 | args = search_parser.parse_args(request) 412 | 413 | try: 414 | query = args.get('query') or '[]' 415 | data = json.loads(query) 416 | res = facade.count_document(collection, data) 417 | return res, 200 418 | 419 | except gmap_exc.CollectionNotExist as err: 420 | app.logger.error(err.message) 421 | api.abort(404, errors=err.message) 422 | 423 | except gmap_exc.DocumentNotExist as err: 424 | app.logger.warning(err.message) 425 | api.abort(404, errors=err.message) 426 | -------------------------------------------------------------------------------- /globomap_api/api/v2/endpoints/graphs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from json.decoder import JSONDecodeError 17 | 18 | from flask import current_app as app 19 | from flask import request 20 | from flask_restplus import Resource 21 | from jsonspec.validators.exceptions import ValidationError 22 | 23 | from globomap_api import exceptions as gmap_exc 24 | from globomap_api.api.v2 import api 25 | from globomap_api.api.v2 import facade 26 | from globomap_api.api.v2.auth import permissions 27 | from globomap_api.api.v2.auth.decorators import permission_classes 28 | from globomap_api.api.v2.parsers import graphs as graphs_parsers 29 | from globomap_api.api.v2.parsers.base import search_parser 30 | from globomap_api.api.v2.util import get_dict 31 | from globomap_api.api.v2.util import validate 32 | 33 | ns = api.namespace('graphs', description='Operations related to graphs') 34 | 35 | specs = app.config['SPECS'] 36 | 37 | 38 | @ns.route('/') 39 | @api.header( 40 | 'Authorization', 41 | 'Token Authorization', 42 | required=True, 43 | default='Token token=' 44 | ) 45 | class Graph(Resource): 46 | 47 | @api.doc(responses={ 48 | 200: 'Success', 49 | 401: 'Unauthorized', 50 | 403: 'Forbidden' 51 | }) 52 | @api.expect(search_parser) 53 | @permission_classes((permissions.Read,)) 54 | def get(self): 55 | """List all graphs from DB.""" 56 | 57 | args = search_parser.parse_args(request) 58 | 59 | try: 60 | page = args.get('page') 61 | per_page = args.get('per_page') 62 | res = facade.list_graphs(page, per_page) 63 | except JSONDecodeError: 64 | raise gmap_exc.SearchException('Parameter query is invalid') 65 | else: 66 | return res, 200 67 | 68 | @api.doc(responses={ 69 | 200: 'Success', 70 | 401: 'Unauthorized', 71 | 403: 'Forbidden', 72 | 400: 'Validation Error', 73 | }) 74 | @api.expect(api.schema_model('GraphPost', get_dict(specs.get('graphs')))) 75 | @permission_classes((permissions.Admin,)) 76 | def post(self): 77 | """Create graph in DB.""" 78 | 79 | try: 80 | data = request.get_json() 81 | app.logger.debug('Receive Data: %s', data) 82 | facade.create_graph(data) 83 | return {}, 200 84 | 85 | except ValidationError as error: 86 | res = validate(error) 87 | app.logger.error(res) 88 | api.abort(400, errors=res) 89 | 90 | 91 | @ns.route('//traversal/') 92 | @api.doc(params={'graph': 'Name Of Graph'}) 93 | @api.header( 94 | 'Authorization', 95 | 'Token Authorization', 96 | required=True, 97 | default='Token token=' 98 | ) 99 | class GraphTraversal(Resource): 100 | 101 | @api.doc(responses={ 102 | 200: 'Success', 103 | 401: 'Unauthorized', 104 | 403: 'Forbidden', 105 | 404: 'Not Found' 106 | }) 107 | @api.expect(graphs_parsers.traversal_parser) 108 | @permission_classes((permissions.Read,)) 109 | def get(self, graph): 110 | """Search traversal.""" 111 | 112 | args = graphs_parsers.traversal_parser.parse_args(request) 113 | 114 | try: 115 | search_dict = { 116 | 'graph_name': graph, 117 | 'start_vertex': args.get('start_vertex'), 118 | 'direction': args.get('direction'), 119 | 'item_order': args.get('item_order'), 120 | 'strategy': args.get('strategy'), 121 | 'order': args.get('order'), 122 | 'edge_uniqueness': args.get('edge_uniqueness'), 123 | 'vertex_uniqueness': args.get('vertex_uniqueness'), 124 | 'max_iter': args.get('max_iter'), 125 | 'min_depth': args.get('min_depth'), 126 | 'max_depth': args.get('max_depth'), 127 | 'init_func': args.get('init_func'), 128 | 'sort_func': args.get('sort_func'), 129 | 'filter_func': args.get('filter_func'), 130 | 'visitor_func': args.get('visitor_func'), 131 | 'expander_func': args.get('expander_func') 132 | } 133 | 134 | res = facade.search_traversal(**search_dict) 135 | return res, 200 136 | 137 | except gmap_exc.GraphTraverseException as err: 138 | app.logger.error(err.message) 139 | api.abort(400, errors=err.message) 140 | 141 | except gmap_exc.GraphNotExist as err: 142 | app.logger.error(err.message) 143 | api.abort(404, errors=err.message) 144 | -------------------------------------------------------------------------------- /globomap_api/api/v2/endpoints/healthcheck.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import flask 17 | import six 18 | from flask import current_app as app 19 | from flask_restplus import Resource 20 | 21 | from globomap_api.api.v2 import api 22 | from globomap_api.api.v2 import facade 23 | from globomap_api.api.v2.auth.facade import Auth 24 | from globomap_api.config import SIMPLE_HEALTHCHECK 25 | 26 | 27 | ns = api.namespace('healthcheck', description='Healthcheck') 28 | 29 | 30 | def text(data, code, headers=None): 31 | return flask.make_response(six.text_type(data)) 32 | 33 | 34 | @ns.route('/') 35 | class Healthcheck(Resource): 36 | representations = { 37 | 'text/plain': text, 38 | } 39 | 40 | @api.doc(responses={ 41 | 200: 'Success', 42 | 503: 'Service Unavailable', 43 | }) 44 | def get(self): 45 | # Temporary Troubleshooting kubernetes healthcheck 46 | if SIMPLE_HEALTHCHECK: 47 | return 'WORKING', 200 48 | # Temporary Troubleshooting kubernetes healthcheck 49 | 50 | deps = { 51 | 'auth': _is_auth_ok(), 52 | 'arango': _is_arango_ok() 53 | } 54 | problems = {} 55 | for key in deps: 56 | if deps[key].get('status') is False: 57 | problems.update({key: deps[key]}) 58 | if problems: 59 | app.logger.error(problems) 60 | self.representations = {} 61 | return problems, 503 62 | return 'WORKING', 200 63 | 64 | 65 | @ns.route('/deps/') 66 | class HealthcheckDeps(Resource): 67 | 68 | @api.doc(responses={200: 'Success'}) 69 | def get(self): 70 | deps = { 71 | 'auth': _is_auth_ok(), 72 | 'arango': _is_arango_ok_details() 73 | } 74 | return deps, 200 75 | 76 | 77 | def _is_arango_ok_details(): 78 | try: 79 | db = app.config['ARANGO_CONN'] 80 | db.get_database() 81 | graphs = get_graphs() 82 | collections = get_collections('document') 83 | edges = get_collections('edge') 84 | except Exception: 85 | app.logger.error('Failed to healthcheck arango.') 86 | deps = {'status': False} 87 | else: 88 | deps = { 89 | 'status': True, 90 | 'graphs': graphs, 91 | 'collections': collections, 92 | 'edges': edges 93 | } 94 | 95 | return deps 96 | 97 | 98 | def _is_arango_ok(): 99 | try: 100 | db = app.config['ARANGO_CONN'] 101 | db.get_database() 102 | facade.list_graphs() 103 | facade.list_collections('document') 104 | facade.list_collections('edge') 105 | except Exception: 106 | app.logger.error('Failed to healthcheck arango.') 107 | deps = {'status': False} 108 | else: 109 | deps = { 110 | 'status': True 111 | } 112 | 113 | return deps 114 | 115 | 116 | def get_graphs(): 117 | page = 1 118 | graphs = [] 119 | while True: 120 | partial_graphs = facade.list_graphs(page=page, per_page=100) 121 | graphs += partial_graphs['graphs'] 122 | if page == partial_graphs['total_pages']: 123 | break 124 | page += 1 125 | return graphs 126 | 127 | 128 | def get_collections(kind): 129 | page = 1 130 | collections = [] 131 | while True: 132 | partial_coll = facade.list_collections(kind, page=page, per_page=100) 133 | collections += partial_coll['collections'] 134 | if page == partial_coll['total_pages']: 135 | break 136 | page += 1 137 | return collections 138 | 139 | 140 | def _is_auth_ok(): 141 | auth_inst = Auth() 142 | status = { 143 | 'status': auth_inst.is_url_ok() 144 | } 145 | 146 | return status 147 | -------------------------------------------------------------------------------- /globomap_api/api/v2/endpoints/plugins.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import configparser 17 | import ast 18 | 19 | from flask import current_app as app 20 | from flask import request 21 | from flask_restplus import Resource 22 | 23 | from globomap_api.api.v2 import api 24 | from globomap_api.api.v2.auth import permissions 25 | from globomap_api.api.v2.auth.decorators import permission_classes 26 | from globomap_api.api_plugins.abstract_plugin import PluginError 27 | from globomap_api.api_plugins.abstract_plugin import PluginNotFoundException 28 | from globomap_api.api_plugins.plugin_loader import ApiPluginLoader 29 | 30 | ns = api.namespace('plugins', description='Plugins') 31 | 32 | 33 | @ns.route('/') 34 | @api.header( 35 | 'Authorization', 36 | 'Token Authorization', 37 | required=True, 38 | default='Token token=' 39 | ) 40 | class Plugins(Resource): 41 | 42 | @api.doc(responses={ 43 | 200: 'Success', 44 | 401: 'Unauthorized', 45 | 403: 'Forbidden' 46 | }) 47 | @permission_classes((permissions.Read,)) 48 | def get(self): 49 | try: 50 | plugins_config = configparser.ConfigParser() 51 | plugins_config.read(app.config['API_PLUGINS_CONFIG_FILE']) 52 | keys = plugins_config.sections() 53 | plugins = [] 54 | 55 | for key in keys: 56 | plugins.append({ 57 | 'name': key, 58 | 'types': ast.literal_eval(plugins_config.get(key, 'types')), 59 | 'parameters': ast.literal_eval(plugins_config.get(key, 'parameters')), 60 | 'description': plugins_config.get(key, 'description'), 61 | 'uri': '{}{}/{}/'.format(api.base_url, 'plugins', key) 62 | }) 63 | 64 | return plugins 65 | except Exception: 66 | err_msg = 'Error in plugin' 67 | app.logger.exception(err_msg) 68 | api.abort(500, errors=err_msg) 69 | 70 | 71 | @ns.route('//') 72 | @api.doc(params={'plugin_name': 'Name Of Plugin'}) 73 | @api.header( 74 | 'Authorization', 75 | 'Token Authorization', 76 | required=True, 77 | default='Token token=' 78 | ) 79 | class PluginData(Resource): 80 | 81 | @api.doc(responses={ 82 | 200: 'Success', 83 | 400: 'Validation Error', 84 | 401: 'Unauthorized', 85 | 403: 'Forbidden', 86 | 404: 'Not Found' 87 | }) 88 | @permission_classes((permissions.Read,)) 89 | def post(self, plugin_name): 90 | try: 91 | args = request.get_json() 92 | plugin_instance = ApiPluginLoader().load_plugin(plugin_name) 93 | data = plugin_instance.get_data(args) 94 | return data 95 | except PluginNotFoundException as err: 96 | app.logger.error(str(err)) 97 | api.abort(404, errors=str(err)) 98 | return data 99 | except PluginError as err: 100 | app.logger.error(str(err)) 101 | api.abort(400, errors=str(err)) 102 | return data 103 | except Exception: 104 | err_msg = 'Error in plugin' 105 | app.logger.exception(err_msg) 106 | api.abort(500, errors=err_msg) 107 | -------------------------------------------------------------------------------- /globomap_api/api/v2/endpoints/queries.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import json 17 | 18 | from flask import current_app as app 19 | from flask import request 20 | from flask_restplus import Resource 21 | from jsonspec.validators.exceptions import ValidationError 22 | 23 | from globomap_api import exceptions as gmap_exc 24 | from globomap_api.api.v2 import api 25 | from globomap_api.api.v2 import facade 26 | from globomap_api.api.v2.auth import permissions 27 | from globomap_api.api.v2.auth.decorators import permission_classes 28 | from globomap_api.api.v2.parsers import queries as query_parsers 29 | from globomap_api.api.v2.util import get_dict 30 | from globomap_api.api.v2.util import validate 31 | 32 | ns = api.namespace( 33 | 'queries', description='Operations related to queries') 34 | 35 | specs = app.config['SPECS'] 36 | 37 | 38 | @ns.route('/') 39 | @api.header( 40 | 'Authorization', 41 | 'Token Authorization', 42 | required=True, 43 | default='Token token=' 44 | ) 45 | class Query(Resource): 46 | 47 | @api.doc(responses={ 48 | 200: 'Success', 49 | 401: 'Unauthorized', 50 | 403: 'Forbidden', 51 | }) 52 | @api.expect(query_parsers.search_query_parser) 53 | @permission_classes((permissions.Read,)) 54 | def get(self): 55 | """List all queries from DB.""" 56 | 57 | args = query_parsers.search_query_parser.parse_args(request) 58 | 59 | page = args.get('page') 60 | per_page = args.get('per_page') 61 | query = args.get('query') 62 | data = json.loads(query) 63 | 64 | queries = facade.list_query(data, page, per_page) 65 | return queries, 200 66 | 67 | @api.doc(responses={ 68 | 200: 'Success', 69 | 400: 'Validation Error', 70 | 401: 'Unauthorized', 71 | 403: 'Forbidden', 72 | 409: 'Document Already Exists' 73 | }) 74 | @api.expect(api.schema_model('QueryPost', 75 | get_dict(specs.get('queries')))) 76 | @permission_classes((permissions.Write,)) 77 | def post(self): 78 | """Create queries in DB.""" 79 | 80 | try: 81 | data = request.get_json() 82 | 83 | app.logger.debug('Receive Data: %s', data) 84 | res = facade.create_query(data) 85 | return res, 200 86 | 87 | except ValidationError as error: 88 | res = validate(error) 89 | app.logger.error(res) 90 | api.abort(400, errors=res) 91 | 92 | except gmap_exc.QueryException as error: 93 | app.logger.warning(error.message) 94 | api.abort(400, errors=error.message) 95 | 96 | except gmap_exc.DocumentAlreadyExist: 97 | res = 'Cannot create query, already created.' 98 | app.logger.warning(res) 99 | api.abort(409, errors=res) 100 | 101 | 102 | @ns.route('//') 103 | @api.doc(params={ 104 | 'key': 'Key Of Query' 105 | }) 106 | @api.header( 107 | 'Authorization', 108 | 'Token Authorization', 109 | required=True, 110 | default='Token token=' 111 | ) 112 | class DocumentQuery(Resource): 113 | 114 | @api.doc(responses={ 115 | 200: 'Success', 116 | 400: 'Validation Error', 117 | 401: 'Unauthorized', 118 | 403: 'Forbidden', 119 | 404: 'Not Found' 120 | }) 121 | @api.expect(api.schema_model('QueryPut', 122 | get_dict(specs.get('queries')))) 123 | @permission_classes((permissions.Write,)) 124 | def put(self, key): 125 | """Update query in DB.""" 126 | 127 | try: 128 | data = request.get_json() 129 | app.logger.debug('Receive Data: %s', data) 130 | res = facade.update_query(key, data) 131 | return res, 200 132 | 133 | except ValidationError as error: 134 | res = validate(error) 135 | app.logger.error(res) 136 | api.abort(400, errors=res) 137 | 138 | except gmap_exc.QueryException as error: 139 | app.logger.warning(error.message) 140 | api.abort(400, errors=error.message) 141 | 142 | except gmap_exc.DocumentNotExist as error: 143 | app.logger.warning(error.message) 144 | api.abort(404, errors=error.message) 145 | 146 | @api.doc(responses={ 147 | 200: 'Success', 148 | 400: 'Validation Error', 149 | 401: 'Unauthorized', 150 | 403: 'Forbidden', 151 | 404: 'Not Found' 152 | }) 153 | @permission_classes((permissions.Read,)) 154 | def get(self, key): 155 | """Get query by key.""" 156 | 157 | try: 158 | res = facade.get_query(key) 159 | return res, 200 160 | 161 | except gmap_exc.DocumentNotExist as error: 162 | app.logger.warning(error.message) 163 | api.abort(404, errors=error.message) 164 | 165 | @api.doc(responses={ 166 | 200: 'Success', 167 | 401: 'Unauthorized', 168 | 403: 'Forbidden', 169 | 404: 'Not Found' 170 | }) 171 | @permission_classes((permissions.Write,)) 172 | def delete(self, key): 173 | """Delete query by key.""" 174 | 175 | try: 176 | res = facade.delete_query(key) 177 | return res, 200 178 | 179 | except gmap_exc.DocumentNotExist as error: 180 | app.logger.warning(error.message) 181 | api.abort(404, errors=error.message) 182 | 183 | 184 | @ns.route('//execute/') 185 | @api.doc(params={ 186 | 'key': 'Key Of Query' 187 | }) 188 | @api.header( 189 | 'Authorization', 190 | 'Token Authorization', 191 | required=True, 192 | default='Token token=' 193 | ) 194 | class ExecuteQuery(Resource): 195 | 196 | @api.doc(responses={ 197 | 200: 'Success', 198 | 400: 'Validation Error', 199 | 401: 'Unauthorized', 200 | 403: 'Forbidden', 201 | 404: 'Not Found' 202 | }) 203 | @api.expect(query_parsers.execute_query_parser) 204 | @permission_classes((permissions.Read,)) 205 | def get(self, key): 206 | """Get query by key.""" 207 | 208 | args = query_parsers.execute_query_parser.parse_args(request) 209 | variable = args.get('variable') 210 | 211 | try: 212 | res = facade.execute_query(key, variable) 213 | return res, 200 214 | 215 | except gmap_exc.DocumentNotExist as error: 216 | app.logger.warning(error.message) 217 | api.abort(404, errors=error.message) 218 | -------------------------------------------------------------------------------- /globomap_api/api/v2/facade.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import math 17 | 18 | from arango.exceptions import GraphTraverseError 19 | from flask import current_app as app 20 | 21 | from globomap_api import exceptions as gmap_exceptions 22 | from globomap_api.api.v2 import util 23 | from globomap_api.errors import GRAPH_TRAVERSE as traverse_err 24 | from globomap_api.models.constructor import Constructor 25 | from globomap_api.models.document import Document 26 | 27 | 28 | ######### 29 | # GRAPH # 30 | ######### 31 | def create_graph(data): 32 | """Create graph in Database""" 33 | 34 | constructor = Constructor() 35 | name = data.get('name') 36 | links = data.get('links') 37 | constructor.factory(kind='Graph', create=True, name=name, links=links) 38 | create_meta_graph_doc(data) 39 | 40 | return True 41 | 42 | 43 | def create_meta_graph_doc(data): 44 | """Create document in meta_graph""" 45 | 46 | constructor = Constructor() 47 | inst_coll = constructor.factory( 48 | kind='Collection', name=app.config['META_GRAPH']) 49 | 50 | inst_doc = Document(inst_coll) 51 | document = { 52 | '_key': data.get('name'), 53 | 'name': data.get('name'), 54 | 'links': data.get('links'), 55 | 'alias': data.get('alias'), 56 | 'icon': data.get('icon'), 57 | 'description': data.get('description'), 58 | } 59 | doc = inst_doc.create_document(document) 60 | 61 | return doc 62 | 63 | 64 | def list_graphs(page=1, per_page=10): 65 | """Return all graph from Database""" 66 | 67 | db_inst = app.config['ARANGO_CONN'] 68 | db_inst.get_database() 69 | cursor = db_inst.search_in_collection( 70 | app.config['META_GRAPH'], None, page, per_page) 71 | 72 | total_pages = int(math.ceil(cursor.statistics()[ 73 | 'fullCount'] / (per_page * 1.0))) 74 | total_graphs = cursor.statistics()['fullCount'] 75 | 76 | graphs = util.filter_graphs(cursor) 77 | res = { 78 | 'total_pages': total_pages, 79 | 'total': len(graphs), 80 | 'total_graphs': total_graphs, 81 | 'graphs': graphs 82 | } 83 | 84 | return res 85 | 86 | 87 | ############## 88 | # COLLECTION # 89 | ############## 90 | def create_meta_collection_doc(data, kind): 91 | """Create document in meta_collection""" 92 | 93 | constructor = Constructor() 94 | inst_coll = constructor.factory( 95 | kind='Collection', name=app.config['META_COLLECTION']) 96 | 97 | inst_doc = Document(inst_coll) 98 | document = { 99 | '_key': data.get('name'), 100 | 'name': data.get('name'), 101 | 'alias': data.get('alias'), 102 | 'kind': kind, 103 | 'icon': data.get('icon'), 104 | 'description': data.get('description'), 105 | 'users': data.get('users'), 106 | } 107 | doc = inst_doc.create_document(document) 108 | 109 | return doc 110 | 111 | 112 | def create_collection_document(data): 113 | """Create collection in Database""" 114 | 115 | constructor = Constructor() 116 | name = data.get('name') 117 | replication_factor = data.get('replicationFactor') 118 | constructor.factory(kind='Collection', create=True, name=name, replication_factor=replication_factor) 119 | create_meta_collection_doc(data, 'document') 120 | 121 | return True 122 | 123 | 124 | def create_collection_edge(data): 125 | """Create edge in Database""" 126 | 127 | constructor = Constructor() 128 | name = data.get('name') 129 | replication_factor = data.get('replicationFactor') 130 | constructor.factory(kind='Edges', create=True, name=name, replication_factor=replication_factor) 131 | create_meta_collection_doc(data, 'edge') 132 | 133 | return True 134 | 135 | 136 | def list_collections(kind, data=[], page=1, per_page=10): 137 | """Return all collections or edges from Database""" 138 | 139 | db_inst = app.config['ARANGO_CONN'] 140 | db_inst.get_database() 141 | filter_coll = [{ 142 | 'field': 'kind', 143 | 'operator': '==', 144 | 'value': kind, 145 | }] 146 | for idx, _ in enumerate(data): 147 | data[idx] += filter_coll 148 | cursor = db_inst.search_in_collection( 149 | app.config['META_COLLECTION'], data, page, per_page) 150 | 151 | total_pages = int(math.ceil(cursor.statistics()[ 152 | 'fullCount'] / (per_page * 1.0))) 153 | total_collections = cursor.statistics()['fullCount'] 154 | 155 | collections = util.filter_collections(cursor) 156 | res = { 157 | 'total_pages': total_pages, 158 | 'total': len(collections), 159 | 'total_collections': total_collections, 160 | 'collections': collections 161 | } 162 | 163 | return res 164 | 165 | 166 | ############ 167 | # DOCUMENT # 168 | ############ 169 | def create_document(name, data): 170 | """Create document in Database""" 171 | 172 | constructor = Constructor() 173 | inst_coll = constructor.factory(kind='Collection', name=name) 174 | 175 | document = data 176 | document = { 177 | '_key': util.make_key(data), 178 | 'id': data['id'], 179 | 'name': data.get('name', ''), 180 | 'provider': data['provider'], 181 | 'timestamp': data['timestamp'], 182 | 'properties': data.get('properties'), 183 | 'properties_metadata': data.get('properties_metadata') 184 | } 185 | 186 | inst_doc = Document(inst_coll) 187 | doc = inst_doc.create_document(document) 188 | 189 | return doc 190 | 191 | 192 | def get_document(name, key): 193 | """Get document from Database""" 194 | 195 | constructor = Constructor() 196 | inst_coll = constructor.factory(kind='Collection', name=name) 197 | 198 | inst_doc = Document(inst_coll) 199 | doc = inst_doc.get_document(key) 200 | 201 | return doc 202 | 203 | 204 | def update_document(name, key, data): 205 | """Update document from Database""" 206 | 207 | get_document(name, key) 208 | document = { 209 | '_key': key, 210 | 'id': data['id'], 211 | 'name': data.get('name', ''), 212 | 'provider': data['provider'], 213 | 'timestamp': data['timestamp'], 214 | 'properties': data.get('properties'), 215 | 'properties_metadata': data.get('properties_metadata') 216 | } 217 | 218 | constructor = Constructor() 219 | inst_coll = constructor.factory(kind='Collection', name=name) 220 | 221 | inst_doc = Document(inst_coll) 222 | doc = inst_doc.update_document(document) 223 | 224 | return doc 225 | 226 | 227 | def patch_document(name, key, data): 228 | """Partial update document from Database""" 229 | 230 | document = get_document(name, key) 231 | 232 | for key in data: 233 | if key == 'properties': 234 | for idx in data[key]: 235 | document['properties'][idx] = data[key][idx] 236 | elif key == 'properties_metadata': 237 | for idx in data[key]: 238 | document['properties_metadata'][idx] = data[key][idx] 239 | elif key == 'name': 240 | if data[key]: 241 | document[key] = data[key] 242 | else: 243 | document[key] = data[key] 244 | 245 | constructor = Constructor() 246 | inst_coll = constructor.factory(kind='Collection', name=name) 247 | 248 | inst_doc = Document(inst_coll) 249 | doc = inst_doc.update_document(document) 250 | 251 | return doc 252 | 253 | 254 | def delete_document(name, key): 255 | """Get document from Database""" 256 | 257 | constructor = Constructor() 258 | inst_coll = constructor.factory(kind='Collection', name=name) 259 | 260 | inst_doc = Document(inst_coll) 261 | inst_doc.delete_document(key) 262 | 263 | return True 264 | 265 | 266 | def count_document(name, search=[]): 267 | """Get count from document""" 268 | 269 | db_inst = app.config['ARANGO_CONN'] 270 | 271 | db_inst.get_database() 272 | cursor = db_inst.count_in_document(name, search) 273 | 274 | count = 0 275 | docs = [doc for doc in cursor] 276 | 277 | if len(docs) == 1: 278 | count = docs[0] 279 | 280 | return count 281 | 282 | 283 | ######## 284 | # EDGE # 285 | ######## 286 | def create_edge(name, data): 287 | """Create document-edge in Database""" 288 | 289 | constructor = Constructor() 290 | inst_edge = constructor.factory(kind='Edges', name=name) 291 | edge = { 292 | '_key': util.make_key(data), 293 | '_from': data['from'], 294 | '_to': data['to'], 295 | 'id': data['id'], 296 | 'name': data.get('name', ''), 297 | 'provider': data['provider'], 298 | 'timestamp': data['timestamp'], 299 | 'properties': data.get('properties'), 300 | 'properties_metadata': data.get('properties_metadata') 301 | } 302 | 303 | inst_doc = Document(inst_edge) 304 | doc = inst_doc.create_document(edge) 305 | 306 | return doc 307 | 308 | 309 | def get_edge(name, key): 310 | """Get edge from Database""" 311 | 312 | constructor = Constructor() 313 | inst_edge = constructor.factory(kind='Edges', name=name) 314 | 315 | inst_doc = Document(inst_edge) 316 | doc = inst_doc.get_document(key) 317 | 318 | return doc 319 | 320 | 321 | def update_edge(name, key, data): 322 | """Update edge from Database""" 323 | 324 | get_edge(name, key) 325 | edge = { 326 | '_key': key, 327 | '_from': data['from'], 328 | '_to': data['to'], 329 | 'id': data['id'], 330 | 'name': data.get('name', ''), 331 | 'provider': data['provider'], 332 | 'timestamp': data['timestamp'], 333 | 'properties': data.get('properties'), 334 | 'properties_metadata': data.get('properties_metadata') 335 | } 336 | 337 | constructor = Constructor() 338 | inst_edge = constructor.factory(kind='Edges', name=name) 339 | 340 | inst_doc = Document(inst_edge) 341 | doc = inst_doc.update_document(edge) 342 | 343 | return doc 344 | 345 | 346 | def patch_edge(name, key, data): 347 | """Partial update edge from Database""" 348 | 349 | edge = get_edge(name, key) 350 | for key in data: 351 | if key == 'from': 352 | edge['_from'] = data[key] 353 | elif key == 'to': 354 | edge['_to'] = data[key] 355 | elif key == 'properties': 356 | for idx in data[key]: 357 | edge['properties'][idx] = data[key][idx] 358 | elif key == 'properties_metadata': 359 | for idx in data[key]: 360 | edge['properties_metadata'][idx] = data[key][idx] 361 | elif key == 'name': 362 | if data[key]: 363 | edge[key] = data[key] 364 | else: 365 | edge[key] = data[key] 366 | 367 | constructor = Constructor() 368 | inst_edge = constructor.factory(kind='Edges', name=name) 369 | 370 | inst_doc = Document(inst_edge) 371 | doc = inst_doc.update_document(edge) 372 | 373 | return doc 374 | 375 | 376 | def delete_edge(name, key): 377 | """Get edge from Database""" 378 | 379 | constructor = Constructor() 380 | inst_edge = constructor.factory(kind='Edges', name=name) 381 | 382 | inst_doc = Document(inst_edge) 383 | inst_doc.delete_document(key) 384 | 385 | return True 386 | 387 | 388 | ########## 389 | # SEARCH # 390 | ########## 391 | def search_traversal(**kwargs): 392 | """Search Traversal in Database""" 393 | 394 | db_inst = app.config['ARANGO_CONN'] 395 | db_inst.get_database() 396 | graph = db_inst.get_graph(kwargs.get('graph_name')) 397 | try: 398 | traversal_results = graph.traverse( 399 | start_vertex=kwargs.get('start_vertex'), 400 | direction=kwargs.get('direction'), 401 | item_order=kwargs.get('item_order'), 402 | strategy=kwargs.get('strategy'), 403 | order=kwargs.get('order'), 404 | edge_uniqueness=kwargs.get('edge_uniqueness'), 405 | vertex_uniqueness=kwargs.get('vertex_uniqueness'), 406 | max_iter=kwargs.get('max_iter'), 407 | min_depth=kwargs.get('min_depth'), 408 | max_depth=kwargs.get('max_depth'), 409 | init_func=kwargs.get('init_func'), 410 | sort_func=kwargs.get('sort_func'), 411 | filter_func=kwargs.get('filter_func'), 412 | visitor_func=kwargs.get('visitor_func'), 413 | expander_func=kwargs.get('expander_func') 414 | ) 415 | except GraphTraverseError as err: 416 | 417 | if traverse_err.get(err.error_code): 418 | if err.error_code == 1202: 419 | msg = traverse_err.get(1202) 420 | raise gmap_exceptions.GraphTraverseException(msg) 421 | raise gmap_exceptions.GraphTraverseException( 422 | traverse_err.get(err.error_code).format(err.message)) 423 | 424 | else: 425 | raise gmap_exceptions.GraphTraverseException( 426 | traverse_err.get(0).format( 427 | kwargs.get('graph_name'), err.message)) 428 | 429 | except Exception as err: 430 | raise gmap_exceptions.GraphTraverseException( 431 | traverse_err.get(0).format(kwargs.get('graph_name'), err)) 432 | 433 | traversal_results = util.filter_transversal(traversal_results) 434 | traversal_results.update({'graph': kwargs.get('graph_name')}) 435 | 436 | return traversal_results 437 | 438 | 439 | def search(name, data, page, per_page): 440 | """Search in Database""" 441 | 442 | db_inst = app.config['ARANGO_CONN'] 443 | 444 | db_inst.get_database() 445 | cursor = db_inst.search_in_collection(name, data, page, per_page) 446 | 447 | total_pages = int(math.ceil(cursor.statistics()[ 448 | 'fullCount'] / (per_page * 1.0))) 449 | total_documents = cursor.statistics()['fullCount'] 450 | 451 | docs = [doc for doc in cursor] 452 | 453 | res = { 454 | 'total_pages': total_pages, 455 | 'total': len(docs), 456 | 'total_documents': total_documents, 457 | 'documents': docs 458 | } 459 | 460 | return res 461 | 462 | 463 | def clear_collection(name, data): 464 | """Clear document in Collection""" 465 | 466 | db_inst = app.config['ARANGO_CONN'] 467 | 468 | db_inst.get_database() 469 | db_inst.clear_collection(name, data) 470 | 471 | return {} 472 | 473 | 474 | def search_collections(collections, data, page, per_page): 475 | """Search in Database""" 476 | 477 | db_inst = app.config['ARANGO_CONN'] 478 | 479 | db_inst.get_database() 480 | cursor = db_inst.search_in_collections(collections, data, page, per_page) 481 | 482 | total_pages = int(math.ceil(cursor.statistics()[ 483 | 'fullCount'] / (per_page * 1.0))) 484 | total_documents = cursor.statistics()['fullCount'] 485 | 486 | docs = [doc for doc in cursor] 487 | 488 | res = { 489 | 'total_pages': total_pages, 490 | 'total': len(docs), 491 | 'total_documents': total_documents, 492 | 'documents': docs 493 | } 494 | 495 | return res 496 | 497 | 498 | ########### 499 | # QUERIES # 500 | ########### 501 | def make_query(data): 502 | """Validate query and make document""" 503 | 504 | query = data 505 | key = 'query_{}'.format(data.get('name')) 506 | query = { 507 | '_key': key, 508 | 'name': data.get('name'), 509 | 'description': data.get('description'), 510 | 'query': data.get('query'), 511 | 'params': data.get('params'), 512 | 'collection': data.get('collection') 513 | } 514 | 515 | db_inst = app.config['ARANGO_CONN'] 516 | db_inst.get_database() 517 | db_inst.validate_aql(data.get('query')) 518 | 519 | return query 520 | 521 | 522 | def create_query(data): 523 | """Create query in Database""" 524 | 525 | query = make_query(data) 526 | 527 | constructor = Constructor() 528 | inst_coll = constructor.factory(kind='Collection', 529 | name=app.config['META_QUERY']) 530 | 531 | inst_doc = Document(inst_coll) 532 | doc = inst_doc.create_document(query) 533 | 534 | return doc 535 | 536 | 537 | def update_query(key, data): 538 | """Update query in Database""" 539 | 540 | query = make_query(data) 541 | 542 | constructor = Constructor() 543 | inst_coll = constructor.factory(kind='Collection', 544 | name=app.config['META_QUERY']) 545 | 546 | inst_doc = Document(inst_coll) 547 | doc = inst_doc.update_document(query) 548 | 549 | return doc 550 | 551 | 552 | def get_query(key): 553 | """Get query from Database""" 554 | 555 | # TODO: validate key 556 | return get_document(app.config['META_QUERY'], key) 557 | 558 | 559 | def delete_query(key): 560 | """Delete query in Database""" 561 | 562 | # TODO: validate key 563 | return delete_document(app.config['META_QUERY'], key) 564 | 565 | 566 | def list_query(data, page, per_page): 567 | """List query in Database""" 568 | 569 | return search(app.config['META_QUERY'], data, page, per_page) 570 | 571 | 572 | def execute_query(key, variable): 573 | query = get_query(key) 574 | if variable: 575 | query['params']['variable'] = variable 576 | 577 | db_inst = app.config['ARANGO_CONN'] 578 | 579 | db_inst.get_database() 580 | cursor = db_inst.execute_aql(query['query'], query['params']) 581 | 582 | docs = [doc for doc in cursor] 583 | 584 | return docs 585 | -------------------------------------------------------------------------------- /globomap_api/api/v2/parsers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /globomap_api/api/v2/parsers/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from flask_restplus import reqparse 17 | 18 | 19 | search_parser = reqparse.RequestParser() 20 | search_parser.add_argument( 21 | 'page', 22 | type=int, 23 | required=False, 24 | default=1, 25 | help='Page number' 26 | ) 27 | search_parser.add_argument( 28 | 'per_page', 29 | type=int, 30 | required=False, 31 | default=10, 32 | help='Items number per page' 33 | ) 34 | search_parser.add_argument( 35 | 'query', 36 | type=str, 37 | required=False, 38 | default='[[{"field":"name","operator":"LIKE","value":""}]]', 39 | help='Query' 40 | ) 41 | -------------------------------------------------------------------------------- /globomap_api/api/v2/parsers/collections.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from globomap_api.api.v2.parsers import base 17 | 18 | search_all_parser = base.search_parser.copy() 19 | search_all_parser.add_argument( 20 | 'collections', 21 | type=str, 22 | required=True, 23 | default='', 24 | help='Collections' 25 | ) 26 | 27 | search_parser = base.search_parser.copy() 28 | -------------------------------------------------------------------------------- /globomap_api/api/v2/parsers/edges.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from globomap_api.api.v2.parsers import base 17 | 18 | search_all_parser = base.search_parser.copy() 19 | search_all_parser.add_argument( 20 | 'edges', 21 | type=str, 22 | required=True, 23 | default='', 24 | help='Edges Document' 25 | ) 26 | 27 | search_parser = base.search_parser.copy() 28 | -------------------------------------------------------------------------------- /globomap_api/api/v2/parsers/graphs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from flask_restplus import reqparse 17 | 18 | 19 | traversal_parser = reqparse.RequestParser() 20 | traversal_parser.add_argument( 21 | 'start_vertex', 22 | type=str, 23 | required=True, 24 | help='Start Vertex' 25 | ) 26 | traversal_parser.add_argument( 27 | 'direction', 28 | type=str, 29 | required=False, 30 | choices=['outbound', 'inbound', 'any'], 31 | default='outbound', 32 | help='Direction' 33 | ) 34 | traversal_parser.add_argument( 35 | 'item_order', 36 | type=str, 37 | required=False, 38 | choices=['forward', 'backward'], 39 | default='forward', 40 | help='Item Order' 41 | ) 42 | traversal_parser.add_argument( 43 | 'strategy', 44 | type=str, 45 | required=False, 46 | choices=['dfs', 'bfs'], 47 | default='dfs', 48 | help='Strategy' 49 | ) 50 | traversal_parser.add_argument( 51 | 'order', 52 | type=str, 53 | required=False, 54 | choices=['preorder', 'postorder', 55 | 'preorder-expander'], 56 | default=None, 57 | help='Order' 58 | ) 59 | traversal_parser.add_argument( 60 | 'edge_uniqueness', 61 | type=str, 62 | required=False, 63 | choices=['global', 'path'], 64 | default=None, 65 | help='Edge Uniqueness' 66 | ) 67 | traversal_parser.add_argument( 68 | 'vertex_uniqueness', 69 | type=str, 70 | required=False, 71 | choices=['global', 'path'], 72 | default=None, 73 | help='Vertex Uniqueness' 74 | ) 75 | traversal_parser.add_argument( 76 | 'max_iter', 77 | type=int, 78 | required=False, 79 | default=None, 80 | help='Max Iteration' 81 | ) 82 | traversal_parser.add_argument( 83 | 'min_depth', 84 | type=int, 85 | required=False, 86 | default=None, 87 | help='Min Depth' 88 | ) 89 | traversal_parser.add_argument( 90 | 'max_depth', 91 | type=int, 92 | required=False, 93 | default=1, 94 | help='Max Depth' 95 | ) 96 | traversal_parser.add_argument( 97 | 'init_func', 98 | type=str, 99 | required=False, 100 | default=None, help='Init Func') 101 | traversal_parser.add_argument( 102 | 'sort_func', 103 | type=str, 104 | required=False, 105 | default=None, 106 | help='Sort Function' 107 | ) 108 | traversal_parser.add_argument( 109 | 'filter_func', 110 | type=str, 111 | required=False, 112 | default=None, 113 | help='Filter Function' 114 | ) 115 | traversal_parser.add_argument( 116 | 'visitor_func', 117 | type=str, 118 | required=False, 119 | default=None, 120 | help='Visitor Function' 121 | ) 122 | traversal_parser.add_argument( 123 | 'expander_func', 124 | type=str, 125 | required=False, 126 | default=None, 127 | help='Expander Function' 128 | ) 129 | -------------------------------------------------------------------------------- /globomap_api/api/v2/parsers/queries.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from flask_restplus import reqparse 17 | 18 | search_query_parser = reqparse.RequestParser() 19 | search_query_parser.add_argument( 20 | 'page', 21 | type=int, 22 | required=False, 23 | default=1, 24 | help='Page number' 25 | ) 26 | search_query_parser.add_argument( 27 | 'per_page', 28 | type=int, 29 | required=False, 30 | default=10, 31 | help='Items number per page' 32 | ) 33 | search_query_parser.add_argument( 34 | 'query', 35 | type=str, 36 | required=False, 37 | default='[[{"field":"name","operator":"LIKE","value":""}]]', 38 | help='Query' 39 | ) 40 | 41 | execute_query_parser = reqparse.RequestParser() 42 | execute_query_parser.add_argument( 43 | 'variable', 44 | type=str, 45 | required=False, 46 | help='Variable' 47 | ) 48 | -------------------------------------------------------------------------------- /globomap_api/api/v2/util.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import json 17 | 18 | from jsonspec.validators import load 19 | 20 | 21 | def get_dict(json_file): 22 | 23 | with open(json_file) as data_file: 24 | data = json.load(data_file) 25 | 26 | return data 27 | 28 | 29 | def json_validate(json_file): 30 | 31 | with open(json_file) as data_file: 32 | data = json.load(data_file) 33 | validator = load(data) 34 | 35 | return validator 36 | 37 | 38 | def validate(error): 39 | msg = [] 40 | if error.flatten(): 41 | for pointer, reasons in error.flatten().items(): 42 | msg.append({ 43 | 'error_pointer': pointer, 44 | 'error_reasons': list(reasons) 45 | }) 46 | else: 47 | msg.append({ 48 | 'error_pointer': error[0], 49 | 'error_reasons': list(error[1]) 50 | }) 51 | return msg 52 | 53 | 54 | def filter_transversal(data): 55 | edges = [] 56 | nodes = [] 57 | for p in data.get('paths'): 58 | edges += p.get('edges', []) 59 | nodes += p.get('vertices', []) 60 | for v in data.get('vertices'): 61 | nodes += v.get('vertices', []) 62 | 63 | nodes = list({node['_id']: node for node in nodes}.values()) 64 | edges = list({edge['_id']: edge for edge in edges}.values()) 65 | 66 | data = { 67 | 'nodes': nodes, 68 | 'edges': edges 69 | } 70 | return data 71 | 72 | 73 | def filter_graphs(data): 74 | graphs = [] 75 | for graph in data: 76 | gra = { 77 | 'name': graph.get('name'), 78 | 'alias': graph.get('alias'), 79 | 'icon': graph.get('icon'), 80 | 'description': graph.get('description'), 81 | 'links': graph.get('links'), 82 | } 83 | graphs.append(gra) 84 | return graphs 85 | 86 | 87 | def filter_collections(colls): 88 | collections = [] 89 | for coll in colls: 90 | collections.append({ 91 | 'alias': coll.get('alias'), 92 | 'name': coll.get('name'), 93 | 'kind': coll.get('kind'), 94 | 'icon': coll.get('icon'), 95 | 'description': coll.get('description'), 96 | 'users': coll.get('users'), 97 | }) 98 | return collections 99 | 100 | 101 | def make_key(document): 102 | key = '{}_{}'.format( 103 | document['provider'], 104 | document['id'] 105 | ) 106 | return key 107 | -------------------------------------------------------------------------------- /globomap_api/api_plugins/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /globomap_api/api_plugins/abstract_plugin.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | 18 | class AbstractPlugin(object): 19 | 20 | def get_data(self, params): 21 | raise NotImplementedError() 22 | 23 | 24 | class PluginNotFoundException(Exception): 25 | 26 | pass 27 | 28 | 29 | class PluginError(Exception): 30 | 31 | def __init__(self, message): 32 | super(PluginError, self).__init__(message) 33 | 34 | self.message = message 35 | -------------------------------------------------------------------------------- /globomap_api/api_plugins/plugin_loader.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import configparser 17 | import importlib 18 | import logging 19 | 20 | from flask import current_app as app 21 | 22 | from globomap_api.api_plugins.abstract_plugin import PluginNotFoundException 23 | 24 | 25 | class ApiPluginLoader(object): 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | def load_plugin(self, plugin_name): 30 | plugins_config = configparser.ConfigParser() 31 | plugins_config.read(app.config['API_PLUGINS_CONFIG_FILE']) 32 | 33 | if not plugins_config.has_section(plugin_name): 34 | raise PluginNotFoundException( 35 | 'Plugin {} not found'.format(plugin_name) 36 | ) 37 | 38 | try: 39 | plugin_desc = plugins_config.get(plugin_name, 'description') 40 | plugin_module = plugins_config.get(plugin_name, 'module') 41 | app.logger.debug("Lading '{}'".format(plugin_desc)) 42 | return self.create_plugin_instance(plugin_module) 43 | except: 44 | logging.exception('Error loading api plugin') 45 | raise PluginNotFoundException( 46 | 'It was not possible to load plugin {}'.format(plugin_name) 47 | ) 48 | 49 | def create_plugin_instance(self, class_path): 50 | components = class_path.split('.') 51 | class_name = components[-1] 52 | package_path = '.'.join(components[0:-1]) 53 | plugin_class = getattr( 54 | importlib.import_module(package_path), 55 | class_name 56 | ) 57 | return plugin_class() 58 | -------------------------------------------------------------------------------- /globomap_api/app.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import binascii 17 | import os 18 | from logging import config 19 | 20 | from flask import Flask 21 | 22 | 23 | def create_app(config_module=None): 24 | app = Flask(__name__) 25 | app.secret_key = binascii.hexlify(os.urandom(24)) 26 | app.config.from_object(config_module or os.environ.get('FLASK_CONFIG') or 27 | 'globomap_api.config') 28 | app.config['LOGGER_HANDLER_POLICY'] = 'default' 29 | app.config['LOGGER_NAME'] = 'api' 30 | app.config['BUNDLE_ERRORS'] = True 31 | app.config['RESTPLUS_VALIDATE'] = True 32 | 33 | app.logger 34 | 35 | with app.app_context(): 36 | from globomap_api.api.v2.api import blueprint as api_v2 37 | from globomap_api.models.db import DB 38 | 39 | config.dictConfig(app.config['LOGGING']) 40 | 41 | app.config['ARANGO_CONN'] = DB(app.config) 42 | 43 | app.register_blueprint(api_v2) 44 | 45 | return app 46 | -------------------------------------------------------------------------------- /globomap_api/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import os 17 | 18 | ARANGO_DB = os.getenv('ARANGO_DB') 19 | ARANGO_USERNAME = os.getenv('ARANGO_USERNAME') 20 | ARANGO_PASSWORD = os.getenv('ARANGO_PASSWORD') 21 | ARANGO_PROTOCOL = os.getenv('ARANGO_PROTOCOL') 22 | ARANGO_HOST = os.getenv('ARANGO_HOST') 23 | ARANGO_PORT = os.getenv('ARANGO_PORT') 24 | 25 | MAX_PER_PAGE = int(os.getenv('MAX_PER_PAGE', 100)) 26 | 27 | FLASK_DEBUG = os.getenv('FLASK_DEBUG', False) 28 | 29 | API_PLUGINS_CONFIG_FILE = 'api_plugins' 30 | 31 | PROJECT_ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) 32 | SPECS = { 33 | 'auth': os.path.join(PROJECT_ROOT_PATH, 'specs/auth.json'), 34 | 'documents': os.path.join(PROJECT_ROOT_PATH, 'specs/documents.json'), 35 | 'edges': os.path.join(PROJECT_ROOT_PATH, 'specs/edges.json'), 36 | 'documents_partial': os.path.join(PROJECT_ROOT_PATH, 'specs/documents_partial.json'), 37 | 'edges_partial': os.path.join(PROJECT_ROOT_PATH, 'specs/edges_partial.json'), 38 | 'collections': os.path.join(PROJECT_ROOT_PATH, 'specs/collections.json'), 39 | 'graphs': os.path.join(PROJECT_ROOT_PATH, 'specs/graphs.json'), 40 | 'search': os.path.join(PROJECT_ROOT_PATH, 'specs/search.json'), 41 | 'clear': os.path.join(PROJECT_ROOT_PATH, 'specs/clear.json'), 42 | 'queries': os.path.join(PROJECT_ROOT_PATH, 'specs/queries.json'), 43 | } 44 | 45 | ZABBIX_UI_URL = os.getenv('ZABBIX_UI_URL') 46 | ZABBIX_API_URL = os.getenv('ZABBIX_API_URL') 47 | ZABBIX_API_USER = os.getenv('ZABBIX_API_USER') 48 | ZABBIX_API_PASSWORD = os.getenv('ZABBIX_API_PASSWORD') 49 | 50 | # Keystone 51 | KEYSTONE_USERNAME = os.getenv('KEYSTONE_USERNAME') 52 | KEYSTONE_PASSWORD = os.getenv('KEYSTONE_PASSWORD') 53 | 54 | # Roles 55 | ADMIN = 'globomap_admin' 56 | READ = 'globomap_read' 57 | WRITE = 'globomap_write' 58 | 59 | # Meta Collections 60 | META_COLLECTION = 'meta_collection' 61 | META_GRAPH = 'meta_graph' 62 | META_QUERY = 'meta_query' 63 | INTERNAL_METADATA = 'internal_metadata' 64 | 65 | # Logging 66 | SENTRY_DSN = os.getenv('SENTRY_DSN') 67 | LOGGING = { 68 | 'version': 1, 69 | 'disable_existing_loggers': False, 70 | 'formatters': { 71 | 'verbose': { 72 | 'format': 'level=%(levelname)s timestamp=%(asctime)s module=%(module)s line=%(lineno)d' + 73 | 'message=%(message)s ' 74 | } 75 | }, 76 | 'handlers': { 77 | 'default': { 78 | 'level': 'WARNING', 79 | 'class': 'logging.StreamHandler', 80 | 'stream': 'ext://sys.stdout', 81 | 'formatter': 'verbose', 82 | }, 83 | 'sentry': { 84 | 'level': 'ERROR', 85 | 'class': 'raven.handlers.logging.SentryHandler', 86 | 'dsn': SENTRY_DSN, 87 | }, 88 | }, 89 | 'loggers': { 90 | 'api': { 91 | 'handlers': ['default', 'sentry'], 92 | 'level': 'WARNING', 93 | 'propagate': True 94 | }, 95 | 'werkzeug': {'propagate': True}, 96 | } 97 | } 98 | 99 | # Temporary Troubleshooting kubernetes healthcheck 100 | SIMPLE_HEALTHCHECK = os.getenv('SIMPLE_HEALTHCHECK', False) 101 | # Temporary Troubleshooting kubernetes healthcheck 102 | -------------------------------------------------------------------------------- /globomap_api/errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | DOCUMENT = { 17 | 1210: 'Cannot create document {}, document already created.', 18 | 1202: 'Cannot update document {}, document not found.', 19 | 0: 'Cannot create document {}. Error {}' 20 | } 21 | 22 | DATABASE = { 23 | 1228: 'Database {} not found.', 24 | 1207: 'Cannot create database {}, duplicate name.', 25 | 1203: 'Collection {} not found.', 26 | 0: 'Database {}. Error: {}', 27 | 1: 'Error in search: {}', 28 | } 29 | 30 | COLLECTION = { 31 | 1228: 'Collection {} not found.', 32 | 1207: 'Cannot create collection {}, duplicate name.', 33 | 0: 'Collection {}. Error: {}', 34 | } 35 | 36 | EDGE = { 37 | 1228: 'Edge {} not found.', 38 | 1207: 'Cannot create edge {}, duplicate name.', 39 | 0: 'Edge {}. Error: {}', 40 | } 41 | 42 | GRAPH = { 43 | 1924: 'Graph {} not found.', 44 | 1925: 'Cannot create graph {}, duplicate name.', 45 | 0: 'Graph {}. Error: {}', 46 | 1: 'Error to create edge definition in graph {}. Error: {}' 47 | } 48 | 49 | GRAPH_TRAVERSE = { 50 | 1202: 'Invalid startVertex.', 51 | 0: 'Graph {}. Error: {}', 52 | } 53 | 54 | AQL_QUERY = { 55 | 1501: 'Syntax error in Query.', 56 | 0: 'Eroor in Query. Error: {}', 57 | } 58 | -------------------------------------------------------------------------------- /globomap_api/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | 18 | class DatabaseNotExist(Exception): 19 | 20 | def __init__(self, message): 21 | super(DatabaseNotExist, self).__init__(message) 22 | 23 | self.message = message 24 | 25 | 26 | class DatabaseAlreadyExist(Exception): 27 | 28 | def __init__(self, message): 29 | super(DatabaseAlreadyExist, self).__init__(message) 30 | 31 | self.message = message 32 | 33 | 34 | class DatabaseException(Exception): 35 | 36 | def __init__(self, message): 37 | super(DatabaseException, self).__init__(message) 38 | 39 | self.message = message 40 | 41 | 42 | class CollectionNotExist(Exception): 43 | 44 | def __init__(self, message): 45 | super(CollectionNotExist, self).__init__(message) 46 | 47 | self.message = message 48 | 49 | 50 | class CollectionAlreadyExist(Exception): 51 | 52 | def __init__(self, message): 53 | super(CollectionAlreadyExist, self).__init__(message) 54 | 55 | self.message = message 56 | 57 | 58 | class CollectionException(Exception): 59 | 60 | def __init__(self, message): 61 | super(CollectionException, self).__init__(message) 62 | 63 | self.message = message 64 | 65 | 66 | class EdgeNotExist(Exception): 67 | 68 | def __init__(self, message): 69 | super(EdgeNotExist, self).__init__(message) 70 | 71 | self.message = message 72 | 73 | 74 | class EdgeAlreadyExist(Exception): 75 | 76 | def __init__(self, message): 77 | super(EdgeAlreadyExist, self).__init__(message) 78 | 79 | self.message = message 80 | 81 | 82 | class EdgeException(Exception): 83 | 84 | def __init__(self, message): 85 | super(EdgeException, self).__init__(message) 86 | 87 | self.message = message 88 | 89 | 90 | class DocumentNotExist(Exception): 91 | 92 | def __init__(self, message): 93 | super(DocumentNotExist, self).__init__(message) 94 | 95 | self.message = message 96 | 97 | 98 | class DocumentAlreadyExist(Exception): 99 | 100 | def __init__(self, message): 101 | super(DocumentAlreadyExist, self).__init__(message) 102 | 103 | self.message = message 104 | 105 | 106 | class DocumentException(Exception): 107 | 108 | def __init__(self, message): 109 | super(DocumentException, self).__init__(message) 110 | 111 | self.message = message 112 | 113 | 114 | class GraphNotExist(Exception): 115 | 116 | def __init__(self, message): 117 | super(GraphNotExist, self).__init__(message) 118 | 119 | self.message = message 120 | 121 | 122 | class GraphAlreadyExist(Exception): 123 | 124 | def __init__(self, message): 125 | super(GraphAlreadyExist, self).__init__(message) 126 | 127 | self.message = message 128 | 129 | 130 | class GraphException(Exception): 131 | 132 | def __init__(self, message): 133 | super(GraphException, self).__init__(message) 134 | 135 | self.message = message 136 | 137 | 138 | class GraphTraverseException(Exception): 139 | 140 | def __init__(self, message): 141 | super(GraphTraverseException, self).__init__(message) 142 | 143 | self.message = message 144 | 145 | 146 | class ConstructorException(Exception): 147 | 148 | def __init__(self, message): 149 | super(ConstructorException, self).__init__(message) 150 | 151 | self.message = message 152 | 153 | 154 | class SearchException(Exception): 155 | 156 | def __init__(self, message): 157 | super(SearchException, self).__init__(message) 158 | 159 | self.message = message 160 | self.message = message 161 | 162 | 163 | class QueryException(Exception): 164 | 165 | def __init__(self, message): 166 | super(QueryException, self).__init__(message) 167 | 168 | self.message = message 169 | -------------------------------------------------------------------------------- /globomap_api/models/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /globomap_api/models/constructor.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | # from .db import DB 17 | from flask import current_app as app 18 | 19 | from globomap_api.exceptions import ConstructorException 20 | 21 | 22 | class Constructor(object): 23 | 24 | """Contructor of class of type Collection, Edges or Graph""" 25 | 26 | name = None 27 | kind = None 28 | replication_factor = 1 29 | links = None 30 | create = False 31 | create_indexes = True 32 | 33 | def _treat_param(self, kwargs): 34 | for key in kwargs: 35 | setattr(self, key, kwargs[key]) 36 | 37 | def factory(self, **kwargs): 38 | """Return class of type Collection, Edges or Graph""" 39 | self._treat_param(kwargs) 40 | 41 | if self.kind == 'Collection': 42 | class_fact = self._class_collection_factory() 43 | elif self.kind == 'Edges': 44 | class_fact = self._class_edge_factory() 45 | elif self.kind == 'Graph': 46 | class_fact = self._class_graph_factory() 47 | else: 48 | raise ConstructorException('Kind invalid') 49 | return class_fact 50 | 51 | def _class_collection_factory(self): 52 | 53 | db_inst = app.config['ARANGO_CONN'] 54 | db_inst.get_database() 55 | if self.create: 56 | col = db_inst.create_collection(name=self.name, replication_factor=self.replication_factor) 57 | if self.create_indexes: 58 | self._create_indexes(col) 59 | else: 60 | col = db_inst.get_collection(name=self.name) 61 | return col 62 | 63 | def _class_edge_factory(self): 64 | 65 | db_inst = app.config['ARANGO_CONN'] 66 | db_inst.get_database() 67 | if self.create: 68 | col = db_inst.create_edge(name=self.name, replication_factor=self.replication_factor) 69 | if self.create_indexes: 70 | self._create_indexes(col) 71 | else: 72 | col = db_inst.get_edge(name=self.name) 73 | return col 74 | 75 | def _class_graph_factory(self): 76 | 77 | db_inst = app.config['ARANGO_CONN'] 78 | db_inst.get_database() 79 | if self.create and self.links: 80 | graph = db_inst.create_graph(self.name, self.links) 81 | else: 82 | graph = db_inst.get_graph(self.name) 83 | return graph 84 | 85 | def _create_indexes(self, col): 86 | col.add_hash_index(fields=['name']) 87 | col.add_hash_index(fields=['id']) 88 | col.add_hash_index(fields=['properties']) 89 | col.add_skiplist_index(fields=['timestamp']) 90 | col.add_skiplist_index(fields=['provider']) 91 | -------------------------------------------------------------------------------- /globomap_api/models/document.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from arango import exceptions 17 | from flask import current_app as app 18 | 19 | from globomap_api import exceptions as gmap_exceptions 20 | from globomap_api.errors import DOCUMENT as doc_err 21 | 22 | 23 | class Document: 24 | 25 | def __init__(self, collection): 26 | self.collection = collection 27 | 28 | def create_document(self, document): 29 | """Create Document""" 30 | 31 | try: 32 | return self.collection.insert(document) 33 | except exceptions.DocumentInsertError as err: 34 | 35 | if doc_err.get(err.error_code): 36 | if err.error_code == 1210: 37 | app.logger.warning(err.message) 38 | raise gmap_exceptions.DocumentAlreadyExist( 39 | doc_err.get(err.error_code).format(document['_key'])) 40 | else: 41 | app.logger.error(err.message) 42 | raise gmap_exceptions.DocumentException( 43 | doc_err.get(err.error_code).format(document['_key'])) 44 | 45 | else: 46 | raise gmap_exceptions.DocumentException( 47 | doc_err.get(0).format(document['_key'], err.message)) 48 | 49 | except Exception as err: 50 | if err.error_code == 1200: 51 | msg = 'There is a conflict in document create with key {}'.format(document['_key']) 52 | app.logger.warning(msg) 53 | else: 54 | app.logger.error(err) 55 | raise gmap_exceptions.DocumentException( 56 | doc_err.get(0).format(document['_key'], str(err))) 57 | 58 | def update_document(self, document): 59 | """Update Document""" 60 | 61 | try: 62 | return self.collection.replace(document) 63 | except exceptions.DocumentReplaceError as err: 64 | if err.error_code == 1202: 65 | msg = 'There no document with key {}'.format(document['_key']) 66 | app.logger.warning(msg) 67 | raise gmap_exceptions.DocumentNotExist(msg) 68 | elif err.error_code == 1210: 69 | msg = 'Unique constraint violated in index primary with key {}'.format(document['_key']) 70 | app.logger.warning(msg) 71 | else: 72 | app.logger.error(err.message) 73 | raise gmap_exceptions.DocumentException( 74 | doc_err.get(0).format(document['_key'], err.message)) 75 | 76 | except Exception as err: 77 | if err.error_code == 1200: 78 | msg = 'There is a conflict in document update with key {}'.format(document['_key']) 79 | app.logger.warning(msg) 80 | else: 81 | app.logger.error(err) 82 | raise gmap_exceptions.DocumentException( 83 | doc_err.get(0).format(document['_key'], str(err))) 84 | 85 | def upsert_document(self, document): 86 | """Create/Update Document""" 87 | 88 | try: 89 | document = self.update_document(document) 90 | except gmap_exceptions.DocumentNotExist: 91 | document = self.create_document(document) 92 | else: 93 | return document 94 | 95 | def get_document(self, key): 96 | """Get Document""" 97 | 98 | try: 99 | document = self.collection.get(key) 100 | 101 | except Exception as err: 102 | app.logger.error(err) 103 | raise gmap_exceptions.DocumentException( 104 | doc_err.get(0).format(key, str(err))) 105 | else: 106 | if document is None: 107 | msg = 'There no document with key {}'.format(key) 108 | app.logger.warning(msg) 109 | raise gmap_exceptions.DocumentNotExist(msg) 110 | 111 | return document 112 | 113 | def delete_document(self, key): 114 | """Delete Document""" 115 | 116 | try: 117 | self.collection.delete(key) 118 | except exceptions.DocumentDeleteError as err: 119 | 120 | if err.error_code == 1202: 121 | msg = 'There no document with key {}'.format(key) 122 | app.logger.warning(msg) 123 | raise gmap_exceptions.DocumentNotExist(msg) 124 | 125 | else: 126 | app.logger.error(err.message) 127 | raise gmap_exceptions.DocumentException( 128 | doc_err.get(0).format(key, err.message)) 129 | 130 | except Exception as err: 131 | app.logger.error(err) 132 | raise gmap_exceptions.DocumentException( 133 | doc_err.get(0).format(key, str(err))) 134 | else: 135 | return True 136 | -------------------------------------------------------------------------------- /globomap_api/run.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from os import environ 17 | 18 | from globomap_api.wsgi import application 19 | 20 | if __name__ == '__main__': 21 | application.run( 22 | '0.0.0.0', 23 | int(environ.get('PORT', '5000')), 24 | debug=True, 25 | threaded=True 26 | ) 27 | -------------------------------------------------------------------------------- /globomap_api/scripts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from flask.cli import AppGroup 17 | 18 | from globomap_api import config as app_config 19 | from globomap_api import exceptions 20 | from globomap_api.models.constructor import Constructor 21 | from globomap_api.wsgi import application 22 | meta_cli = AppGroup('meta') 23 | 24 | 25 | @meta_cli.command('create') 26 | def create_meta_collections(): 27 | """Creates meta collections.""" 28 | constructor = Constructor() 29 | 30 | try: 31 | constructor.factory(kind='Collection', create=True, 32 | name=app_config.META_COLLECTION, 33 | create_indexes=False) 34 | except exceptions.CollectionAlreadyExist: 35 | pass 36 | 37 | try: 38 | constructor.factory(kind='Collection', create=True, 39 | name=app_config.META_GRAPH, 40 | create_indexes=False) 41 | except exceptions.CollectionAlreadyExist: 42 | pass 43 | 44 | try: 45 | constructor.factory(kind='Collection', create=True, 46 | name=app_config.META_QUERY, 47 | create_indexes=False) 48 | except exceptions.CollectionAlreadyExist: 49 | pass 50 | 51 | try: 52 | constructor.factory(kind='Collection', create=True, 53 | name=app_config.INTERNAL_METADATA, 54 | create_indexes=False) 55 | except exceptions.CollectionAlreadyExist: 56 | pass 57 | 58 | application.cli.add_command(meta_cli) 59 | -------------------------------------------------------------------------------- /globomap_api/specs/auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "object", 4 | "additionalProperties": false, 5 | "properties": { 6 | "username": { 7 | "type": "string" 8 | }, 9 | "password": { 10 | "type": "string" 11 | } 12 | }, 13 | "required": [ 14 | "username", 15 | "password" 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /globomap_api/specs/clear.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "array", 4 | "items": { 5 | "type": "array", 6 | "items": { 7 | "type": "object", 8 | "additionalProperties": false, 9 | "properties": { 10 | "field": { 11 | "type": "string", 12 | "minimum": 1 13 | }, 14 | "operator": { 15 | "type": "string", 16 | "enum": [ 17 | "LIKE", 18 | "NOTIN", 19 | "IN", 20 | "==", 21 | "!=", 22 | ">", 23 | ">=", 24 | "<", 25 | "<=", 26 | "!~", 27 | "=~" 28 | ] 29 | }, 30 | "value": { 31 | "type": [ 32 | "string", 33 | "boolean", 34 | "integer", 35 | "array", 36 | "object" 37 | ], 38 | "minimum": 1 39 | } 40 | }, 41 | "required": [ 42 | "field", 43 | "operator", 44 | "value" 45 | ] 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /globomap_api/specs/collections.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "object", 4 | "additionalProperties": false, 5 | "properties": { 6 | "name": { 7 | "type": "string", 8 | "pattern": "^([a-z_]){1,}$", 9 | "minimum": 1 10 | }, 11 | "replicationFactor": { 12 | "type": "integer", 13 | "default": 2 14 | }, 15 | "alias": { 16 | "type": "string", 17 | "minimum": 1 18 | }, 19 | "icon": { 20 | "type": "string", 21 | "pattern": "^([a-z_-]){1,}$", 22 | "minimum": 1 23 | }, 24 | "description": { 25 | "type": "string", 26 | "minimum": 1 27 | }, 28 | "users": { 29 | "type": "array", 30 | "items": { 31 | "type": "string", 32 | "pattern": "^([a-z_]){1,}$", 33 | "minimum": 1 34 | }, 35 | "minItems": 1 36 | } 37 | }, 38 | "required": [ 39 | "name", 40 | "alias", 41 | "icon", 42 | "description", 43 | "users" 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /globomap_api/specs/documents.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "object", 4 | "additionalProperties": false, 5 | "properties": { 6 | "id": { 7 | "type": "string", 8 | "pattern": "^([a-zA-Z0-9=._-]){1,}$", 9 | "minimum": 1 10 | }, 11 | "name": { 12 | "type": "string" 13 | }, 14 | "provider": { 15 | "type": "string", 16 | "pattern": "^([a-z_]){1,}$", 17 | "minimum": 1 18 | }, 19 | "timestamp": { 20 | "type": "number" 21 | }, 22 | "properties": { 23 | "type": "object" 24 | }, 25 | "properties_metadata": { 26 | "type": "object" 27 | } 28 | }, 29 | "required": [ 30 | "id", 31 | "provider", 32 | "timestamp" 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /globomap_api/specs/documents_partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "object", 4 | "additionalProperties": false, 5 | "properties": { 6 | "id": { 7 | "type": "string", 8 | "pattern": "^([a-zA-Z0-9=._-]){1,}$", 9 | "minimum": 1 10 | }, 11 | "name": { 12 | "type": "string" 13 | }, 14 | "provider": { 15 | "type": "string", 16 | "pattern": "^([a-z_]){1,}$", 17 | "minimum": 1 18 | }, 19 | "timestamp": { 20 | "type": "number" 21 | }, 22 | "properties": { 23 | "type": "object" 24 | }, 25 | "properties_metadata": { 26 | "type": "object" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /globomap_api/specs/edges.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "from": { 6 | "type": "string", 7 | "pattern": "^(([a-z_]){1,}\/([a-z_]){1,}_([a-zA-Z0-9=._-]){1,})$" 8 | }, 9 | "to": { 10 | "type": "string", 11 | "pattern": "^(([a-z_]){1,}\/([a-z_]){1,}_([a-zA-Z0-9=._-]){1,})$" 12 | }, 13 | "id": { 14 | "type": "string", 15 | "pattern": "^([a-zA-Z0-9=._-]){1,}$", 16 | "minimum": 1 17 | }, 18 | "name": { 19 | "type": "string" 20 | }, 21 | "replicationFactor": { 22 | "type": "integer", 23 | "default": 2 24 | }, 25 | "provider": { 26 | "type": "string", 27 | "pattern": "^([a-z_]){1,}$", 28 | "minimum": 1 29 | }, 30 | "timestamp": { 31 | "type": "number" 32 | }, 33 | "properties": { 34 | "type": "object" 35 | }, 36 | "properties_metadata": { 37 | "type": "object" 38 | } 39 | }, 40 | "required": [ 41 | "id", 42 | "provider", 43 | "timestamp", 44 | "from", 45 | "to" 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /globomap_api/specs/edges_partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "object", 4 | "additionalProperties": false, 5 | "properties": { 6 | "from": { 7 | "type": "string", 8 | "pattern": "^(([a-z_]){1,}\/([a-z_]){1,}_([a-zA-Z0-9=._-]){1,})$" 9 | }, 10 | "to": { 11 | "type": "string", 12 | "pattern": "^(([a-z_]){1,}\/([a-z_]){1,}_([a-zA-Z0-9=._-]){1,})$" 13 | }, 14 | "id": { 15 | "type": "string", 16 | "pattern": "^([a-zA-Z0-9=._-]){1,}$", 17 | "minimum": 1 18 | }, 19 | "name": { 20 | "type": "string" 21 | }, 22 | "provider": { 23 | "type": "string", 24 | "pattern": "^([a-z_]){1,}$", 25 | "minimum": 1 26 | }, 27 | "timestamp": { 28 | "type": "number" 29 | }, 30 | "properties": { 31 | "type": "object" 32 | }, 33 | "properties_metadata": { 34 | "type": "object" 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /globomap_api/specs/graphs.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "object", 4 | "additionalProperties": false, 5 | "properties": { 6 | "name": { 7 | "type": "string", 8 | "pattern": "^([a-z_]){1,}$", 9 | "minimum": 1 10 | }, 11 | "alias": { 12 | "type": "string", 13 | "minimum": 1 14 | }, 15 | "icon": { 16 | "type": "string", 17 | "pattern": "^([a-z_-]){1,}$", 18 | "minimum": 1 19 | }, 20 | "description": { 21 | "type": "string", 22 | "minimum": 1 23 | }, 24 | "links": { 25 | "type": "array", 26 | "items": { 27 | "type": "object", 28 | "additionalProperties": false, 29 | "properties": { 30 | "edge": { 31 | "type": "string" 32 | }, 33 | "from_collections": { 34 | "type": "array", 35 | "items": { 36 | "type": "string" 37 | }, 38 | "minItems": 1 39 | }, 40 | "to_collections": { 41 | "type": "array", 42 | "items": { 43 | "type": "string" 44 | }, 45 | "minItems": 1 46 | } 47 | }, 48 | "required": [ 49 | "edge", 50 | "from_collections", 51 | "to_collections" 52 | ] 53 | } 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /globomap_api/specs/queries.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "object", 4 | "additionalProperties": false, 5 | "properties": { 6 | "name": { 7 | "type": "string" 8 | }, 9 | "description": { 10 | "type": "string" 11 | }, 12 | "query": { 13 | "type": "string" 14 | }, 15 | "params": { 16 | "type": "object" 17 | }, 18 | "collection": { 19 | "type": "string" 20 | } 21 | }, 22 | "required": [ 23 | "name", 24 | "description", 25 | "query", 26 | "params", 27 | "collection" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /globomap_api/specs/search.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "type": "array", 4 | "items": { 5 | "type": "array", 6 | "items": { 7 | "type": "object", 8 | "additionalProperties": false, 9 | "properties": { 10 | "field": { 11 | "type": "string", 12 | "minimum": 1 13 | }, 14 | "operator": { 15 | "type": "string", 16 | "enum": [ 17 | "LIKE", 18 | "NOTIN", 19 | "IN", 20 | "==", 21 | "!=", 22 | ">", 23 | ">=", 24 | "<", 25 | "<=", 26 | "!~", 27 | "=~" 28 | ] 29 | }, 30 | "value": { 31 | "type": [ 32 | "string", 33 | "boolean", 34 | "integer", 35 | "array", 36 | "object" 37 | ], 38 | "minimum": 1 39 | } 40 | }, 41 | "required": [ 42 | "field", 43 | "operator", 44 | "value" 45 | ] 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /globomap_api/util.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import json 17 | 18 | from jsonspec.validators import load 19 | 20 | 21 | def get_dict(json_file): 22 | 23 | with open(json_file) as data_file: 24 | data = json.load(data_file) 25 | 26 | return data 27 | 28 | 29 | def json_validate(json_file): 30 | 31 | with open(json_file) as data_file: 32 | data = json.load(data_file) 33 | validator = load(data) 34 | 35 | return validator 36 | 37 | 38 | def validate(error): 39 | msg = [] 40 | if error.flatten(): 41 | for pointer, reasons in error.flatten().items(): 42 | msg.append({ 43 | 'error_pointer': pointer, 44 | 'error_reasons': list(reasons) 45 | }) 46 | else: 47 | msg.append({ 48 | 'error_pointer': error[0], 49 | 'error_reasons': list(error[1]) 50 | }) 51 | return msg 52 | 53 | 54 | def filter_transversal(data): 55 | edges = [] 56 | nodes = [] 57 | for p in data.get('paths'): 58 | edges += p.get('edges', []) 59 | nodes += p.get('vertices', []) 60 | for v in data.get('vertices'): 61 | nodes += v.get('vertices', []) 62 | 63 | nodes = list({node['_id']: node for node in nodes}.values()) 64 | edges = list({edge['_id']: edge for edge in edges}.values()) 65 | 66 | data = { 67 | 'nodes': nodes, 68 | 'edges': edges 69 | } 70 | return data 71 | 72 | 73 | def filter_graphs(data): 74 | graphs = [] 75 | for graph in data: 76 | gra = { 77 | 'name': graph['name'], 78 | 'links': [] 79 | } 80 | for edge_definition in graph['edge_definitions']: 81 | edge = { 82 | 'edge': edge_definition['collection'], 83 | 'from_collections': edge_definition['from'], 84 | 'to_collections': edge_definition['to'] 85 | } 86 | gra['links'].append(edge) 87 | graphs.append(gra) 88 | return graphs 89 | 90 | 91 | def filter_collections(data, kind): 92 | collections = [coll['name'] for coll in data 93 | if coll['system'] is False and 94 | coll['name'] != 'internal_metadata' and 95 | coll['type'] == kind] 96 | return collections 97 | 98 | 99 | def make_key(document): 100 | key = '{}_{}'.format( 101 | document['provider'], 102 | document['id'] 103 | ) 104 | return key 105 | -------------------------------------------------------------------------------- /globomap_api/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from globomap_api.app import create_app 17 | application = create_app() 18 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==1.0.2 2 | flask-cors==3.0.3 3 | flask-restplus==0.10.1 4 | gevent==1.2.2 5 | globomap-auth-manager==0.0.17 6 | gunicorn==19.7.1 7 | json-spec==0.10.1 8 | py-zabbix==1.1.5 9 | python-arango==4.2.1 10 | raven==6.6.0 11 | globomap-plugin-zabbix==0.1.6 12 | globomap-plugin-healthcheck==0.1.4 13 | Werkzeug==0.16.1 -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | coverage==4.4.1 4 | mock==2.0.0 5 | nose==1.3.7 6 | rednose==1.3.0 7 | unittest2==1.1.0 8 | -------------------------------------------------------------------------------- /scripts/arango/arango-remove.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import os 4 | 5 | from arango import ArangoClient 6 | 7 | 8 | def main(): 9 | conn = ArangoClient( 10 | protocol=os.getenv('ARANGO_PROTOCOL', 'http'), 11 | host=os.getenv('ARANGO_HOST', 'arangodb.globomap.dev.globoi.com'), 12 | port=os.getenv('ARANGO_PORT', '8529') 13 | ) 14 | 15 | database = conn.db( 16 | os.getenv('ARANGO_DB', 'globomap'), 17 | username=os.getenv('ARANGO_USERNAME', 'gmap'), 18 | password=os.getenv('ARANGO_PASSWORD', 'Adm4gmap') 19 | ) 20 | 21 | graphs = [ 22 | "foreman", 23 | "keystone", 24 | "acl", 25 | "dns", 26 | "swift", 27 | "cloud_stack", 28 | "database", 29 | "domain", 30 | "custeio", 31 | "networking_topology", 32 | "permission", 33 | "load_balancing", 34 | "galeb", 35 | "zabbix", 36 | "faas", 37 | "tsuru", 38 | "accounts" 39 | ] 40 | 41 | for graph in graphs: 42 | try: 43 | database.delete_graph(name=graph) 44 | except Exception as err: 45 | print("delete graph '{}' error. Error: {}".format(graph, err)) 46 | else: 47 | print("delete graph '{}' ok...".format(graph)) 48 | 49 | edges = [ 50 | "foreman_host_dns", 51 | "custeio_business_service_comp_unit", 52 | "filer_volume", 53 | "custeio_process_storage", 54 | "database_tsuru_service_instance", 55 | "custeio_process_comp_unit", 56 | "custeio_client_storage", 57 | "galeb_target", 58 | "volume_export", 59 | "custeio_sub_component_comp_unit", 60 | "custeio_business_service_component", 61 | "access", 62 | "custeio_process_business_service", 63 | "environment_vlan", 64 | "pool_comp_unit", 65 | "custeio_component_sub_component", 66 | "internet_access", 67 | "swift_account_tsuru_service_instance", 68 | "tsuru_pool_comp_unit", 69 | "custeio_product_storage", 70 | "vlan_network", 71 | "team_access", 72 | "father_environment", 73 | "galeb_virtual_host_rule", 74 | "tsuru_service_service_instance", 75 | "network_comp_unit", 76 | "custeio_component_storage", 77 | "galeb_rule_pool", 78 | "galeb_environment_virtual_host", 79 | "galeb_environment_vip", 80 | "galeb_host_environment", 81 | "export_snapshot", 82 | "tsuru_pool_app", 83 | "zone_region", 84 | "ks_role", 85 | "custeio_client_comp_unit", 86 | "custeio_sub_component_storage", 87 | "zabbix_link", 88 | "zone_host", 89 | "host_comp_unit", 90 | "tsuru_app_service_instance", 91 | "dns_link", 92 | "foreman_host_puppet_class", 93 | "swift_account_ks_project", 94 | "ldap_group_user", 95 | "port", 96 | "custeio_component_comp_unit", 97 | "dns_domain", 98 | "custeio_business_service_storage", 99 | "database_dns", 100 | "custeio_product_comp_unit", 101 | "comp_unit_database", 102 | "bs_user_ldap_user" 103 | ] 104 | 105 | for edge in edges: 106 | try: 107 | database.delete_collection(name=edge) 108 | except Exception as err: 109 | print("delete edge '{}' error. Error: {}".format(edge, err)) 110 | else: 111 | print("delete edge '{}' ok...".format(edge)) 112 | 113 | collections = [ 114 | "unknown", 115 | "volume", 116 | "foreman_host", 117 | "tsuru_pool", 118 | "comp_unit", 119 | "region", 120 | "ks_user", 121 | "zone", 122 | "zabbix_graph", 123 | "tag_firewall", 124 | "swift_account", 125 | "custeio_process", 126 | "network", 127 | "ldap_user", 128 | "tsuru_app", 129 | "tsuru_service_instance", 130 | "dns", 131 | "database", 132 | "galeb_pool", 133 | "galeb_virtual_host", 134 | "galeb_environment", 135 | "export", 136 | "tsuru_service", 137 | "ks_project", 138 | "ldap_group", 139 | "custeio_resource", 140 | "foreman_puppet_class", 141 | "vlan", 142 | "custeio_product", 143 | "custeio_component", 144 | "environment", 145 | "snapshot", 146 | "vip", 147 | "pool", 148 | "galeb_rule", 149 | "custeio_business_service", 150 | "custeio_client", 151 | "custeio_team", 152 | "custeio_sub_component", 153 | "domain", 154 | "internal_metadata", 155 | "meta_collection", 156 | "meta_graph", 157 | "meta_query", 158 | "bs_user" 159 | ] 160 | 161 | for collection in collections: 162 | try: 163 | database.delete_collection(name=collection) 164 | except Exception as err: 165 | print("delete collection '{}' error. Error: {}".format(collection, err)) 166 | else: 167 | print("delete collection '{}' ok...".format(collection)) 168 | 169 | if __name__== "__main__": 170 | main() 171 | -------------------------------------------------------------------------------- /scripts/docker/api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6-stretch 2 | 3 | RUN DEBIAN_FRONTEND=noninteractive apt-get update 4 | RUN DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common vim 5 | RUN DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential python-dev python3-pip python3-venv 6 | RUN DEBIAN_FRONTEND=noninteractive apt-get install -y git telnet curl 7 | 8 | # update pip 9 | RUN python3.6 -m pip install pip --upgrade 10 | RUN python3.6 -m pip install wheel 11 | 12 | WORKDIR /home 13 | 14 | COPY . . 15 | 16 | RUN pip install -r requirements_test.txt 17 | RUN pip install python-dotenv 18 | -------------------------------------------------------------------------------- /scripts/docker/api/meta_collections.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export LC_ALL=C.UTF-8 3 | export LANG=C.UTF-8 4 | export FLASK_APP=globomap_api/scripts.py 5 | flask meta create 6 | -------------------------------------------------------------------------------- /scripts/docker/api/wait_keystone_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Check keystone service 3 | INTERVAL=5 4 | MAX_ATTEMPTS=30 5 | CHECK_COMMAND="curl -sL -w "%{http_code}" http://globomap_keystone:5000 -o /dev/null" 6 | ATTEMPTS=1 7 | HTTP_RET_CODE=`${CHECK_COMMAND}` 8 | while [ "000" = "${HTTP_RET_CODE}" ] && [ ${ATTEMPTS} -le ${MAX_ATTEMPTS} ]; do 9 | echo "Error connecting to keystone. Attempt ${ATTEMPTS}. Retrying in ${INTERVAL} seconds ..." 10 | sleep ${INTERVAL} 11 | ATTEMPTS=$((ATTEMPTS+1)) 12 | HTTP_RET_CODE=`${CHECK_COMMAND}` 13 | done 14 | 15 | if [ "000" = "${HTTP_RET_CODE}" ]; then 16 | echo "Error connecting to keystone. Aborting ..." 17 | exit 2 18 | fi 19 | 20 | 21 | ./scripts/docker/api/meta_collections.sh 22 | make run 23 | -------------------------------------------------------------------------------- /scripts/docker/ci.Dockerfile: -------------------------------------------------------------------------------- 1 | from docker 2 | 3 | RUN apk add make 4 | RUN apk add bash 5 | RUN apk add python3 6 | RUN apk --update add 'py-pip' && pip install 'docker-compose' 7 | 8 | ADD . /app 9 | 10 | WORKDIR /app 11 | -------------------------------------------------------------------------------- /scripts/docker/expose_ports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "${GMAP_DB_PORT}" ]; then 3 | echo "Environment variable GMAP_DB_PORT is not defined." 4 | echo "Setting default ..." 5 | GMAP_DB_PORT=7001 6 | fi 7 | 8 | if [ -z "${GMAP_API_PORT}" ]; then 9 | echo "Environment variable GMAP_API_PORT is not defined." 10 | echo "Setting default ..." 11 | GMAP_API_PORT=7002 12 | fi 13 | 14 | if [ -z "${GMAP_API_DEBUG_PORT}" ]; then 15 | echo "Environment variable GMAP_API_DEBUG_PORT is not defined." 16 | echo "Setting default ..." 17 | GMAP_API_DEBUG_PORT=7003 18 | fi 19 | 20 | if [ -z "${GMAP_KS_ADM_PORT}" ]; then 21 | echo "Environment variable GMAP_KS_ADM_PORT is not defined." 22 | echo "Setting default ..." 23 | GMAP_KS_ADM_PORT=7004 24 | fi 25 | 26 | if [ -z "${GMAP_KS_PORT}" ]; then 27 | echo "Environment variable GMAP_KS_PORT is not defined." 28 | echo "Setting default ..." 29 | GMAP_KS_PORT=7005 30 | fi 31 | 32 | if [ -z "${GMAP_REDIS_PORT}" ]; then 33 | echo "Environment variable GMAP_REDIS_PORT is not defined." 34 | echo "Setting default ..." 35 | GMAP_REDIS_PORT=7006 36 | fi 37 | 38 | echo "GMAP_DB_PORT=$GMAP_DB_PORT" > .env 39 | echo "GMAP_API_PORT=$GMAP_API_PORT" >> .env 40 | echo "GMAP_API_DEBUG_PORT=$GMAP_API_DEBUG_PORT" >> .env 41 | echo "GMAP_KS_ADM_PORT=$GMAP_KS_ADM_PORT" >> .env 42 | echo "GMAP_KS_PORT=$GMAP_KS_PORT" >> .env 43 | echo "GMAP_REDIS_PORT=$GMAP_REDIS_PORT" >> .env 44 | -------------------------------------------------------------------------------- /scripts/docker/globomap.env: -------------------------------------------------------------------------------- 1 | ARANGO_DB=_system 2 | ARANGO_USERNAME=root 3 | ARANGO_PASSWORD= 4 | ARANGO_PROTOCOL=http 5 | ARANGO_HOST=globomap_db 6 | ARANGO_PORT=8529 7 | 8 | KEYSTONE_AUTH_URL=http://globomap_keystone:5000/v3 9 | KEYSTONE_PROJECT_NAME=Globomap 10 | KEYSTONE_USERNAME=u_globomap_api 11 | KEYSTONE_PASSWORD=u_globomap_api 12 | KEYSTONE_USER_DOMAIN_NAME=default 13 | KEYSTONE_PROJECT_DOMAIN_NAME=default 14 | 15 | REDIS_HOST=globomap_redis 16 | REDIS_PORT=6379 17 | REDIS_PASSWORD=password 18 | USE_REDIS=1 19 | 20 | CORS=* 21 | 22 | PYTHONPATH=:/app/globomap_api/ 23 | -------------------------------------------------------------------------------- /scripts/docker/keystone/Dockerfile: -------------------------------------------------------------------------------- 1 | from krystism/openstack-keystone 2 | 3 | COPY ./scripts/docker/keystone/keystone.sh /home/keystone.sh 4 | 5 | ENV OS_TENANT_NAME admin 6 | ENV OS_USERNAME admin 7 | -------------------------------------------------------------------------------- /scripts/docker/keystone/keystone.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Check keystone service 3 | INTERVAL=5 4 | MAX_ATTEMPTS=10 5 | CHECK_COMMAND="curl -sL -w "%{http_code}" http://0.0.0.0:35357 -o /dev/null" 6 | ATTEMPTS=1 7 | HTTP_RET_CODE=`${CHECK_COMMAND}` 8 | while [ "000" = "${HTTP_RET_CODE}" ] && [ ${ATTEMPTS} -le ${MAX_ATTEMPTS} ]; do 9 | echo "Error connecting to keystone. Attempt ${ATTEMPTS}. Retrying in ${INTERVAL} seconds ..." 10 | sleep ${INTERVAL} 11 | ATTEMPTS=$((ATTEMPTS+1)) 12 | HTTP_RET_CODE=`${CHECK_COMMAND}` 13 | done 14 | 15 | if [ "000" = "${HTTP_RET_CODE}" ]; then 16 | echo "Error connecting to keystone. Aborting ..." 17 | exit 2 18 | fi 19 | 20 | while [ ! -f /root/openrc ] 21 | do 22 | sleep 2 23 | done 24 | cd /root 25 | source openrc 26 | 27 | token_issue() 28 | { 29 | echo "Testing list token issue" 30 | user_id=$(openstack token issue | grep user_id) 31 | if [[ -z "$user_id" ]]; then 32 | sleep 2 33 | token_issue 34 | fi 35 | } 36 | project_list() 37 | { 38 | echo "Testing list projects" 39 | project=$(openstack project list | grep admin) 40 | if [[ -z "$project" ]]; then 41 | sleep 2 42 | project_list 43 | fi 44 | } 45 | user_list() 46 | { 47 | echo "Testing list users" 48 | user=$(openstack user list | grep admin) 49 | if [[ -z "$user" ]]; then 50 | sleep 2 51 | user_list 52 | fi 53 | } 54 | token_issue 55 | project_list 56 | user_list 57 | 58 | ## Creating Project 59 | while : ; do 60 | echo "Creating project GloboMap" 61 | project=$(openstack project list | grep Globomap) 62 | if [[ ! -z "$project" ]]; then 63 | echo "Project created" 64 | break 65 | else 66 | openstack project create --domain default --description "GloboMap" Globomap 67 | sleep 1 68 | fi 69 | done 70 | 71 | ## Creating Users 72 | create_user() 73 | { 74 | user=$1 75 | echo "User $user" 76 | while : ; do 77 | echo "Searching user $user" 78 | user_match=$(openstack user list | grep $user) 79 | if [[ ! -z "$user_match" ]]; then 80 | echo "User created" 81 | break 82 | else 83 | echo "Creating user $user" 84 | openstack user create --domain default $user --password $user 85 | sleep 1 86 | fi 87 | done 88 | } 89 | 90 | create_user u_globomap_api 91 | 92 | ## Creating Roles 93 | create_role() 94 | { 95 | role=$1 96 | echo "Role $role" 97 | while : ; do 98 | echo "Searching role $role" 99 | role_match=$(openstack role list | grep $role) 100 | if [[ ! -z "$role_match" ]]; then 101 | echo "Role created" 102 | break 103 | else 104 | echo "Creating role $role" 105 | openstack role create $role 106 | sleep 1 107 | fi 108 | done 109 | } 110 | 111 | create_role globomap_admin 112 | create_role globomap_read 113 | create_role globomap_write 114 | create_role globomap_loader 115 | 116 | ## Associating Roles 117 | create_assoc_role() 118 | { 119 | user=$1 120 | role=$2 121 | while : ; do 122 | echo "Searching user $user with role $role" 123 | role_match=$(openstack role assignment list --user $user --names | grep $role) 124 | if [[ ! -z "$role_match" ]]; then 125 | echo "Role associated" 126 | break 127 | else 128 | echo "Associating role $role with user $user" 129 | openstack role add --project Globomap --user $user $role 130 | sleep 1 131 | fi 132 | done 133 | } 134 | create_assoc_role u_globomap_api admin 135 | create_assoc_role u_globomap_api globomap_admin 136 | create_assoc_role u_globomap_api globomap_read 137 | create_assoc_role u_globomap_api globomap_write 138 | create_assoc_role u_globomap_api globomap_loader 139 | -------------------------------------------------------------------------------- /scripts/docker/keystone/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | while : ; do 3 | project=$(docker ps | grep globomap_keystone) 4 | if [[ ! -z "$project" ]]; then 5 | break; 6 | else 7 | sleep 1; 8 | fi 9 | done 10 | docker exec globomap_keystone "/home/keystone.sh" 11 | -------------------------------------------------------------------------------- /test_config.py: -------------------------------------------------------------------------------- 1 | TESTING = True 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2017 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /tests/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from globomap_api.config import ADMIN 17 | from globomap_api.config import API_PLUGINS_CONFIG_FILE 18 | from globomap_api.config import ARANGO_DB 19 | from globomap_api.config import ARANGO_HOST 20 | from globomap_api.config import ARANGO_PASSWORD 21 | from globomap_api.config import ARANGO_PORT 22 | from globomap_api.config import ARANGO_PROTOCOL 23 | from globomap_api.config import ARANGO_USERNAME 24 | from globomap_api.config import KEYSTONE_PASSWORD 25 | from globomap_api.config import KEYSTONE_USERNAME 26 | from globomap_api.config import META_COLLECTION 27 | from globomap_api.config import META_GRAPH 28 | from globomap_api.config import META_QUERY 29 | from globomap_api.config import INTERNAL_METADATA 30 | from globomap_api.config import READ 31 | from globomap_api.config import SPECS 32 | from globomap_api.config import WRITE 33 | 34 | 35 | __all__ = [ 36 | 'ADMIN', 'API_PLUGINS_CONFIG_FILE', 'ARANGO_DB', 'ARANGO_HOST', 37 | 'ARANGO_PASSWORD', 'ARANGO_PORT', 'ARANGO_PROTOCOL', 'ARANGO_USERNAME', 38 | 'KEYSTONE_PASSWORD', 'KEYSTONE_USERNAME', 39 | 'META_COLLECTION', 'META_GRAPH', 'META_QUERY', 'INTERNAL_METADATA', 'READ', 'SPECS', 'WRITE', 40 | 'ZABBIX_API_URL', 'ZABBIX_API_USER', 'ZABBIX_API_PASSWORD', 'ZABBIX_UI_URL', 41 | 'LOGGING' 42 | ] 43 | 44 | 45 | ZABBIX_API_URL = 'zabbix_api_url' 46 | ZABBIX_API_USER = 'zabbix_api_user' 47 | ZABBIX_API_PASSWORD = 'zabbix_api_password' 48 | ZABBIX_UI_URL = 'zabbix_ui_url' 49 | 50 | # Logging 51 | LOGGING = { 52 | 'version': 1, 53 | 'disable_existing_loggers': False, 54 | 'formatters': { 55 | 'verbose': { 56 | 'format': 'level=%(levelname)s timestamp=%(asctime)s module=%(module)s line=%(lineno)d' + 57 | 'message=%(message)s ' 58 | } 59 | }, 60 | 'handlers': { 61 | 'default': { 62 | 'level': 'INFO', 63 | 'class': 'logging.StreamHandler', 64 | 'stream': 'ext://sys.stdout', 65 | 'formatter': 'verbose', 66 | }, 67 | }, 68 | 'loggers': { 69 | 'api': { 70 | 'handlers': ['default'], 71 | 'level': 'INFO', 72 | 'propagate': True 73 | }, 74 | 'werkzeug': {'propagate': True}, 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /tests/integration/api/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /tests/integration/api/collections_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import json 17 | 18 | import unittest2 19 | 20 | from globomap_api.app import create_app 21 | from tests.integration.cleanup import cleanup 22 | 23 | 24 | def login_user(self): 25 | return self.client.post( 26 | '/v2/auth/', 27 | data=json.dumps(dict( 28 | username='u_globomap_api', 29 | password='u_globomap_api' 30 | )), 31 | content_type='application/json' 32 | ) 33 | 34 | 35 | class CollectionsTestCase(unittest2.TestCase): 36 | 37 | def setUp(self): 38 | self.app = create_app('tests.config') 39 | self.client = self.app.test_client() 40 | 41 | @classmethod 42 | def setUpClass(cls): 43 | cleanup() 44 | 45 | @classmethod 46 | def tearDownClass(cls): 47 | cleanup() 48 | 49 | def test_order1(self): 50 | """Test list collections empty""" 51 | 52 | login_response = login_user(self) 53 | response = self.client.get( 54 | '/v2/collections/', 55 | headers=dict( 56 | Authorization='Token token=' + json.loads( 57 | login_response.data.decode() 58 | ).get('token') 59 | ) 60 | ) 61 | data = json.loads(response.data.decode()) 62 | 63 | self.assertEqual(response.status_code, 200) 64 | self.assertListEqual(data['collections'], []) 65 | 66 | def test_order2(self): 67 | """Test create collection""" 68 | 69 | login_response = login_user(self) 70 | data = dict( 71 | name='coll', 72 | icon='coll_icon', 73 | description='Test Coll', 74 | alias='Collection', 75 | users=['user_abc'], 76 | ) 77 | response = self.client.post( 78 | '/v2/collections/', 79 | json=data, 80 | headers=dict( 81 | Authorization='Token token=' + json.loads( 82 | login_response.data.decode() 83 | ).get('token') 84 | ) 85 | ) 86 | self.assertEqual(response.status_code, 200) 87 | data = json.loads(response.data.decode()) 88 | 89 | def test_order3(self): 90 | """Test list collections again""" 91 | 92 | login_response = login_user(self) 93 | response = self.client.get( 94 | '/v2/collections/', 95 | headers=dict( 96 | Authorization='Token token=' + json.loads( 97 | login_response.data.decode() 98 | ).get('token') 99 | ) 100 | ) 101 | data = json.loads(response.data.decode()) 102 | self.assertEqual(response.status_code, 200) 103 | collections = [ 104 | { 105 | 'alias': 'Collection', 106 | 'name': 'coll', 107 | 'kind': 'document', 108 | 'icon': 'coll_icon', 109 | 'description': 'Test Coll', 110 | 'users': ['user_abc'] 111 | } 112 | ] 113 | self.assertListEqual(data['collections'], collections) 114 | -------------------------------------------------------------------------------- /tests/integration/api/edges_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import json 17 | 18 | import unittest2 19 | 20 | from globomap_api.app import create_app 21 | from tests.integration.cleanup import cleanup 22 | 23 | 24 | def login_user(self): 25 | return self.client.post( 26 | '/v2/auth/', 27 | data=json.dumps(dict( 28 | username='u_globomap_api', 29 | password='u_globomap_api' 30 | )), 31 | content_type='application/json' 32 | ) 33 | 34 | 35 | class CollectionsTestCase(unittest2.TestCase): 36 | 37 | def setUp(self): 38 | self.app = create_app('tests.config') 39 | self.client = self.app.test_client() 40 | 41 | @classmethod 42 | def setUpClass(cls): 43 | cleanup() 44 | 45 | @classmethod 46 | def tearDownClass(cls): 47 | cleanup() 48 | 49 | def test_order1(self): 50 | """Test list edges empty""" 51 | 52 | login_response = login_user(self) 53 | response = self.client.get( 54 | '/v2/edges/', 55 | headers=dict( 56 | Authorization='Token token=' + json.loads( 57 | login_response.data.decode() 58 | ).get('token') 59 | ) 60 | ) 61 | data = json.loads(response.data.decode()) 62 | 63 | self.assertEqual(response.status_code, 200) 64 | self.assertListEqual(data['collections'], []) 65 | 66 | def test_order2(self): 67 | """Test create collection""" 68 | 69 | login_response = login_user(self) 70 | data = dict( 71 | name='coll', 72 | icon='coll_icon', 73 | description='Test Coll', 74 | alias='Collection', 75 | users=['user_abc'], 76 | ) 77 | response = self.client.post( 78 | '/v2/edges/', 79 | json=data, 80 | headers=dict( 81 | Authorization='Token token=' + json.loads( 82 | login_response.data.decode() 83 | ).get('token') 84 | ) 85 | ) 86 | self.assertEqual(response.status_code, 200) 87 | data = json.loads(response.data.decode()) 88 | 89 | def test_order3(self): 90 | """Test list edges again""" 91 | 92 | login_response = login_user(self) 93 | response = self.client.get( 94 | '/v2/edges/', 95 | headers=dict( 96 | Authorization='Token token=' + json.loads( 97 | login_response.data.decode() 98 | ).get('token') 99 | ) 100 | ) 101 | data = json.loads(response.data.decode()) 102 | self.assertEqual(response.status_code, 200) 103 | edges = [ 104 | { 105 | 'alias': 'Collection', 106 | 'name': 'coll', 107 | 'kind': 'edge', 108 | 'icon': 'coll_icon', 109 | 'description': 'Test Coll', 110 | 'users': ['user_abc'] 111 | } 112 | ] 113 | self.assertListEqual(data['collections'], edges) 114 | -------------------------------------------------------------------------------- /tests/integration/api_plugins/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /tests/integration/api_plugins/plugin_data_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2017 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import json 17 | from unittest.mock import Mock 18 | from unittest.mock import patch 19 | 20 | import unittest2 21 | 22 | from globomap_api.app import create_app 23 | 24 | 25 | class TestPluginData(unittest2.TestCase): 26 | 27 | def setUp(self): 28 | # patch('globomap_api.api_plugins.zabbix.config').start() 29 | self.app = create_app('tests.config') 30 | self.client = self.app.test_client() 31 | 32 | self.hosts = {'result': [{'hostid': 1}]} 33 | self.triggers = {'result': [ 34 | {'triggerid': 1, 'description': 'CPU 99%', 'value': 1}]} 35 | 36 | def test_get_zabbix_data(self): 37 | self._mock_zabbix_api(self.hosts, self.triggers) 38 | self._mock_token() 39 | response = self._post('/v2/plugins/zabbix-triggers/', json={"ips": "10.132.41.183"}) 40 | json_response = json.loads(response.data) 41 | 42 | self.assertEqual(200, response.status_code) 43 | self.assertEqual(1, len(json_response)) 44 | self.assertEqual('CPU 99%', json_response[0]['key']) 45 | self.assertEqual(1, json_response[0]['value']) 46 | 47 | def test_error_response(self): 48 | self._mock_zabbix_api(Exception('Error')) 49 | self._mock_token() 50 | response = self._post('/v2/plugins/zabbix-triggers/', json={"ips": "10.132.41.183"}) 51 | json_response = json.loads(response.data) 52 | 53 | self.assertEqual(500, response.status_code) 54 | self.assertEqual('Error in plugin', json_response['errors']) 55 | 56 | def test_get_data_from_invalid_plugin(self): 57 | self._mock_token() 58 | response = self._post('/v2/plugins/UNKNOW/', json={"ips": "10.132.41.183"}) 59 | json_response = json.loads(response.data) 60 | 61 | self.assertEqual(404, response.status_code) 62 | self.assertEqual('Plugin UNKNOW not found', json_response['errors']) 63 | 64 | def _get(self, uri): 65 | return self.client.get(uri, follow_redirects=True) 66 | 67 | def _post(self, uri, json): 68 | return self.client.post(uri, json=json, follow_redirects=True) 69 | 70 | def _mock_zabbix_api(self, hosts, triggers=None): 71 | py_zabbix_mock = patch( 72 | 'globomap_plugin_zabbix.zabbix.ZabbixAPI').start() 73 | do_request_mock = Mock() 74 | do_request_mock.do_request.side_effect = [hosts, triggers] 75 | py_zabbix_mock.return_value = do_request_mock 76 | return py_zabbix_mock 77 | 78 | def _mock_token(self): 79 | validate_token = patch( 80 | 'globomap_api.api.v2.auth.decorators.validate_token').start() 81 | validate_token.return_value.get_token_data_details.return_value = { 82 | 'roles': [{'name': self.app.config['READ']}]} 83 | -------------------------------------------------------------------------------- /tests/integration/cleanup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2018 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | from globomap_api.app import create_app 17 | from globomap_api.models.db import DB 18 | 19 | 20 | def cleanup(): 21 | app = create_app('tests.config') 22 | db_inst = DB(app.config) 23 | db_name = app.config['ARANGO_DB'] 24 | db_inst.conn_database(db_name) 25 | for col in db_inst.database.collections(): 26 | if 'meta' not in col['name'] and col['system'] is False: 27 | db_inst.database.delete_collection(col['name']) 28 | db_inst.database.collection('meta_collection').truncate() 29 | 30 | 31 | if __name__ == '__main__': 32 | cleanup() 33 | -------------------------------------------------------------------------------- /tests/integration/models/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2017 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /tests/integration/models/collection_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2017 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import unittest2 17 | 18 | from globomap_api import exceptions as gmap_exceptions 19 | from globomap_api.app import create_app 20 | from globomap_api.models.db import DB 21 | 22 | 23 | class TestCollection(unittest2.TestCase): 24 | 25 | def setUp(self): 26 | self.app = create_app('tests.config') 27 | self.db_inst = DB(self.app.config) 28 | 29 | self.conn_db() 30 | self.cleanup() 31 | self.db_inst.database.create_database('test') 32 | 33 | self.db_inst.conn_database('test') 34 | 35 | def tearDown(self): 36 | self.conn_db() 37 | self.cleanup() 38 | 39 | def conn_db(self): 40 | db_name = self.app.config['ARANGO_DB'] 41 | self.db_inst.conn_database(db_name) 42 | 43 | def cleanup(self): 44 | try: 45 | self.db_inst.database.delete_database('test') 46 | except: 47 | pass 48 | 49 | ############## 50 | # COLLECTION # 51 | ############## 52 | def test_get_collection(self): 53 | """Test get collection""" 54 | 55 | with self.app.app_context(): 56 | col_name = 'get_collection' 57 | self.db_inst.create_collection(col_name) 58 | 59 | col = self.db_inst.get_collection(col_name) 60 | self.assertEqual(col.name, col_name) 61 | 62 | def test_create_collection(self): 63 | """Test create collection""" 64 | 65 | with self.app.app_context(): 66 | col_name = 'create_collection' 67 | self.db_inst.create_collection(col_name) 68 | col = self.db_inst.get_collection(col_name) 69 | self.assertEqual(col.name, col_name) 70 | 71 | def test_delete_collection(self): 72 | """Test delete collection""" 73 | 74 | with self.app.app_context(): 75 | col_name = 'delete_collection' 76 | self.db_inst.create_collection(col_name) 77 | self.db_inst.delete_collection(col_name) 78 | with self.assertRaises(gmap_exceptions.CollectionNotExist): 79 | self.db_inst.get_collection(col_name) 80 | 81 | def test_get_collection_not_exists(self): 82 | """Test if get collection not exists""" 83 | 84 | with self.app.app_context(): 85 | col_name = 'collection_not_exist' 86 | with self.assertRaises(gmap_exceptions.CollectionNotExist): 87 | self.db_inst.get_collection(col_name) 88 | 89 | def test_create_collection_duplicated(self): 90 | """Test if create collection with duplicated name""" 91 | 92 | with self.app.app_context(): 93 | col_name = 'collection_duplicated' 94 | self.db_inst.create_collection(col_name) 95 | with self.assertRaises(gmap_exceptions.CollectionAlreadyExist): 96 | self.db_inst.create_collection(col_name) 97 | 98 | def test_delete_collection_not_exists(self): 99 | """Test if delete collection not exists""" 100 | 101 | with self.app.app_context(): 102 | col_name = 'collection_not_exist' 103 | with self.assertRaises(gmap_exceptions.CollectionNotExist): 104 | self.db_inst.delete_collection(col_name) 105 | 106 | ######### 107 | # EDGES # 108 | ######### 109 | def test_get_edge(self): 110 | """Test get edge""" 111 | 112 | with self.app.app_context(): 113 | col_name = 'get_edge' 114 | self.db_inst.create_edge(col_name) 115 | 116 | col = self.db_inst.get_edge(col_name) 117 | self.assertEqual(col.name, col_name) 118 | 119 | def test_create_edge(self): 120 | """Test create edge""" 121 | 122 | with self.app.app_context(): 123 | col_name = 'create_edge' 124 | self.db_inst.create_edge(col_name) 125 | col = self.db_inst.get_edge(col_name) 126 | self.assertEqual(col.name, col_name) 127 | 128 | def test_delete_edge(self): 129 | """Test delete edge""" 130 | 131 | with self.app.app_context(): 132 | col_name = 'delete_edge' 133 | self.db_inst.create_edge(col_name) 134 | self.db_inst.delete_edge(col_name) 135 | with self.assertRaises(gmap_exceptions.EdgeNotExist): 136 | self.db_inst.get_edge(col_name) 137 | 138 | def test_get_edge_not_exists(self): 139 | """Test if get edge not exists""" 140 | 141 | with self.app.app_context(): 142 | col_name = 'edge_not_exist' 143 | with self.assertRaises(gmap_exceptions.EdgeNotExist): 144 | self.db_inst.get_edge(col_name) 145 | 146 | def test_create_edge_duplicated(self): 147 | """Test if create edge with duplicated name""" 148 | 149 | with self.app.app_context(): 150 | col_name = 'edge_duplicated' 151 | self.db_inst.create_edge(col_name) 152 | with self.assertRaises(gmap_exceptions.EdgeAlreadyExist): 153 | self.db_inst.create_edge(col_name) 154 | 155 | def test_delete_edge_not_exists(self): 156 | """Test if delete edge not exists""" 157 | 158 | with self.app.app_context(): 159 | col_name = 'edge_not_exist' 160 | with self.assertRaises(gmap_exceptions.EdgeNotExist): 161 | self.db_inst.delete_edge(col_name) 162 | -------------------------------------------------------------------------------- /tests/integration/models/db_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2017 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import unittest 17 | 18 | from globomap_api.app import create_app 19 | from globomap_api.models.db import DB 20 | 21 | 22 | class TestDB(unittest.TestCase): 23 | 24 | def setUp(self): 25 | self.app = create_app('tests.config') 26 | self.db_inst = DB(self.app.config) 27 | 28 | self.conn_db() 29 | self.cleanup() 30 | self.db_inst.database.create_database('test') 31 | 32 | self.db_inst.conn_database('test') 33 | 34 | def tearDown(self): 35 | self.conn_db() 36 | self.cleanup() 37 | 38 | def conn_db(self): 39 | db_name = self.app.config['ARANGO_DB'] 40 | self.db_inst.conn_database(db_name) 41 | 42 | def cleanup(self): 43 | try: 44 | self.db_inst.database.delete_database('test') 45 | except: 46 | pass 47 | 48 | def test_get_database(self): 49 | """Test get database""" 50 | 51 | col = self.db_inst.get_database('test') 52 | self.assertEqual(col.name, 'test') 53 | -------------------------------------------------------------------------------- /tests/integration/models/document_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2017 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import unittest2 17 | 18 | from globomap_api import exceptions as gmap_exceptions 19 | from globomap_api.app import create_app 20 | from globomap_api.models.db import DB 21 | from globomap_api.models.document import Document 22 | 23 | 24 | class TestDocument(unittest2.TestCase): 25 | 26 | def setUp(self): 27 | self.app = create_app('tests.config') 28 | self.db_inst = DB(self.app.config) 29 | 30 | self.conn_db() 31 | self.cleanup() 32 | self.db_inst.database.create_database('test') 33 | 34 | self.db_inst.conn_database('test') 35 | self.db_inst.database.create_collection('test_collection_db') 36 | self.db_inst.get_collection('test_collection_db') 37 | 38 | def tearDown(self): 39 | self.conn_db() 40 | self.cleanup() 41 | 42 | def conn_db(self): 43 | db_name = self.app.config['ARANGO_DB'] 44 | self.db_inst.conn_database(db_name) 45 | 46 | def cleanup(self): 47 | try: 48 | self.db_inst.database.delete_database('test') 49 | except: 50 | pass 51 | 52 | def test_search_document(self): 53 | """Test search document by property""" 54 | 55 | with self.app.app_context(): 56 | col_name = 'test_collection_db' 57 | self._import_bulk(col_name) 58 | search = [[{'field': 'value', 'operator': '==', 'value': 1}]] 59 | docs = self.db_inst.search_in_collection( 60 | 'test_collection_db', search) 61 | docs = (set(sorted([d['_key'] for d in docs]))) 62 | 63 | self.assertEqual(docs, {'doc04', 'doc05'}) 64 | 65 | def test_get_document(self): 66 | """Test get document""" 67 | 68 | with self.app.app_context(): 69 | self._import_bulk('test_collection_db') 70 | inst_doc = Document(self.db_inst.collection) 71 | doc = inst_doc.get_document('doc04') 72 | doc = {'_key': doc['_key'], 'value': doc['value'], } 73 | 74 | self.assertDictEqual(doc, {'_key': 'doc04', 'value': 1}) 75 | 76 | def test_create_document(self): 77 | """Test create document""" 78 | 79 | with self.app.app_context(): 80 | inst_doc = Document(self.db_inst.collection) 81 | doc = inst_doc.create_document({'_key': 'doc04', 'value': 1}) 82 | doc = {'_key': doc['_key'], '_id': doc['_id'], } 83 | 84 | self.assertDictEqual( 85 | doc, {'_key': 'doc04', '_id': 'test_collection_db/doc04', }) 86 | 87 | def test_get_document_not_exist(self): 88 | """Test get document not existing""" 89 | 90 | with self.app.app_context(): 91 | inst_doc = Document(self.db_inst.collection) 92 | 93 | with self.assertRaises(gmap_exceptions.DocumentNotExist): 94 | inst_doc.get_document('doc04') 95 | 96 | def test_delete_document(self): 97 | """Test delete document""" 98 | 99 | with self.app.app_context(): 100 | col_name = 'test_collection_db' 101 | self._import_bulk(col_name) 102 | 103 | inst_doc = Document(self.db_inst.collection) 104 | inst_doc.delete_document('doc04') 105 | 106 | with self.assertRaises(gmap_exceptions.DocumentNotExist): 107 | inst_doc.get_document('doc04') 108 | 109 | def test_delete_document_not_exist(self): 110 | """Test delee document not existing""" 111 | 112 | with self.app.app_context(): 113 | inst_doc = Document(self.db_inst.collection) 114 | 115 | with self.assertRaises(gmap_exceptions.DocumentNotExist): 116 | inst_doc.delete_document('doc04') 117 | 118 | def _import_bulk(self, col_name): 119 | collection = self.db_inst.database.collection(col_name) 120 | collection.import_bulk([ 121 | {'_key': 'doc04', 'value': 1}, 122 | {'_key': 'doc05', 'value': 1}, 123 | {'_key': 'doc06', 'value': 3}, 124 | ]) 125 | -------------------------------------------------------------------------------- /tests/integration/models/graph_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2017 Globo.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import unittest2 17 | 18 | from globomap_api import exceptions as gmap_exceptions 19 | from globomap_api.app import create_app 20 | from globomap_api.models.db import DB 21 | 22 | graphs = ['test_graph_db'] 23 | 24 | 25 | class TestGraph(unittest2.TestCase): 26 | 27 | def setUp(self): 28 | self.app = create_app('tests.config') 29 | self.db_inst = DB(self.app.config) 30 | 31 | self.conn_db() 32 | self.cleanup() 33 | self.db_inst.database.create_database('test') 34 | 35 | self.db_inst.conn_database('test') 36 | 37 | def tearDown(self): 38 | self.conn_db() 39 | self.cleanup() 40 | 41 | def conn_db(self): 42 | db_name = self.app.config['ARANGO_DB'] 43 | self.db_inst.conn_database(db_name) 44 | 45 | def cleanup(self): 46 | try: 47 | self.db_inst.database.delete_database('test') 48 | except: 49 | pass 50 | 51 | def test_get_graph(self): 52 | """Test get graph""" 53 | 54 | graph_name = 'test_graph_db' 55 | self.db_inst.create_graph(graph_name) 56 | col = self.db_inst.get_graph(graph_name) 57 | self.assertEqual(col.name, graph_name) 58 | 59 | def test_create_graph_without_def(self): 60 | """Test create graph without def""" 61 | 62 | graph_name = 'test_graph_db' 63 | self.db_inst.create_graph(graph_name) 64 | col = self.db_inst.get_graph(graph_name) 65 | self.assertEqual(col.name, graph_name) 66 | self.assertEqual(col.name, graph_name) 67 | 68 | def test_create_graph_one_def(self): 69 | """Test create graph with one def """ 70 | 71 | graph_name = 'test_graph_db' 72 | definitions = [{ 73 | 'edge': 'edge_test', 74 | 'from_collections': ['coll_test'], 75 | 'to_collections': ['coll_test'] 76 | }] 77 | self.db_inst.create_graph(graph_name, definitions) 78 | col = self.db_inst.get_graph(graph_name) 79 | self.assertEqual(col.name, graph_name) 80 | self.assertEqual(col.name, graph_name) 81 | 82 | def test_create_graph_two_def(self): 83 | """Test create 2 graphs with def""" 84 | 85 | graph_name = 'test_graph_db' 86 | definitions = [{ 87 | 'edge': 'edge_test', 88 | 'from_collections': ['coll_test'], 89 | 'to_collections': ['coll_test'] 90 | }, { 91 | 'edge': 'edge_test2', 92 | 'from_collections': ['coll_test2'], 93 | 'to_collections': ['coll_test2'] 94 | }] 95 | self.db_inst.create_graph(graph_name, definitions) 96 | col = self.db_inst.get_graph(graph_name) 97 | self.assertEqual(col.name, graph_name) 98 | 99 | def test_delete_graph(self): 100 | """Test delete graph""" 101 | 102 | graph_name = 'test_graph_db' 103 | self.db_inst.create_graph(graph_name) 104 | self.db_inst.delete_graph(graph_name) 105 | with self.assertRaises(gmap_exceptions.GraphNotExist): 106 | self.db_inst.get_graph(graph_name) 107 | 108 | def test_get_graph_not_exists(self): 109 | """Test if get graph that not exists""" 110 | 111 | graph_name = 'test_graph_db_2' 112 | with self.assertRaises(gmap_exceptions.GraphNotExist): 113 | self.db_inst.get_graph(graph_name) 114 | 115 | def test_create_graph_duplicated(self): 116 | """Test if create graph with duplicated name""" 117 | 118 | graph_name = 'test_graph_db' 119 | self.db_inst.create_graph(graph_name) 120 | with self.assertRaises(gmap_exceptions.GraphAlreadyExist): 121 | self.db_inst.create_graph(graph_name) 122 | 123 | def test_delete_graph_not_exists(self): 124 | """Test if delete graph that not exists""" 125 | 126 | graph_name = 'test_graph_db_2' 127 | with self.assertRaises(gmap_exceptions.GraphNotExist): 128 | self.db_inst.delete_graph(graph_name) 129 | -------------------------------------------------------------------------------- /tsuru.yml: -------------------------------------------------------------------------------- 1 | healthcheck: 2 | path: /v2/healthcheck/ 3 | method: GET 4 | status: 200 5 | match: .*WORKING.* 6 | allowed_failures: 5 7 | use_in_router: true 8 | router_body: WORKING 9 | --------------------------------------------------------------------------------