├── .github ├── ISSUE_TEMPLATE │ ├── 01_question.md │ ├── 02_bug.md │ ├── 03_feature.md │ └── config.yml ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── codeql.yml │ └── parser_ci.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── GUIDELINES.md ├── LICENSE ├── PARSING.md ├── README.md ├── SECURITY.md ├── app.py ├── conf └── conf.env.sample ├── img ├── dash-in-action.gif ├── dashboard.png ├── data-source.png ├── database.png ├── document.png ├── engineering.png ├── ext-monitor.png ├── graph.gif ├── images.txt ├── layout.png ├── monitor.png ├── reference.png ├── requirements.txt ├── servers.png ├── services │ ├── cassandra.svg │ ├── clickhouse.svg │ ├── elasticsearch.svg │ ├── flink.svg │ ├── grafana.svg │ ├── influxdb.svg │ ├── kafka-connect.svg │ ├── kafka.svg │ ├── kafka_connect.svg │ ├── m3aggregator.svg │ ├── m3coordinator.svg │ ├── m3db.svg │ ├── mysql.svg │ ├── opensearch.svg │ ├── pg.svg │ └── redis.svg ├── sql_reference.png ├── table.png ├── tag.png ├── unknown.png ├── user.png └── warning.png ├── lib ├── bindings │ └── utils.js ├── tom-select │ ├── tom-select.complete.min.js │ └── tom-select.css ├── vis-9.0.4 │ ├── vis-network.css │ └── vis-network.min.js └── vis-9.1.2 │ ├── vis-network.css │ └── vis-network.min.js ├── main.py ├── requirements.txt ├── scripts ├── create_mysql_tbl.sql ├── create_mysql_usr.sql ├── create_pg_tbl.sql ├── create_services.sh ├── delete_services.sh └── pg_queries.md ├── src ├── __init__.py ├── add_grafana_dashboard.py ├── backup.py ├── explore_service.py ├── flink.py ├── grafana.py ├── integration.py ├── kafka.py ├── kafka_connect.py ├── mysql.py ├── opensearch.py ├── pg.py ├── pg_store_tbl.sql ├── pyvis_display.py ├── redis.py ├── sql.py └── tag.py └── write_pg.py /.github/ISSUE_TEMPLATE/01_question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: ❓ Ask a question 3 | about: Got stuck or missing something from the docs? Ask away! 4 | --- 5 | 6 | # What can we help you with? 7 | 8 | 9 | 10 | # Where would you expect to find this information? 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/02_bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🐜 Report a bug 3 | about: Spotted a problem? Let us know 4 | --- 5 | 6 | # What happened? 7 | 8 | 9 | 10 | # What did you expect to happen? 11 | 12 | 13 | 14 | # What else do we need to know? 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/03_feature.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 💡 Feature suggestion 3 | about: What would make this even better? 4 | --- 5 | 6 | # What is currently missing? 7 | 8 | 9 | 10 | # How could this be improved? 11 | 12 | 13 | 14 | # Is this a feature you would work on yourself? 15 | 16 | * [ ] I plan to open a pull request for this feature 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Aiven Security Bug Bounty 4 | url: https://hackerone.com/aiven_ltd 5 | about: Our bug bounty program. 6 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | # About this change - What it does 3 | 4 | 5 | 6 | 7 | Resolves: #xxxxx 8 | 9 | # Why this way 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "main" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "main" ] 20 | schedule: 21 | - cron: '34 13 * * 1' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | # Runner size impacts CodeQL analysis time. To learn more, please see: 27 | # - https://gh.io/recommended-hardware-resources-for-running-codeql 28 | # - https://gh.io/supported-runners-and-hardware-resources 29 | # - https://gh.io/using-larger-runners 30 | # Consider using larger runners for possible analysis time improvements. 31 | runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} 32 | timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} 33 | permissions: 34 | actions: read 35 | contents: read 36 | security-events: write 37 | 38 | strategy: 39 | fail-fast: false 40 | matrix: 41 | language: [ 'javascript', 'python' ] 42 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] 43 | # Use only 'java' to analyze code written in Java, Kotlin or both 44 | # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both 45 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 46 | 47 | steps: 48 | - name: Checkout repository 49 | uses: actions/checkout@v3 50 | 51 | # Initializes the CodeQL tools for scanning. 52 | - name: Initialize CodeQL 53 | uses: github/codeql-action/init@v2 54 | with: 55 | languages: ${{ matrix.language }} 56 | # If you wish to specify custom queries, you can do so here or in a config file. 57 | # By default, queries listed here will override any specified in a config file. 58 | # Prefix the list here with "+" to use these queries and those in the config file. 59 | 60 | # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 61 | # queries: security-extended,security-and-quality 62 | 63 | 64 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). 65 | # If this step fails, then you should remove it and run the build manually (see below) 66 | - name: Autobuild 67 | uses: github/codeql-action/autobuild@v2 68 | 69 | # ℹ️ Command-line programs to run using the OS shell. 70 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 71 | 72 | # If the Autobuild fails above, remove it and uncomment the following three lines. 73 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 74 | 75 | # - run: | 76 | # echo "Run, Build Application using script" 77 | # ./location_of_script_within_repo/buildscript.sh 78 | 79 | - name: Perform CodeQL Analysis 80 | uses: github/codeql-action/analyze@v2 81 | with: 82 | category: "/language:${{matrix.language}}" 83 | -------------------------------------------------------------------------------- /.github/workflows/parser_ci.yml: -------------------------------------------------------------------------------- 1 | name: parser CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | permissions: 12 | contents: read 13 | 14 | env: 15 | CONFIG_FILE_PATH: ./conf/conf.env 16 | BASE_URL: https://api.aiven.io 17 | 18 | jobs: 19 | test_parser: 20 | runs-on: ubuntu-latest 21 | strategy: 22 | matrix: 23 | python-version: ["3.8", "3.9", "3.10", "3.11"] 24 | 25 | steps: 26 | - uses: actions/checkout@v3 27 | 28 | - name: Set up Python ${{ matrix.python-version }} 29 | uses: actions/setup-python@v4 30 | with: 31 | python-version: ${{ matrix.python-version }} 32 | 33 | - name: Install dependencies 34 | run: | 35 | python -m pip install --upgrade pip 36 | [[ -f requirements.txt ]] && pip install -r requirements.txt 37 | 38 | - name: Set temporary config file 39 | run: | 40 | cat < $CONFIG_FILE_PATH 41 | #!/bin/bash 42 | PROJECT=${{ vars.PROJECT }} 43 | TOKEN=${{ secrets.TOKEN }} 44 | BASE_URL=$BASE_URL 45 | EOF 46 | 47 | - name: Run main.py script 48 | run: | 49 | python main.py 50 | 51 | - name: Check that the script created the files 52 | run: | 53 | file_name="graph_data.dot" 54 | [[ -f ./$file_name ]] && echo "$file_name created - OK" || exit 1 55 | file_name="graph_data.gml" 56 | [[ -f ./$file_name ]] && echo "$file_name created - OK" || exit 1 57 | file_name="nx.html" 58 | [[ -f ./$file_name ]] && echo "$file_name created - OK" || exit 1 59 | 60 | - name: Run app.py server 61 | run: | 62 | python app.py & 63 | - name: Check that the server is reachable 64 | run: | 65 | head=$(curl --head http://127.0.0.1:8050/ | head -n 1) 66 | [[ $head = *"200 OK"* ]] && echo "$head" || exit 1 67 | 68 | do_something_else: 69 | runs-on: ubuntu-latest 70 | needs: test_parser 71 | steps: 72 | - run: echo "Doing something else" 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | file.* 2 | *test* 3 | conf/conf.env 4 | __pycache__ 5 | certs/* 6 | output.* 7 | graph.py 8 | *.html 9 | *.dot 10 | *.gml 11 | kcat.config 12 | tmp 13 | .vscode 14 | *.log 15 | .ropeproject 16 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | opensource@aiven.io. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Welcome! 2 | 3 | Contributions are very welcome on Aiven Metadata Parser. When contributing please keep this in mind: 4 | 5 | - Open an issue to discuss new bigger features. 6 | - Write code consistent with the project style and make sure the tests are passing. 7 | - Stay in touch with us if we have follow up questions or requests for further changes. 8 | 9 | # Development 10 | 11 | More information about how the metadata parser is currently parsing the instances can be found in the [PARSING document](PARSING.md). 12 | 13 | Guidelines for naming of nodes can be found in [GUIDELINES document](GUIDELINES.md). 14 | 15 | 16 | ## Manual testing 17 | 18 | You can test the metadata parser by executing it over any Aiven project. 19 | If you don't have a project with running services, you can create them with the `create_services.sh` script as defined in the main [README](README.md). 20 | 21 | 22 | # Opening a PR 23 | 24 | - Commit messages should describe the changes, not the filenames. Win our admiration by following 25 | the [excellent advice from Chris Beams](https://chris.beams.io/posts/git-commit/) when composing 26 | commit messages. 27 | - Choose a meaningful title for your pull request. 28 | - The pull request description should focus on what changed and why. 29 | - Check that the tests pass (and add test coverage for your changes if appropriate). 30 | -------------------------------------------------------------------------------- /GUIDELINES.md: -------------------------------------------------------------------------------- 1 | # Network guidelines 2 | 3 | For any service the metadata parser is analysing, it should extract the maximum detail of metadata. 4 | 5 | For any entity you can find, create a node. Entities can be: 6 | 7 | * Indexes 8 | * Table 9 | * namespaces 10 | * topics 11 | * ACLs 12 | 13 | For anything that links two entities create an edge, like 14 | * topic to Apache Kafka® service 15 | * table to index 16 | * index to user 17 | 18 | # Node Id Naming Rules 19 | 20 | The standard naming for the node `id` used so far is: 21 | 22 | ``` 23 | ~~~ 24 | ``` 25 | 26 | e.g. a user `franco` belonging to the Kafka instance `demo-kfk` will generate a node with `id` 27 | 28 | ``` 29 | kafka~demo-kfk~user~franco 30 | ``` 31 | 32 | In case there can be multiple objects with the same name (think a PG table in multiple schemas), we need to prefix the node `id` with all the identifiers making it unique. e.g. 33 | 34 | ``` 35 | pg~~schema~~table~ 36 | ``` 37 | 38 | You can add as many properties to the node as you want, the required ones are: 39 | 40 | * `service_type`: allows an easier filtering of nodes by belonging service type 41 | * `type`: defines the type of node 42 | * `label`: defines what is shown in the graph 43 | 44 | 45 | Apache, Apache Kafka, Kafka, Apache Flink, Flink, Apache Cassandra, and Cassandra are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. ClickHouse is a registered trademark of ClickHouse, Inc. https://clickhouse.com. M3, M3 Aggregator, M3 Coordinator, OpenSearch, PostgreSQL, MySQL, InfluxDB, Grafana, Terraform, and Kubernetes are trademarks and property of their respective owners. *Redis is a trademark of Redis Ltd. and the Redis box logo is a mark of Redis Ltd. Any rights therein are reserved to Redis Ltd. Any use by Aiven is for referential purposes only and does not indicate any sponsorship, endorsement or affiliation between Redis and Aiven. All product and service names used in this website are for identification purposes only and do not imply endorsement. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright (c) 2021 Aiven, Helsinki, Finland. https://aiven.io/ 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /PARSING.md: -------------------------------------------------------------------------------- 1 | # How to Parse services 2 | 3 | This doc explains how the Aiven metadata parser currently parses the services: 4 | 5 | ## Apache Kafka® 6 | 7 | * **topics**: Aiven Client `list_service_topics` 8 | * **users**: Aiven Client `get_service` 9 | * **acl**: Aiven Client `get_service` 10 | 11 | ## Apache Kafka Connect® 12 | 13 | * **connectors**: Aiven Client `list_kafka_connectors` 14 | * **connector**: Parsing JSON of each connector separately 15 | 16 | ## MirrorMaker2® 17 | 18 | To be done 19 | 20 | ## Apache Flink® 21 | 22 | * **tables**: Aiven Client `list_flink_tables` 23 | * **jobs**: Aiven Client `list_flink_jobs` 24 | 25 | ## OpenSearch® 26 | 27 | * **indexes**: Aiven Client `get_service_indexes` 28 | 29 | ## ElasticSearch® 30 | 31 | To be done (phasing out product) 32 | 33 | ## InfluxDB® 34 | 35 | To be done 36 | 37 | ## Grafana® 38 | 39 | * **users**: Aiven Client `get_service` 40 | * **datasources**: REST API `/api/datasources` endpoint 41 | * **dashboards**: REST API `/api/search?dash-folder` endpoint 42 | 43 | ## Cassandra 44 | 45 | To be done 46 | 47 | ## Redis 48 | 49 | To be done 50 | 51 | ## M3DB, M3 Aggregator, M3coordinator 52 | 53 | To be done 54 | 55 | ## MySQL 56 | 57 | To be done 58 | 59 | ## PostgreSQL 60 | 61 | Using psycopg2 62 | 63 | * **databases**: query `SELECT datname FROM pg_database;` 64 | * **namespaces**: query `select catalog_name, schema_name, schema_owner from information_schema.schemata;` 65 | * **tables**: query `SELECT schemaname, tablename, tableowner FROM pg_tables where tableowner <> 'postgres';` 66 | * **users**: query `SELECT * FROM pg_user;` 67 | * **role_table_grants**: query `SELECT grantee, table_schema, table_name, privilege_type,is_grantable FROM information_schema.role_table_grants;` 68 | * **table columns**: query `select table_catalog, table_schema, table_name, column_name, data_type, is_nullable from information_schema.columns where table_schema not in ('information_schema', 'pg_catalog');` 69 | 70 | ## External Endpoints 71 | 72 | Currenty listing the external endpoints only using Aiven Client `get_service_integration_endpoints` 73 | 74 | 75 | Apache, Apache Kafka, Kafka, Apache Flink, Flink, Apache Cassandra, and Cassandra are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. ClickHouse is a registered trademark of ClickHouse, Inc. https://clickhouse.com. M3, M3 Aggregator, M3 Coordinator, OpenSearch, PostgreSQL, MySQL, InfluxDB, Grafana, Terraform, and Kubernetes are trademarks and property of their respective owners. *Redis is a trademark of Redis Ltd. and the Redis box logo is a mark of Redis Ltd. Any rights therein are reserved to Redis Ltd. Any use by Aiven is for referential purposes only and does not indicate any sponsorship, endorsement or affiliation between Redis and Aiven. All product and service names used in this website are for identification purposes only and do not imply endorsement. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Aiven Metadata Parser 2 | ======================== 3 | 4 | Want to get all the juicy metadata out of your Aiven services? Check out the Aiven Metadata Parser! 5 | 6 | Based on the `aiven-client` python library, the Aiven Metadata Parser exports the metadata from your services in order to build a connected graph showing all interconnections. 7 | 8 | ![Graph in action](img/graph.gif) 9 | 10 | Currently the Aiven Metadata Parser is a work in progress and extracts only a limited amount of information and connections. 11 | 12 | Create a set of test services 13 | ============================= 14 | 15 | You can use the `create_services.sh` and `delete_services.sh` to create and destroy a pre-set of services. 16 | The `create_services.sh` uses: 17 | 18 | * [jq](https://stedolan.github.io/jq/) to parse JSON output 19 | * [kcat](https://github.com/edenhill/kcat) for Apache Kafka® 20 | * [psql](https://www.postgresql.org/docs/current/app-psql.html) for PostgreSQL® 21 | * [The Aiven CLI](https://github.com/aiven/aiven-client) to connect and create services 22 | * The Python packages in the `requirements.txt` file to push data to Grafana 23 | 24 | Before starting it, please install the required dependences, login with the [Aiven CLI](https://github.com/aiven/aiven-client) and execute: 25 | 26 | ```bash 27 | ./scripts/create_services.sh 28 | ``` 29 | 30 | Where `` is the name of your Aiven project. 31 | 32 | To delete the services you can call: 33 | 34 | ```bash 35 | ./scripts/delete_services.sh 36 | ``` 37 | 38 | If `` is not passed, the default project will be used. 39 | 40 | Start with the Aiven Metadata Parser 41 | ======================================= 42 | 43 | You need to be on Python 3.7, install the required libraries with: 44 | 45 | ```bash 46 | pip install -r requirements.txt 47 | ``` 48 | 49 | Copy the `conf.env.sample` file to `conf.env` in the `conf` folder and edit the token parameter and the project name from which you want to extract parameters. 50 | If you don't have a project with services already running you can create a sample set of services with the `create_services.sh` file, which requires the `aiven-client` to be installed and the user to be logged in. 51 | 52 | Once `conf.env` is set, you can start the metadata extraction with: 53 | 54 | ```bash 55 | python main.py 56 | ``` 57 | 58 | This will generate: 59 | * A file `graph_data.dot` containing the information in [DOT format](https://graphviz.org/doc/info/lang.html) 60 | * A file `graph_data.gml` containing the information in [GML format](https://en.wikipedia.org/wiki/Geography_Markup_Language) 61 | * A file `nx.html` containing the complete interactive graph 62 | ![Graph in action](img/graph.gif) 63 | * (Disabled for the moment) A file `filtered.html` containing the complete interactive graph filtered on the node with id `pg~demo-pg~schema~public~table~pasta` (this might error out if you don't have such node) 64 | 65 | Furthermore if, after executing the `main.py` you also execute the following: 66 | 67 | ```bash 68 | python app.py 69 | ``` 70 | 71 | The `app.py` reads the `graph_data.gml` file generated at step 1 and creates a Reactive Web Applications with [Plotly](https://plot.ly/python/) and [Dash](https://plot.ly/dash/) (code taken from [here](https://towardsdatascience.com/python-interactive-network-visualization-using-networkx-plotly-and-dash-e44749161ed7)). 72 | 73 | Lastly the following allows you to push the content to a PG database passing the PG URI: 74 | 75 | ``` 76 | python write_pg.py PG_URI 77 | ``` 78 | 79 | **Warning** 80 | The code is a bare minimum product, doesn't cover all services and options and is quite chaotic! It demonstrates the idea and how we could start implementing it. 81 | 82 | Possible issues and solutions 83 | ============ 84 | 85 | - If you run `python app.py` and see an error saying `No such file or directory: 'neato'`, you will have to install `graphviz` on your machine (the package version from pip doesn't seem to work) - find out how to install it [here](https://graphviz.org/download/). 86 | 87 | Contributing 88 | ============ 89 | 90 | Wanna contribute? Check the [CONTRIBUTING file](CONTRIBUTING.md) 91 | 92 | License 93 | ============ 94 | Aiven Metadata Extractor is licensed under the Apache license, version 2.0. Full license text is available in the [LICENSE](LICENSE) file. 95 | 96 | Please note that the project explicitly does not require a CLA (Contributor License Agreement) from its contributors. 97 | 98 | Contact 99 | ============ 100 | Bug reports and patches are very welcome, please post them as GitHub issues and pull requests at https://github.com/aiven/metadata-parser . 101 | To report any possible vulnerabilities or other serious issues please see our [security](SECURITY.md) policy. 102 | 103 | All images under the `src` folder and shown in `nx.html` file are taken from https://www.flaticon.com/ 104 | All images under the `src/services` folder are a property of Aiven 105 | 106 | 107 | Apache, Apache Kafka, Kafka, Apache Flink, Flink, Apache Cassandra, and Cassandra are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. ClickHouse is a registered trademark of ClickHouse, Inc. https://clickhouse.com. M3, M3 Aggregator, M3 Coordinator, OpenSearch, PostgreSQL, MySQL, InfluxDB, Grafana, Terraform, and Kubernetes are trademarks and property of their respective owners. *Redis is a trademark of Redis Ltd. and the Redis box logo is a mark of Redis Ltd. Any rights therein are reserved to Redis Ltd. Any use by Aiven is for referential purposes only and does not indicate any sponsorship, endorsement or affiliation between Redis and Aiven. All product and service names used in this website are for identification purposes only and do not imply endorsement. 108 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | We release patches for security vulnerabilities. Which versions are eligible 6 | receiving such patches depend on the CVSS v3.0 Rating: 7 | 8 | | CVSS v3.0 | Supported Versions | 9 | | --------- | ----------------------------------------- | 10 | | 4.0-10.0 | Most recent release | 11 | 12 | ## Reporting a Vulnerability 13 | 14 | Please report (suspected) security vulnerabilities to our **[bug bounty 15 | program](https://hackerone.com/aiven_ltd)**. You will receive a response from 16 | us within 2 working days. If the issue is confirmed, we will release a patch as 17 | soon as possible depending on impact and complexity. 18 | 19 | ## Qualifying Vulnerabilities 20 | 21 | Any reproducible vulnerability that has a severe effect on the security or 22 | privacy of our users is likely to be in scope for the program. 23 | 24 | We generally **aren't** interested in the following issues: 25 | * Social engineering (e.g. phishing, vishing, smishing) attacks 26 | * Brute force, DoS, text injection 27 | * Missing best practices such as HTTP security headers (CSP, X-XSS, etc.), 28 | email (SPF/DKIM/DMARC records), SSL/TLS configuration. 29 | * Software version disclosure / Banner identification issues / Descriptive 30 | error messages or headers (e.g. stack traces, application or server errors). 31 | * Clickjacking on pages with no sensitive actions 32 | * Theoretical vulnerabilities where you can't demonstrate a significant 33 | security impact with a proof of concept. 34 | -------------------------------------------------------------------------------- /conf/conf.env.sample: -------------------------------------------------------------------------------- 1 | TOKEN= 2 | PROJECT= 3 | BASE_URL=https://api.aiven.io -------------------------------------------------------------------------------- /img/dash-in-action.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/dash-in-action.gif -------------------------------------------------------------------------------- /img/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/dashboard.png -------------------------------------------------------------------------------- /img/data-source.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/data-source.png -------------------------------------------------------------------------------- /img/database.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/database.png -------------------------------------------------------------------------------- /img/document.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/document.png -------------------------------------------------------------------------------- /img/engineering.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/engineering.png -------------------------------------------------------------------------------- /img/ext-monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/ext-monitor.png -------------------------------------------------------------------------------- /img/graph.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/graph.gif -------------------------------------------------------------------------------- /img/images.txt: -------------------------------------------------------------------------------- 1 | All images are taken from https://www.flaticon.com/ -------------------------------------------------------------------------------- /img/layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/layout.png -------------------------------------------------------------------------------- /img/monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/monitor.png -------------------------------------------------------------------------------- /img/reference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/reference.png -------------------------------------------------------------------------------- /img/requirements.txt: -------------------------------------------------------------------------------- 1 | aiven-client 2 | configparser 3 | pyvis 4 | networkx 5 | psycopg2 6 | re -------------------------------------------------------------------------------- /img/servers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/servers.png -------------------------------------------------------------------------------- /img/services/cassandra.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/clickhouse.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/elasticsearch.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/flink.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/grafana.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/influxdb.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/kafka-connect.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/kafka.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/kafka_connect.svg: -------------------------------------------------------------------------------- 1 | icon_kafka_connect -------------------------------------------------------------------------------- /img/services/m3aggregator.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/m3coordinator.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/m3db.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/mysql.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/opensearch.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/pg.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/services/redis.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/sql_reference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/sql_reference.png -------------------------------------------------------------------------------- /img/table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/table.png -------------------------------------------------------------------------------- /img/tag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/tag.png -------------------------------------------------------------------------------- /img/unknown.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/unknown.png -------------------------------------------------------------------------------- /img/user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/user.png -------------------------------------------------------------------------------- /img/warning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/img/warning.png -------------------------------------------------------------------------------- /lib/bindings/utils.js: -------------------------------------------------------------------------------- 1 | function neighbourhoodHighlight(params) { 2 | // console.log("in nieghbourhoodhighlight"); 3 | allNodes = nodes.get({ returnType: "Object" }); 4 | // originalNodes = JSON.parse(JSON.stringify(allNodes)); 5 | // if something is selected: 6 | if (params.nodes.length > 0) { 7 | highlightActive = true; 8 | var i, j; 9 | var selectedNode = params.nodes[0]; 10 | var degrees = 2; 11 | 12 | // mark all nodes as hard to read. 13 | for (let nodeId in allNodes) { 14 | // nodeColors[nodeId] = allNodes[nodeId].color; 15 | allNodes[nodeId].color = "rgba(200,200,200,0.5)"; 16 | if (allNodes[nodeId].hiddenLabel === undefined) { 17 | allNodes[nodeId].hiddenLabel = allNodes[nodeId].label; 18 | allNodes[nodeId].label = undefined; 19 | } 20 | } 21 | var connectedNodes = network.getConnectedNodes(selectedNode); 22 | var allConnectedNodes = []; 23 | 24 | // get the second degree nodes 25 | for (i = 1; i < degrees; i++) { 26 | for (j = 0; j < connectedNodes.length; j++) { 27 | allConnectedNodes = allConnectedNodes.concat( 28 | network.getConnectedNodes(connectedNodes[j]) 29 | ); 30 | } 31 | } 32 | 33 | // all second degree nodes get a different color and their label back 34 | for (i = 0; i < allConnectedNodes.length; i++) { 35 | // allNodes[allConnectedNodes[i]].color = "pink"; 36 | allNodes[allConnectedNodes[i]].color = "rgba(150,150,150,0.75)"; 37 | if (allNodes[allConnectedNodes[i]].hiddenLabel !== undefined) { 38 | allNodes[allConnectedNodes[i]].label = 39 | allNodes[allConnectedNodes[i]].hiddenLabel; 40 | allNodes[allConnectedNodes[i]].hiddenLabel = undefined; 41 | } 42 | } 43 | 44 | // all first degree nodes get their own color and their label back 45 | for (i = 0; i < connectedNodes.length; i++) { 46 | // allNodes[connectedNodes[i]].color = undefined; 47 | allNodes[connectedNodes[i]].color = nodeColors[connectedNodes[i]]; 48 | if (allNodes[connectedNodes[i]].hiddenLabel !== undefined) { 49 | allNodes[connectedNodes[i]].label = 50 | allNodes[connectedNodes[i]].hiddenLabel; 51 | allNodes[connectedNodes[i]].hiddenLabel = undefined; 52 | } 53 | } 54 | 55 | // the main node gets its own color and its label back. 56 | // allNodes[selectedNode].color = undefined; 57 | allNodes[selectedNode].color = nodeColors[selectedNode]; 58 | if (allNodes[selectedNode].hiddenLabel !== undefined) { 59 | allNodes[selectedNode].label = allNodes[selectedNode].hiddenLabel; 60 | allNodes[selectedNode].hiddenLabel = undefined; 61 | } 62 | } else if (highlightActive === true) { 63 | // console.log("highlightActive was true"); 64 | // reset all nodes 65 | for (let nodeId in allNodes) { 66 | // allNodes[nodeId].color = "purple"; 67 | allNodes[nodeId].color = nodeColors[nodeId]; 68 | // delete allNodes[nodeId].color; 69 | if (allNodes[nodeId].hiddenLabel !== undefined) { 70 | allNodes[nodeId].label = allNodes[nodeId].hiddenLabel; 71 | allNodes[nodeId].hiddenLabel = undefined; 72 | } 73 | } 74 | highlightActive = false; 75 | } 76 | 77 | // transform the object into an array 78 | var updateArray = []; 79 | if (params.nodes.length > 0) { 80 | for (let nodeId in allNodes) { 81 | if (allNodes.hasOwnProperty(nodeId)) { 82 | // console.log(allNodes[nodeId]); 83 | updateArray.push(allNodes[nodeId]); 84 | } 85 | } 86 | nodes.update(updateArray); 87 | } else { 88 | // console.log("Nothing was selected"); 89 | for (let nodeId in allNodes) { 90 | if (allNodes.hasOwnProperty(nodeId)) { 91 | // console.log(allNodes[nodeId]); 92 | // allNodes[nodeId].color = {}; 93 | updateArray.push(allNodes[nodeId]); 94 | } 95 | } 96 | nodes.update(updateArray); 97 | } 98 | } 99 | 100 | function filterHighlight(params) { 101 | allNodes = nodes.get({ returnType: "Object" }); 102 | // if something is selected: 103 | if (params.nodes.length > 0) { 104 | filterActive = true; 105 | let selectedNodes = params.nodes; 106 | 107 | // hiding all nodes and saving the label 108 | for (let nodeId in allNodes) { 109 | allNodes[nodeId].hidden = true; 110 | if (allNodes[nodeId].savedLabel === undefined) { 111 | allNodes[nodeId].savedLabel = allNodes[nodeId].label; 112 | allNodes[nodeId].label = undefined; 113 | } 114 | } 115 | 116 | for (let i=0; i < selectedNodes.length; i++) { 117 | allNodes[selectedNodes[i]].hidden = false; 118 | if (allNodes[selectedNodes[i]].savedLabel !== undefined) { 119 | allNodes[selectedNodes[i]].label = allNodes[selectedNodes[i]].savedLabel; 120 | allNodes[selectedNodes[i]].savedLabel = undefined; 121 | } 122 | } 123 | 124 | } else if (filterActive === true) { 125 | // reset all nodes 126 | for (let nodeId in allNodes) { 127 | allNodes[nodeId].hidden = false; 128 | if (allNodes[nodeId].savedLabel !== undefined) { 129 | allNodes[nodeId].label = allNodes[nodeId].savedLabel; 130 | allNodes[nodeId].savedLabel = undefined; 131 | } 132 | } 133 | filterActive = false; 134 | } 135 | 136 | // transform the object into an array 137 | var updateArray = []; 138 | if (params.nodes.length > 0) { 139 | for (let nodeId in allNodes) { 140 | if (allNodes.hasOwnProperty(nodeId)) { 141 | updateArray.push(allNodes[nodeId]); 142 | } 143 | } 144 | nodes.update(updateArray); 145 | } else { 146 | for (let nodeId in allNodes) { 147 | if (allNodes.hasOwnProperty(nodeId)) { 148 | updateArray.push(allNodes[nodeId]); 149 | } 150 | } 151 | nodes.update(updateArray); 152 | } 153 | } 154 | 155 | function selectNode(nodes) { 156 | network.selectNodes(nodes); 157 | neighbourhoodHighlight({ nodes: nodes }); 158 | return nodes; 159 | } 160 | 161 | function selectNodes(nodes) { 162 | network.selectNodes(nodes); 163 | filterHighlight({nodes: nodes}); 164 | return nodes; 165 | } 166 | 167 | function highlightFilter(filter) { 168 | let selectedNodes = [] 169 | let selectedProp = filter['property'] 170 | if (filter['item'] === 'node') { 171 | let allNodes = nodes.get({ returnType: "Object" }); 172 | for (let nodeId in allNodes) { 173 | if (allNodes[nodeId][selectedProp] && filter['value'].includes((allNodes[nodeId][selectedProp]).toString())) { 174 | selectedNodes.push(nodeId) 175 | } 176 | } 177 | } 178 | else if (filter['item'] === 'edge'){ 179 | let allEdges = edges.get({returnType: 'object'}); 180 | // check if the selected property exists for selected edge and select the nodes connected to the edge 181 | for (let edge in allEdges) { 182 | if (allEdges[edge][selectedProp] && filter['value'].includes((allEdges[edge][selectedProp]).toString())) { 183 | selectedNodes.push(allEdges[edge]['from']) 184 | selectedNodes.push(allEdges[edge]['to']) 185 | } 186 | } 187 | } 188 | selectNodes(selectedNodes) 189 | } -------------------------------------------------------------------------------- /lib/tom-select/tom-select.css: -------------------------------------------------------------------------------- 1 | /** 2 | * tom-select.css (v2.0.0-rc.4) 3 | * Copyright (c) contributors 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this 6 | * file except in compliance with the License. You may obtain a copy of the License at: 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under 10 | * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | * ANY KIND, either express or implied. See the License for the specific language 12 | * governing permissions and limitations under the License. 13 | * 14 | */ 15 | .ts-wrapper.plugin-drag_drop.multi > .ts-control > div.ui-sortable-placeholder { 16 | visibility: visible !important; 17 | background: #f2f2f2 !important; 18 | background: rgba(0, 0, 0, 0.06) !important; 19 | border: 0 none !important; 20 | box-shadow: inset 0 0 12px 4px #fff; } 21 | 22 | .ts-wrapper.plugin-drag_drop .ui-sortable-placeholder::after { 23 | content: '!'; 24 | visibility: hidden; } 25 | 26 | .ts-wrapper.plugin-drag_drop .ui-sortable-helper { 27 | box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2); } 28 | 29 | .plugin-checkbox_options .option input { 30 | margin-right: 0.5rem; } 31 | 32 | .plugin-clear_button .ts-control { 33 | padding-right: calc( 1em + (3 * 6px)) !important; } 34 | 35 | .plugin-clear_button .clear-button { 36 | opacity: 0; 37 | position: absolute; 38 | top: 8px; 39 | right: calc(8px - 6px); 40 | margin-right: 0 !important; 41 | background: transparent !important; 42 | transition: opacity 0.5s; 43 | cursor: pointer; } 44 | 45 | .plugin-clear_button.single .clear-button { 46 | right: calc(8px - 6px + 2rem); } 47 | 48 | .plugin-clear_button.focus.has-items .clear-button, 49 | .plugin-clear_button:hover.has-items .clear-button { 50 | opacity: 1; } 51 | 52 | .ts-wrapper .dropdown-header { 53 | position: relative; 54 | padding: 10px 8px; 55 | border-bottom: 1px solid #d0d0d0; 56 | background: #f8f8f8; 57 | border-radius: 3px 3px 0 0; } 58 | 59 | .ts-wrapper .dropdown-header-close { 60 | position: absolute; 61 | right: 8px; 62 | top: 50%; 63 | color: #303030; 64 | opacity: 0.4; 65 | margin-top: -12px; 66 | line-height: 20px; 67 | font-size: 20px !important; } 68 | 69 | .ts-wrapper .dropdown-header-close:hover { 70 | color: black; } 71 | 72 | .plugin-dropdown_input.focus.dropdown-active .ts-control { 73 | box-shadow: none; 74 | border: 1px solid #d0d0d0; } 75 | 76 | .plugin-dropdown_input .dropdown-input { 77 | border: 1px solid #d0d0d0; 78 | border-width: 0 0 1px 0; 79 | display: block; 80 | padding: 8px 8px; 81 | box-shadow: none; 82 | width: 100%; 83 | background: transparent; } 84 | 85 | .ts-wrapper.plugin-input_autogrow.has-items .ts-control > input { 86 | min-width: 0; } 87 | 88 | .ts-wrapper.plugin-input_autogrow.has-items.focus .ts-control > input { 89 | flex: none; 90 | min-width: 4px; } 91 | .ts-wrapper.plugin-input_autogrow.has-items.focus .ts-control > input::-webkit-input-placeholder { 92 | color: transparent; } 93 | .ts-wrapper.plugin-input_autogrow.has-items.focus .ts-control > input::-ms-input-placeholder { 94 | color: transparent; } 95 | .ts-wrapper.plugin-input_autogrow.has-items.focus .ts-control > input::placeholder { 96 | color: transparent; } 97 | 98 | .ts-dropdown.plugin-optgroup_columns .ts-dropdown-content { 99 | display: flex; } 100 | 101 | .ts-dropdown.plugin-optgroup_columns .optgroup { 102 | border-right: 1px solid #f2f2f2; 103 | border-top: 0 none; 104 | flex-grow: 1; 105 | flex-basis: 0; 106 | min-width: 0; } 107 | 108 | .ts-dropdown.plugin-optgroup_columns .optgroup:last-child { 109 | border-right: 0 none; } 110 | 111 | .ts-dropdown.plugin-optgroup_columns .optgroup:before { 112 | display: none; } 113 | 114 | .ts-dropdown.plugin-optgroup_columns .optgroup-header { 115 | border-top: 0 none; } 116 | 117 | .ts-wrapper.plugin-remove_button .item { 118 | display: inline-flex; 119 | align-items: center; 120 | padding-right: 0 !important; } 121 | 122 | .ts-wrapper.plugin-remove_button .item .remove { 123 | color: inherit; 124 | text-decoration: none; 125 | vertical-align: middle; 126 | display: inline-block; 127 | padding: 2px 6px; 128 | border-left: 1px solid #d0d0d0; 129 | border-radius: 0 2px 2px 0; 130 | box-sizing: border-box; 131 | margin-left: 6px; } 132 | 133 | .ts-wrapper.plugin-remove_button .item .remove:hover { 134 | background: rgba(0, 0, 0, 0.05); } 135 | 136 | .ts-wrapper.plugin-remove_button .item.active .remove { 137 | border-left-color: #cacaca; } 138 | 139 | .ts-wrapper.plugin-remove_button.disabled .item .remove:hover { 140 | background: none; } 141 | 142 | .ts-wrapper.plugin-remove_button.disabled .item .remove { 143 | border-left-color: white; } 144 | 145 | .ts-wrapper.plugin-remove_button .remove-single { 146 | position: absolute; 147 | right: 0; 148 | top: 0; 149 | font-size: 23px; } 150 | 151 | .ts-wrapper { 152 | position: relative; } 153 | 154 | .ts-dropdown, 155 | .ts-control, 156 | .ts-control input { 157 | color: #303030; 158 | font-family: inherit; 159 | font-size: 13px; 160 | line-height: 18px; 161 | font-smoothing: inherit; } 162 | 163 | .ts-control, 164 | .ts-wrapper.single.input-active .ts-control { 165 | background: #fff; 166 | cursor: text; } 167 | 168 | .ts-control { 169 | border: 1px solid #d0d0d0; 170 | padding: 8px 8px; 171 | width: 100%; 172 | overflow: hidden; 173 | position: relative; 174 | z-index: 1; 175 | box-sizing: border-box; 176 | box-shadow: none; 177 | border-radius: 3px; 178 | display: flex; 179 | flex-wrap: wrap; } 180 | .ts-wrapper.multi.has-items .ts-control { 181 | padding: calc( 8px - 2px - 0) 8px calc( 8px - 2px - 3px - 0); } 182 | .full .ts-control { 183 | background-color: #fff; } 184 | .disabled .ts-control, 185 | .disabled .ts-control * { 186 | cursor: default !important; } 187 | .focus .ts-control { 188 | box-shadow: none; } 189 | .ts-control > * { 190 | vertical-align: baseline; 191 | display: inline-block; } 192 | .ts-wrapper.multi .ts-control > div { 193 | cursor: pointer; 194 | margin: 0 3px 3px 0; 195 | padding: 2px 6px; 196 | background: #f2f2f2; 197 | color: #303030; 198 | border: 0 solid #d0d0d0; } 199 | .ts-wrapper.multi .ts-control > div.active { 200 | background: #e8e8e8; 201 | color: #303030; 202 | border: 0 solid #cacaca; } 203 | .ts-wrapper.multi.disabled .ts-control > div, .ts-wrapper.multi.disabled .ts-control > div.active { 204 | color: #7d7c7c; 205 | background: white; 206 | border: 0 solid white; } 207 | .ts-control > input { 208 | flex: 1 1 auto; 209 | min-width: 7rem; 210 | display: inline-block !important; 211 | padding: 0 !important; 212 | min-height: 0 !important; 213 | max-height: none !important; 214 | max-width: 100% !important; 215 | margin: 0 !important; 216 | text-indent: 0 !important; 217 | border: 0 none !important; 218 | background: none !important; 219 | line-height: inherit !important; 220 | -webkit-user-select: auto !important; 221 | -moz-user-select: auto !important; 222 | -ms-user-select: auto !important; 223 | user-select: auto !important; 224 | box-shadow: none !important; } 225 | .ts-control > input::-ms-clear { 226 | display: none; } 227 | .ts-control > input:focus { 228 | outline: none !important; } 229 | .has-items .ts-control > input { 230 | margin: 0 4px !important; } 231 | .ts-control.rtl { 232 | text-align: right; } 233 | .ts-control.rtl.single .ts-control:after { 234 | left: 15px; 235 | right: auto; } 236 | .ts-control.rtl .ts-control > input { 237 | margin: 0 4px 0 -2px !important; } 238 | .disabled .ts-control { 239 | opacity: 0.5; 240 | background-color: #fafafa; } 241 | .input-hidden .ts-control > input { 242 | opacity: 0; 243 | position: absolute; 244 | left: -10000px; } 245 | 246 | .ts-dropdown { 247 | position: absolute; 248 | top: 100%; 249 | left: 0; 250 | width: 100%; 251 | z-index: 10; 252 | border: 1px solid #d0d0d0; 253 | background: #fff; 254 | margin: 0.25rem 0 0 0; 255 | border-top: 0 none; 256 | box-sizing: border-box; 257 | box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); 258 | border-radius: 0 0 3px 3px; } 259 | .ts-dropdown [data-selectable] { 260 | cursor: pointer; 261 | overflow: hidden; } 262 | .ts-dropdown [data-selectable] .highlight { 263 | background: rgba(125, 168, 208, 0.2); 264 | border-radius: 1px; } 265 | .ts-dropdown .option, 266 | .ts-dropdown .optgroup-header, 267 | .ts-dropdown .no-results, 268 | .ts-dropdown .create { 269 | padding: 5px 8px; } 270 | .ts-dropdown .option, .ts-dropdown [data-disabled], .ts-dropdown [data-disabled] [data-selectable].option { 271 | cursor: inherit; 272 | opacity: 0.5; } 273 | .ts-dropdown [data-selectable].option { 274 | opacity: 1; 275 | cursor: pointer; } 276 | .ts-dropdown .optgroup:first-child .optgroup-header { 277 | border-top: 0 none; } 278 | .ts-dropdown .optgroup-header { 279 | color: #303030; 280 | background: #fff; 281 | cursor: default; } 282 | .ts-dropdown .create:hover, 283 | .ts-dropdown .option:hover, 284 | .ts-dropdown .active { 285 | background-color: #f5fafd; 286 | color: #495c68; } 287 | .ts-dropdown .create:hover.create, 288 | .ts-dropdown .option:hover.create, 289 | .ts-dropdown .active.create { 290 | color: #495c68; } 291 | .ts-dropdown .create { 292 | color: rgba(48, 48, 48, 0.5); } 293 | .ts-dropdown .spinner { 294 | display: inline-block; 295 | width: 30px; 296 | height: 30px; 297 | margin: 5px 8px; } 298 | .ts-dropdown .spinner:after { 299 | content: " "; 300 | display: block; 301 | width: 24px; 302 | height: 24px; 303 | margin: 3px; 304 | border-radius: 50%; 305 | border: 5px solid #d0d0d0; 306 | border-color: #d0d0d0 transparent #d0d0d0 transparent; 307 | animation: lds-dual-ring 1.2s linear infinite; } 308 | 309 | @keyframes lds-dual-ring { 310 | 0% { 311 | transform: rotate(0deg); } 312 | 100% { 313 | transform: rotate(360deg); } } 314 | 315 | .ts-dropdown-content { 316 | overflow-y: auto; 317 | overflow-x: hidden; 318 | max-height: 200px; 319 | overflow-scrolling: touch; 320 | scroll-behavior: smooth; } 321 | 322 | .ts-hidden-accessible { 323 | border: 0 !important; 324 | clip: rect(0 0 0 0) !important; 325 | -webkit-clip-path: inset(50%) !important; 326 | clip-path: inset(50%) !important; 327 | height: 1px !important; 328 | overflow: hidden !important; 329 | padding: 0 !important; 330 | position: absolute !important; 331 | width: 1px !important; 332 | white-space: nowrap !important; } 333 | 334 | /*# sourceMappingURL=tom-select.css.map */ -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from aiven.client import client 2 | from src import explore_service 3 | import src.pyvis_display as pyvis_display 4 | import configparser 5 | 6 | # Reading conf.env configuration file 7 | with open("conf/conf.env", "r") as f: 8 | config_string = "[DEFAULT]\n" + f.read() 9 | config = configparser.ConfigParser() 10 | config.read_string(config_string) 11 | 12 | # Creating Aiven client instance 13 | myclient = client.AivenClient(base_url=config["DEFAULT"]["BASE_URL"]) 14 | 15 | # Authenticating and storing the token 16 | # result = myclient.authenticate_user(email=config['DEFAULT']['USERNAME'], password=config['DEFAULT']['PASSWORD']) 17 | myclient.auth_token = config["DEFAULT"]["TOKEN"] 18 | 19 | # Creating empty nodes and edges lists 20 | nodes = [] 21 | edges = [] 22 | 23 | # The service order helps analysis first "standalone" services and then connection services, this might be useful to parse first services which can be either source or sink of other services (e.g. Kafka connect might use a PG table as a source) 24 | 25 | services_order = {} 26 | services_order["opensearch"] = 1 27 | services_order["elasticsearch"] = 1 28 | services_order["pg"] = 1 29 | services_order["redis"] = 1 30 | services_order["mysql"] = 1 31 | services_order["clickhouse"] = 1 32 | services_order["cassandra"] = 1 33 | services_order["redis"] = 1 34 | services_order["m3db"] = 1 35 | services_order["m3aggregator"] = 1 36 | services_order["m3coordinator"] = 1 37 | 38 | services_order["influxdb"] = 1 39 | services_order["kafka"] = 2 40 | services_order["kafka_connect"] = 3 41 | services_order["kafka_mirrormaker"] = 3 42 | services_order["flink"] = 3 43 | services_order["grafana"] = 3 44 | 45 | 46 | # Listing the services 47 | services = myclient.get_services(project=config["DEFAULT"]["PROJECT"]) 48 | 49 | # Ordering based on service_order 50 | services.sort(key=lambda x: services_order[x["service_type"]]) 51 | 52 | 53 | # Initial loop to find all ip/hostname of existing services 54 | print("Locate IP/hostname of each service") 55 | for i, service in enumerate(services, start=1): 56 | # if service["service_name"]!='test': 57 | print( 58 | f"{i}/{len(services)} {service['service_name']} {service['service_type']}" 59 | ) 60 | # if service["service_type"]=='grafana': 61 | explore_service.populate_service_map( 62 | myclient, 63 | service["service_type"], 64 | service["service_name"], 65 | project=config["DEFAULT"]["PROJECT"], 66 | ) 67 | 68 | # Second loop to find details of each service 69 | print() 70 | print("Find details of each service") 71 | for i, service in enumerate(services, start=1): 72 | print( 73 | f"{i}/{len(services)} Query {service['service_name']} {service['service_type']}" 74 | ) 75 | # if service["service_name"] != 'test': 76 | (newnodes, newedges) = explore_service.explore( 77 | myclient, 78 | service["service_type"], 79 | service["service_name"], 80 | project=config["DEFAULT"]["PROJECT"], 81 | ) 82 | nodes = nodes + newnodes 83 | edges = edges + newedges 84 | 85 | (newnodes, newedges) = explore_service.explore_ext_endpoints( 86 | myclient, project=config["DEFAULT"]["PROJECT"] 87 | ) 88 | nodes = nodes + newnodes 89 | edges = edges + newedges 90 | 91 | # Creating viz with pyviz 92 | pyvis_display.pyviz_graphy(nodes, edges) 93 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary 2 | requests 3 | simplejson 4 | aiven.client 5 | configparser 6 | pyvis==0.3.1 7 | networkx==2.8.8 8 | dash 9 | plotly 10 | pymysql 11 | pydot 12 | colour 13 | cryptography 14 | sqllineage 15 | numpy 16 | -------------------------------------------------------------------------------- /scripts/create_mysql_tbl.sql: -------------------------------------------------------------------------------- 1 | create table stock (stock_id int primary key, stock_name varchar(100)); 2 | insert into stock(stock_id, stock_name) values (1, 'Ferrari'),(2,'Mercedes'),(3,'Apple'); 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /scripts/create_mysql_usr.sql: -------------------------------------------------------------------------------- 1 | CREATE USER 'mytestuser' IDENTIFIED BY 'password123'; 2 | 3 | GRANT SELECT, SHOW VIEW ON defaultdb.* TO 'mytestuser'; -------------------------------------------------------------------------------- /scripts/create_pg_tbl.sql: -------------------------------------------------------------------------------- 1 | create table pasta (pasta_id serial, pasta_name varchar(100), cooking_minutes int, primary key(pasta_id)); 2 | insert into pasta(pasta_name, cooking_minutes) values ('pennette',8),('fusilli',7),('spaghetti',9); 3 | 4 | create table pasta_eater(pasta_id int, eater_name varchar(100), constraint pasta_exitst_fk foreign key(pasta_id) references pasta(pasta_id)); 5 | insert into pasta_eater values(1, 'Francesco'), (2, 'Ewelina'), (3, 'Lorna'); 6 | 7 | 8 | create view vw_pasta_view AS 9 | select pasta.pasta_id, pasta_name, cooking_minutes, eater_name 10 | from pasta join pasta_eater on pasta.pasta_id = pasta_eater.pasta_id; 11 | -------------------------------------------------------------------------------- /scripts/delete_services.sh: -------------------------------------------------------------------------------- 1 | PROJECT_NAME=$1 2 | . conf/conf.env 3 | 4 | 5 | avn --auth-token $TOKEN project switch $PROJECT_NAME 6 | 7 | 8 | avn --auth-token $TOKEN service terminate demo-kafka --force 9 | avn --auth-token $TOKEN service terminate demo-flink --force 10 | avn --auth-token $TOKEN service terminate demo-pg --force 11 | avn --auth-token $TOKEN service terminate demo-mysql --force 12 | avn --auth-token $TOKEN service terminate demo-opensearch --force 13 | avn --auth-token $TOKEN service terminate demo-grafana --force 14 | avn --auth-token $TOKEN service terminate demo-kafka-connect --force 15 | avn --auth-token $TOKEN service terminate demo-redis --force 16 | -------------------------------------------------------------------------------- /scripts/pg_queries.md: -------------------------------------------------------------------------------- 1 | # Check the Nodes 2 | 3 | ``` 4 | select id, json_content from metadata_parser_nodes where id='pg~demo-pg~database~defaultdb'; 5 | ``` 6 | 7 | ``` 8 | select json_content from metadata_parser_nodes where id='service_type~pg~service_name~demo-pg~backup~2022-10-11_07-01_0.00000000.pghoard'; 9 | ``` 10 | 11 | ``` 12 | select json_content from metadata_parser_nodes where id='opensearch~demo-opensearch~index~my_pg_source.public.pasta'; 13 | ``` 14 | 15 | # Check the Edges 16 | 17 | ``` 18 | select * from metadata_parser_edges; 19 | ``` 20 | 21 | # Explain filtering not using index 22 | 23 | ``` 24 | explain select id, json_content from metadata_parser_nodes where 25 | json_content ->> 'type' = 'user'; 26 | ``` 27 | 28 | # Explain filtering using index 29 | ``` 30 | explain select id, json_content from metadata_parser_nodes where 31 | json_content @> '{"type": "user"}'; 32 | ``` 33 | 34 | # Recursive query to get the users who can interact with the `pasta` TABLE 35 | 36 | ``` 37 | with recursive paths (id, last_label, last_type, last_service_type, list_of_edges, nr_items) as ( 38 | select 39 | id, 40 | json_content ->> 'label', 41 | json_content ->> 'type', 42 | json_content ->> 'service_type', 43 | ARRAY[((n.json_content ->> 'type') || ':' || (n.json_content ->> 'label'))], 44 | 1 45 | from metadata_parser_nodes n 46 | where json_content ->> 'label' = 'pasta' 47 | UNION ALL 48 | select 49 | n.id, 50 | n.json_content ->> 'label', 51 | n.json_content ->> 'type', 52 | n.json_content ->> 'service_type', 53 | list_of_edges || ((n.json_content ->> 'type') || ':' || (n.json_content ->> 'label')), 54 | nr_items + 1 55 | from paths p 56 | join metadata_parser_edges e on p.id = e.source_id 57 | join metadata_parser_nodes n on e.destination_id = n.id 58 | where 59 | 1=1 60 | and n.json_content ->> 'type' <> 'service' 61 | ) CYCLE id SET is_cycle USING items_ids 62 | select last_label, last_type, last_service_type, list_of_edges from paths where is_cycle = False and last_type = 'user' order by nr_items; 63 | ``` -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/metadata-parser/b5585adbea2642f33fe4ed7f5c7d35b01365c367/src/__init__.py -------------------------------------------------------------------------------- /src/add_grafana_dashboard.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | import requests 3 | import json 4 | import sys 5 | from aiven.client import client 6 | 7 | with open('conf/conf.env', 'r') as f: 8 | config_string = '[DEFAULT]\n' + f.read() 9 | config = configparser.ConfigParser() 10 | config.read_string(config_string) 11 | 12 | 13 | aiven_client = client.AivenClient(base_url=config['DEFAULT']['BASE_URL']) 14 | 15 | #result = aiven_client.authenticate_user(email=config['DEFAULT']['USERNAME'], password=config['DEFAULT']['PASSWORD']) 16 | aiven_client.auth_token=config['DEFAULT']['TOKEN'] 17 | 18 | project=config['DEFAULT']['PROJECT'] 19 | prj = sys.argv[1] 20 | 21 | if prj: 22 | project = prj 23 | 24 | service = aiven_client.get_service(project=project, service="demo-grafana") 25 | users = service["users"] 26 | 27 | password = "fake" 28 | for user in users: 29 | if user["username"]== "avnadmin": 30 | password = user["password"] 31 | 32 | base_url = "https://"+service["service_uri_params"]["host"]+":443" 33 | datasources = requests.get(base_url+"/api/datasources", auth=("avnadmin", password)) 34 | for datasource in json.loads(datasources.text): 35 | uid = -1 36 | if datasource["name"] == "aiven-pg-demo-pg": 37 | uid = datasource["uid"] 38 | content='''{ 39 | 40 | "dashboard": { 41 | "annotations": { 42 | "list": [ 43 | { 44 | "builtIn": 1, 45 | "datasource": "-- Grafana --", 46 | "enable": true, 47 | "hide": true, 48 | "iconColor": "rgba(0, 211, 255, 1)", 49 | "name": "Annotations & Alerts", 50 | "target": { 51 | "limit": 100, 52 | "matchAny": false, 53 | "tags": [], 54 | "type": "dashboard" 55 | }, 56 | "type": "dashboard" 57 | } 58 | ] 59 | }, 60 | "editable": true, 61 | "fiscalYearStartMonth": 0, 62 | "graphTooltip": 0, 63 | "id": null, 64 | "links": [], 65 | "liveNow": false, 66 | "panels": [ 67 | { 68 | "datasource": { 69 | "type": "postgres", 70 | "uid": "REPLACE_UID" 71 | }, 72 | "fieldConfig": { 73 | "defaults": { 74 | "color": { 75 | "mode": "thresholds" 76 | }, 77 | "custom": { 78 | "align": "auto", 79 | "displayMode": "auto" 80 | }, 81 | "mappings": [], 82 | "thresholds": { 83 | "mode": "absolute", 84 | "steps": [ 85 | { 86 | "color": "green", 87 | "value": null 88 | }, 89 | { 90 | "color": "red", 91 | "value": 80 92 | } 93 | ] 94 | } 95 | }, 96 | "overrides": [] 97 | }, 98 | "gridPos": { 99 | "h": 9, 100 | "w": 12, 101 | "x": 0, 102 | "y": 0 103 | }, 104 | "id": 2, 105 | "options": { 106 | "footer": { 107 | "fields": "", 108 | "reducer": [ 109 | "sum" 110 | ], 111 | "show": false 112 | }, 113 | "showHeader": true 114 | }, 115 | "pluginVersion": "8.3.2", 116 | "targets": [ 117 | { 118 | "datasource": { 119 | "type": "postgres", 120 | "uid": "REPLACE_UID" 121 | }, 122 | "format": "time_series", 123 | "group": [], 124 | "hide": false, 125 | "metricColumn": "pasta_name", 126 | "rawQuery": false, 127 | "rawSql": "SELECT NOW(), pasta_name AS metric, cooking_minutes FROM pasta ORDER BY 1,2", 128 | "refId": "A", 129 | "select": [ 130 | [ 131 | { 132 | "params": [ 133 | "cooking_minutes" 134 | ], 135 | "type": "column" 136 | } 137 | ] 138 | ], 139 | "table": "pasta", 140 | "timeColumn": "NOW()", 141 | "where": [] 142 | } 143 | ], 144 | "title": "Panel Title", 145 | "type": "table" 146 | } 147 | ], 148 | "schemaVersion": 33, 149 | "style": "dark", 150 | "tags": [], 151 | "templating": { 152 | "list": [] 153 | }, 154 | "time": { 155 | "from": "now-12h", 156 | "to": "now" 157 | }, 158 | "timepicker": {}, 159 | "timezone": "", 160 | "title": "my_dashboard", 161 | "uid": "kq-3gR-7k", 162 | "weekStart": "" 163 | }, 164 | "id": null, 165 | "overwrite": true 166 | }''' 167 | 168 | headers={"Content-Type": 'application/json'} 169 | rep = requests.post(base_url+"/api/dashboards/import", auth=("avnadmin", password), data=content.replace('REPLACE_UID',uid),headers=headers) 170 | print(rep.text) 171 | 172 | -------------------------------------------------------------------------------- /src/backup.py: -------------------------------------------------------------------------------- 1 | """Parsing service backups""" 2 | 3 | 4 | def explore_backups(self, service_name, service_type, project): 5 | """Parsing service backups""" 6 | 7 | nodes = [] 8 | edges = [] 9 | 10 | backups = self.get_service_backups(service=service_name, project=project) 11 | for backup in backups: 12 | nodes.append( 13 | { 14 | "id": "service_type~" 15 | + service_type 16 | + "~service_name~" 17 | + service_name 18 | + "~backup~" 19 | + backup["backup_name"], 20 | "label": "backup-" + backup["backup_time"], 21 | "type": "backup", 22 | "service_type": service_type, 23 | **backup, 24 | } 25 | ) 26 | edges.append( 27 | { 28 | "from": service_name, 29 | "label": "backup", 30 | "to": "service_type~" 31 | + service_type 32 | + "~service_name~" 33 | + service_name 34 | + "~backup~" 35 | + backup["backup_name"], 36 | } 37 | ) 38 | return nodes, edges 39 | -------------------------------------------------------------------------------- /src/explore_service.py: -------------------------------------------------------------------------------- 1 | """Explores a service""" 2 | 3 | from src import ( 4 | backup, 5 | kafka, 6 | kafka_connect, 7 | pg, 8 | tag, 9 | integration, 10 | grafana, 11 | redis, 12 | mysql, 13 | opensearch, 14 | flink, 15 | ) 16 | 17 | 18 | SERVICE_MAP = {} 19 | EXPLORER_METHODS = {} 20 | 21 | 22 | def add_explorer(service_type): 23 | """Register an explorer method for the given service_type""" 24 | 25 | def adder(method): 26 | global EXPLORER_METHODS 27 | EXPLORER_METHODS[service_type] = method 28 | return method 29 | 30 | return adder 31 | 32 | 33 | # This method parses all services to indentify internal IPs or hostnames 34 | 35 | 36 | def populate_service_map(self, service_type, service_name, project): 37 | """Populates the service map""" 38 | global SERVICE_MAP 39 | 40 | service = self.get_service(project=project, service=service_name) 41 | if service["state"] != "RUNNING": 42 | return 43 | 44 | try: 45 | if service_type == "kafka": 46 | SERVICE_MAP[service["service_uri_params"]["host"]] = service_name 47 | for url in service["connection_info"]["kafka"]: 48 | host = url.split(":")[0] 49 | SERVICE_MAP[host] = service_name 50 | elif service_type == "flink": 51 | SERVICE_MAP[service["service_uri_params"]["host"]] = service_name 52 | for url in service["connection_info"]["flink"]: 53 | host = url.split(":")[0] 54 | SERVICE_MAP[host] = service_name 55 | elif service_type == "pg": 56 | SERVICE_MAP[ 57 | service["connection_info"]["pg_params"][0]["host"] 58 | ] = service_name 59 | for component in service["components"]: 60 | if component["component"] == "pg": 61 | SERVICE_MAP[component["host"]] = service_name 62 | elif service_type == "mysql": 63 | SERVICE_MAP[ 64 | service["connection_info"]["mysql_params"][0]["host"] 65 | ] = service_name 66 | elif service_type in [ 67 | "grafana", 68 | "opensearch", 69 | "elasticsearch", 70 | "kafka_connect", 71 | "mirrormaker", 72 | "clickhouse", 73 | "cassandra", 74 | "redis", 75 | "m3db", 76 | "m3aggregator", 77 | "m3coordinator", 78 | "influxdb", 79 | ]: 80 | host = service["service_uri_params"]["host"] 81 | SERVICE_MAP[host] = service_name 82 | print(host) 83 | else: 84 | print( 85 | f"Ignoring RUNNING {service_type} service {service_name}" 86 | f"with unrecognised type {service_type}" 87 | ) 88 | except KeyError as err: 89 | print( 90 | f"Error looking up host for RUNNING {service_type}" 91 | f"service {service_name}: {err}" 92 | ) 93 | 94 | 95 | def explore(self, service_type, service_name, project): 96 | """Explores a service""" 97 | edges = [] 98 | nodes = [] 99 | global SERVICE_MAP 100 | host = "no-host" 101 | service = self.get_service(project=project, service=service_name) 102 | if service["state"] != "RUNNING": 103 | return nodes, edges 104 | 105 | try: 106 | explorer_fn = EXPLORER_METHODS[service_type] 107 | except KeyError: 108 | print( 109 | f"Don't know how to explore RUNNING {service_type}" 110 | f"service {service_name}" 111 | ) 112 | 113 | return nodes, edges 114 | 115 | try: 116 | cloud = service["cloud_name"] 117 | plan = service["plan"] 118 | host, port, new_nodes, new_edges, SERVICE_MAP = explorer_fn( 119 | self, service, service_name, project 120 | ) 121 | 122 | nodes = nodes + new_nodes 123 | edges = edges + new_edges 124 | 125 | # Setting the node for the service 126 | nodes.append( 127 | { 128 | "id": service_name, 129 | "host": host, 130 | "port": port, 131 | "cloud": cloud, 132 | "plan": plan, 133 | "service_type": service_type, 134 | "type": "service", 135 | "label": service_name, 136 | } 137 | ) 138 | 139 | # Getting components 140 | 141 | (new_nodes, new_edges) = explore_components( 142 | service_name, service_type, service["components"] 143 | ) 144 | nodes = nodes + new_nodes 145 | edges = edges + new_edges 146 | 147 | # Getting integrations 148 | 149 | (new_nodes, new_edges) = integration.explore_integrations( 150 | self, service_name, service_type, project 151 | ) 152 | nodes = nodes + new_nodes 153 | edges = edges + new_edges 154 | 155 | # Looking for service tags 156 | 157 | new_nodes, new_edges = tag.explore_tags( 158 | self, service_name, service_type, project 159 | ) 160 | nodes = nodes + new_nodes 161 | edges = edges + new_edges 162 | 163 | new_nodes, new_edges = backup.explore_backups( 164 | self, service_name, service_type, project 165 | ) 166 | nodes = nodes + new_nodes 167 | edges = edges + new_edges 168 | 169 | except Exception as err: 170 | print( 171 | f"Error looking up data for RUNNING {service_type}" 172 | f" service {service_name}:" 173 | f" {err.__class__.__name__} {err}" 174 | ) 175 | 176 | return nodes, edges 177 | 178 | 179 | @add_explorer("influxdb") 180 | def explore_influxdb(self, service, service_name, project): 181 | """Explores an InfluxDB service""" 182 | print(str(self) + service_name + project) 183 | nodes = [] 184 | edges = [] 185 | host = service["service_uri_params"]["host"] 186 | port = service["service_uri_params"]["port"] 187 | # need to finish 188 | return host, port, nodes, edges, SERVICE_MAP 189 | 190 | 191 | @add_explorer("elasticsearch") 192 | def explore_elasticsearch(self, service, service_name, project): 193 | """Explores an ElasticSearch service""" 194 | print(str(self) + service_name + project) 195 | nodes = [] 196 | edges = [] 197 | host = service["service_uri_params"]["host"] 198 | port = service["service_uri_params"]["port"] 199 | # need to finish 200 | return host, port, nodes, edges, SERVICE_MAP 201 | 202 | 203 | @add_explorer("m3aggregator") 204 | def explore_m3aggregator(self, service, service_name, project): 205 | """Explores an M3 aggregator service""" 206 | print(str(self) + service_name + project) 207 | nodes = [] 208 | edges = [] 209 | host = service["service_uri_params"]["host"] 210 | port = service["service_uri_params"]["port"] 211 | # need to finish 212 | return host, port, nodes, edges, SERVICE_MAP 213 | 214 | 215 | @add_explorer("m3coordinator") 216 | def explore_m3coordinator(self, service, service_name, project): 217 | """Explores an M3Coordinator service""" 218 | print(str(self) + service_name + project) 219 | nodes = [] 220 | edges = [] 221 | host = service["service_uri_params"]["host"] 222 | port = service["service_uri_params"]["port"] 223 | # need to finish 224 | return host, port, nodes, edges, SERVICE_MAP 225 | 226 | 227 | @add_explorer("clickhouse") 228 | def explore_clickhouse(self, service, service_name, project): 229 | """Explores an Clickhouse service""" 230 | print(str(self) + service_name + project) 231 | nodes = [] 232 | edges = [] 233 | host = service["service_uri_params"]["host"] 234 | port = service["service_uri_params"]["port"] 235 | # need to finish 236 | return host, port, nodes, edges, SERVICE_MAP 237 | 238 | 239 | @add_explorer("kafka_mirrormaker") 240 | def explore_mirrormaker(self, service, service_name, project): 241 | """Explores an MM2 service""" 242 | nodes = [] 243 | edges = [] 244 | host = service["service_uri_params"]["host"] 245 | port = 443 246 | print(str(self) + service_name + project) 247 | # need to finish 248 | return host, port, nodes, edges, SERVICE_MAP 249 | 250 | 251 | @add_explorer("m3db") 252 | def explore_m3db(self, service, service_name, project): 253 | """Explores an M3DB service""" 254 | nodes = [] 255 | edges = [] 256 | host = service["service_uri_params"]["host"] 257 | port = service["service_uri_params"]["port"] 258 | print(str(self) + service_name + project) 259 | # need to finish 260 | return host, port, nodes, edges, SERVICE_MAP 261 | 262 | 263 | @add_explorer("redis") 264 | def explore_redis_fun(self, service, service_name, project): 265 | """Explores an Redis service""" 266 | return redis.explore_redis( 267 | self, service, service_name, project, SERVICE_MAP 268 | ) 269 | 270 | 271 | @add_explorer("cassandra") 272 | def explore_cassandra_fun(self, service, service_name, project): 273 | """Explores an InfluxDB service""" 274 | nodes = [] 275 | edges = [] 276 | host = service["service_uri_params"]["host"] 277 | port = service["service_uri_params"]["port"] 278 | print(str(self) + service_name + project) 279 | # need to finish 280 | return host, port, nodes, edges, SERVICE_MAP 281 | 282 | 283 | @add_explorer("grafana") 284 | def explore_grafana_fun(self, service, service_name, project): 285 | """Explores an Grafana service""" 286 | return grafana.explore_grafana(service, service_name, SERVICE_MAP) 287 | 288 | 289 | @add_explorer("opensearch") 290 | def explore_opensearch_fun(self, service, service_name, project): 291 | """Explores an OpenSearch service""" 292 | return opensearch.explore_opensearch( 293 | self, service, service_name, project, SERVICE_MAP 294 | ) 295 | 296 | 297 | @add_explorer("flink") 298 | def explore_flink_fun(self, service, service_name, project): 299 | """Explores an Apache Flink service""" 300 | return flink.explore_flink( 301 | self, service, service_name, project, SERVICE_MAP 302 | ) 303 | 304 | 305 | # Exploring Kafka Services 306 | 307 | 308 | @add_explorer("kafka") 309 | def explore_kafka_fun(self, service, service_name, project): 310 | """Explores an Apache Kafka service""" 311 | return kafka.explore_kafka( 312 | self, service, service_name, project, SERVICE_MAP 313 | ) 314 | 315 | 316 | # Exploring Kafka Services 317 | 318 | 319 | @add_explorer("kafka_connect") 320 | def explore_kafka_connect_fun(self, service, service_name, project): 321 | """Explore a Kafka Connect service. 322 | 323 | Note that `service` will be None if we're called from explore_kafka 324 | """ 325 | return kafka_connect.explore_kafka_connect( 326 | self, service, service_name, project, SERVICE_MAP 327 | ) 328 | 329 | 330 | @add_explorer("mysql") 331 | def explore_mysql_fun(self, service, service_name, project): 332 | """Explores an MySQL service""" 333 | return mysql.explore_mysql(service, service_name, SERVICE_MAP) 334 | 335 | 336 | @add_explorer("pg") 337 | def explore_pg_fun(self, service, service_name, project): 338 | """Explores an PG service""" 339 | return pg.explore_pg(self, service, service_name, project, SERVICE_MAP) 340 | 341 | 342 | def explore_ext_endpoints(self, project): 343 | """Explores an Endpoint""" 344 | nodes = [] 345 | edges = [] 346 | ext_endpoints = self.get_service_integration_endpoints(project=project) 347 | 348 | for ext_endpoint in ext_endpoints: 349 | nodes.append( 350 | { 351 | "id": "ext" + ext_endpoint["endpoint_name"], 352 | "service_type": ext_endpoint["endpoint_type"], 353 | "type": "external_endpoint", 354 | "label": ext_endpoint["endpoint_name"], 355 | } 356 | ) 357 | 358 | return nodes, edges 359 | 360 | 361 | def explore_components(service_name, service_type, components): 362 | nodes = [] 363 | edges = [] 364 | 365 | global SERVICE_MAP 366 | 367 | for component in components: 368 | nodes.append( 369 | { 370 | "id": service_type 371 | + "~" 372 | + service_name 373 | + "~node~" 374 | + component.get("host"), 375 | "host": component.get("host"), 376 | "port": component.get("port"), 377 | "privatelink_connection_id": component.get( 378 | "privatelink_connection_id" 379 | ), 380 | "route": component.get("route"), 381 | "usage": component.get("usage"), 382 | "type": "service_node", 383 | "service_type": service_type, 384 | "label": component.get("host"), 385 | } 386 | ) 387 | edges.append( 388 | { 389 | "from": service_type 390 | + "~" 391 | + service_name 392 | + "~node~" 393 | + component.get("host"), 394 | "to": service_name, 395 | "label": "partition", 396 | } 397 | ) 398 | SERVICE_MAP[component.get("host")] = service_name 399 | return nodes, edges 400 | -------------------------------------------------------------------------------- /src/flink.py: -------------------------------------------------------------------------------- 1 | """Parsing Apache Flink® services""" 2 | 3 | 4 | def explore_flink(self, service, service_name, project, service_map): 5 | """Explores an Apache Flink service""" 6 | nodes = [] 7 | edges = [] 8 | host = service["service_uri_params"]["host"] 9 | port = 443 10 | 11 | # Parsing Applications 12 | new_nodes, new_edges = explore_flink_applications( 13 | self, service_name, project 14 | ) 15 | nodes = nodes + new_nodes 16 | edges = edges + new_edges 17 | 18 | return host, port, nodes, edges, service_map 19 | 20 | 21 | def explore_flink_applications(self, service_name, project): 22 | """Explores Flink applications""" 23 | nodes = [] 24 | edges = [] 25 | 26 | applications = self.flink_list_applications( 27 | service=service_name, project=project 28 | ) 29 | for application in applications["applications"]: 30 | # Creating a node beween table and service 31 | nodes.append( 32 | { 33 | "id": "flink~" 34 | + service_name 35 | + "~application~" 36 | + application["name"], 37 | "service_type": "flink", 38 | "type": "flink application", 39 | "application_id": application["id"], 40 | "label": application["name"], 41 | } 42 | ) 43 | 44 | edges.append( 45 | { 46 | "from": "flink~" 47 | + service_name 48 | + "~application~" 49 | + application["name"], 50 | "to": service_name, 51 | "label": "flink application", 52 | } 53 | ) 54 | 55 | application_details = self.flink_get_application( 56 | service=service_name, 57 | project=project, 58 | application_id=application["id"], 59 | ) 60 | 61 | for application_version in application_details["application_versions"]: 62 | nodes.append( 63 | { 64 | "id": "flink~" 65 | + service_name 66 | + "~application~" 67 | + application["name"] 68 | + "~version~" 69 | + str(application_version["version"]), 70 | "service_type": "flink", 71 | "type": "flink application version", 72 | "application_version_id": application_version["id"], 73 | "label": str(application_version["version"]), 74 | } 75 | ) 76 | 77 | edges.append( 78 | { 79 | "from": "flink~" 80 | + service_name 81 | + "~application~" 82 | + application["name"], 83 | "to": "flink~" 84 | + service_name 85 | + "~application~" 86 | + application["name"] 87 | + "~version~" 88 | + str(application_version["version"]), 89 | "label": "flink application version", 90 | } 91 | ) 92 | 93 | for source in application_version["sources"]: 94 | print(source) 95 | # TODO - explore application versions 96 | 97 | return nodes, edges 98 | 99 | 100 | def explore_flink_tables(self, service_name, project): 101 | """Explores Flink tables - old, not to be used""" 102 | 103 | nodes = [] 104 | edges = [] 105 | 106 | tables = self.list_flink_tables(service=service_name, project=project) 107 | tables_map = {} 108 | 109 | # Checking each table definition in Flink 110 | for table in tables: 111 | # Creating a node beween table and service 112 | nodes.append( 113 | { 114 | "id": "flink~" 115 | + service_name 116 | + "~table~" 117 | + table["table_name"], 118 | "service_type": "flink", 119 | "type": "flink table", 120 | "table_id": table["table_id"], 121 | "label": table["table_name"], 122 | } 123 | ) 124 | # For each column in the table 125 | new_nodes, new_edges = explore_flink_columns( 126 | service_name, 127 | table["table_name"], 128 | table["table_id"], 129 | table["columns"], 130 | ) 131 | nodes = nodes + new_nodes 132 | edges = edges + new_edges 133 | 134 | # List the integrations 135 | integrations = self.get_service_integrations( 136 | service=service_name, project=project 137 | ) 138 | src_name = "" 139 | i = 0 140 | # Look for an integration that has the same id as the table 141 | # Probably we want to do once per flink service 142 | # rather than doing it for every table 143 | while src_name == "": 144 | if ( 145 | integrations[i]["service_integration_id"] 146 | == table["integration_id"] 147 | ): 148 | # print(integrations[i]) 149 | src_name = integrations[i]["source_service"] 150 | i = i + 1 151 | # Creatind edge between table and service 152 | edges.append( 153 | { 154 | "from": "flink~" 155 | + service_name 156 | + "~table~" 157 | + table["table_name"], 158 | "to": service_name, 159 | "label": "topic", 160 | } 161 | ) 162 | 163 | service = self.get_service(project=project, service=src_name) 164 | # table_details = self.get_flink_table( 165 | # service=service_name, project=project, table_id=table["table_id"] 166 | # ) 167 | # print(table_details) 168 | # tobedone parse more details of the table (each column?) 169 | tables_map[table["table_id"]] = table["table_name"] 170 | 171 | # Creating the edge between table and target topic/table/index 172 | if service["service_type"] == "pg": 173 | edges.append( 174 | { 175 | "from": "flink~" 176 | + service_name 177 | + "~table~" 178 | + table["table_name"], 179 | "to": src_name, 180 | "label": "flink pg src", 181 | } 182 | ) 183 | # TO_DO once flink returns the src table or topic name, 184 | # link that one 185 | elif service["service_type"] == "opensearch": 186 | edges.append( 187 | { 188 | "from": "flink~" 189 | + service_name 190 | + "~table~" 191 | + table["table_name"], 192 | "to": src_name, 193 | "label": "flink opensearch src", 194 | } 195 | ) 196 | else: 197 | edges.append( 198 | { 199 | "from": "flink~" 200 | + service_name 201 | + "~table~" 202 | + table["table_name"], 203 | "to": src_name, 204 | "label": "flink kafka src", 205 | } 206 | ) 207 | # TO_DO once flink returns the src table or topic name, 208 | # link that one 209 | return nodes, edges 210 | 211 | 212 | def explore_flink_jobs(self, service_name, project): 213 | """Explores Flink jobs""" 214 | 215 | nodes = [] 216 | edges = [] 217 | jobs = self.list_flink_jobs(service=service_name, project=project) 218 | for job in jobs: 219 | job_det = self.get_flink_job( 220 | service=service_name, project=project, job_id=job["id"] 221 | ) 222 | # Adding Job node 223 | nodes.append( 224 | { 225 | "id": "flink~" + service_name + "~job~" + job_det["name"], 226 | "service_type": "flink", 227 | "type": "flink job", 228 | "job_id": job_det["jid"], 229 | "label": job_det["name"], 230 | } 231 | ) 232 | # Adding edges between Job node and service 233 | edges.append( 234 | { 235 | "from": "flink~" + service_name + "~job~" + job_det["name"], 236 | "to": service_name, 237 | "label": "flink job", 238 | } 239 | ) 240 | # tobedone: once the api returns the Flink tables used for the job, 241 | # create edges between tables and jobs 242 | return nodes, edges 243 | 244 | 245 | def explore_flink_columns(service_name, table_name, table_id, columns): 246 | """Explores Flink tables columns""" 247 | 248 | nodes = [] 249 | edges = [] 250 | for column in columns: 251 | # Create node for the column 252 | nodes.append( 253 | { 254 | "id": "flink~" 255 | + service_name 256 | + "~table~" 257 | + table_name 258 | + "~column~" 259 | + column["name"], 260 | "service_type": "flink", 261 | "type": "flink table column", 262 | "table_id": table_id, 263 | "datatype": column["data_type"], 264 | "nullable": column["nullable"], 265 | "label": column["name"], 266 | } 267 | ) 268 | # Create edge between table and column 269 | edges.append( 270 | { 271 | "from": "flink~" 272 | + service_name 273 | + "~table~" 274 | + table_name 275 | + "~column~" 276 | + column["name"], 277 | "to": "flink~" + service_name + "~table~" + table_name, 278 | "label": "table_column", 279 | } 280 | ) 281 | # tobedone: once the api returns the Flink tables used for the job, 282 | # create edges between tables and jobs 283 | return nodes, edges 284 | -------------------------------------------------------------------------------- /src/grafana.py: -------------------------------------------------------------------------------- 1 | """Parsing Grafana service""" 2 | 3 | import json 4 | import requests 5 | 6 | 7 | def explore_grafana(service, service_name, service_map): 8 | """Parsing Grafana service""" 9 | 10 | nodes = [] 11 | edges = [] 12 | 13 | host = service["service_uri_params"]["host"] 14 | port = service["service_uri_params"]["port"] 15 | 16 | avnadmin_password = "fake" 17 | 18 | # Parse the users 19 | 20 | new_nodes, new_edges, avnadmin_password = explore_grafana_users( 21 | service, service_name 22 | ) 23 | nodes = nodes + new_nodes 24 | edges = edges + new_edges 25 | 26 | # Define the base URL 27 | base_url = "https://" + service["service_uri_params"]["host"] + ":443" 28 | auth = ("avnadmin", avnadmin_password) 29 | 30 | # Get datasources 31 | 32 | ( 33 | new_nodes, 34 | new_edges, 35 | service_map, 36 | datasources_map, 37 | ) = explore_grafana_datasources(service_name, base_url, auth, service_map) 38 | nodes = nodes + new_nodes 39 | edges = edges + new_edges 40 | 41 | # getting dashboards 42 | 43 | new_nodes, new_edges = explore_grafana_dashboards( 44 | service_name, base_url, auth, avnadmin_password, datasources_map 45 | ) 46 | nodes = nodes + new_nodes 47 | edges = edges + new_edges 48 | 49 | return host, port, nodes, edges, service_map 50 | 51 | 52 | def explore_grafana_users(service, service_name): 53 | """Parsing Grafana users""" 54 | 55 | nodes = [] 56 | edges = [] 57 | users = service["users"] 58 | 59 | for user in users: 60 | # Create node per user 61 | nodes.append( 62 | { 63 | "id": "grafana~" + service_name + "~user~" + user["username"], 64 | "user-type": user["type"], 65 | "service_type": "grafana", 66 | "type": "user", 67 | "label": user["username"], 68 | } 69 | ) 70 | # Create edge between user and service 71 | edges.append( 72 | { 73 | "from": "grafana~" 74 | + service_name 75 | + "~user~" 76 | + user["username"], 77 | "to": service_name, 78 | "label": "user", 79 | } 80 | ) 81 | 82 | # Take theavnadmin_password of the avnadmin user for further ispection 83 | if user["username"] == "avnadmin": 84 | avnadmin_password = user["password"] 85 | 86 | return nodes, edges, avnadmin_password 87 | 88 | 89 | def explore_grafana_datasources(service_name, base_url, auth, service_map): 90 | """Parsing Grafana dashboards""" 91 | 92 | nodes = [] 93 | edges = [] 94 | 95 | datasources_map = {} 96 | datasources = requests.get(base_url + "/api/datasources", auth=auth) 97 | 98 | for datasource in json.loads(datasources.text): 99 | # Create a map of the datasources id -> name 100 | datasources_map[datasource["uid"]] = datasource["name"] 101 | # Create node per datasource 102 | nodes.append( 103 | { 104 | "id": "grafana~" 105 | + service_name 106 | + "~datasource~" 107 | + datasource["name"], 108 | "datasource-type": datasource["type"], 109 | "service_type": "grafana", 110 | "type": "datasource", 111 | "label": datasource["name"], 112 | "tgt_url": datasource["url"], 113 | } 114 | ) 115 | # Create edge between datasource and service 116 | edges.append( 117 | { 118 | "from": "grafana~" 119 | + service_name 120 | + "~datasource~" 121 | + datasource["name"], 122 | "to": service_name, 123 | "label": "datasource", 124 | } 125 | ) 126 | 127 | # Look for target host 128 | target_host = ( 129 | datasource["url"] 130 | .replace("http://", "") 131 | .replace("https://", "") 132 | .split(":")[0] 133 | ) 134 | 135 | # Check if the host in the list of target hosts already 136 | dest_service = service_map.get(target_host) 137 | # If host doesn't exist yet 138 | if dest_service is None: 139 | # Create new node for external service host 140 | nodes.append( 141 | { 142 | "id": "ext-src-" + target_host, 143 | "service_type": "ext-service", 144 | "type": "external-service", 145 | "label": target_host, 146 | } 147 | ) 148 | service_map[target_host] = target_host 149 | # Create new edge between external service host and datasource 150 | edges.append( 151 | { 152 | "from": "grafana~" 153 | + service_name 154 | + "~datasource~" 155 | + datasource["name"], 156 | "to": "ext-src-" + target_host, 157 | "label": "datasource", 158 | } 159 | ) 160 | else: 161 | # Create new edge between existing service host and datasource 162 | edges.append( 163 | { 164 | "from": "grafana~" 165 | + service_name 166 | + "~datasource~" 167 | + datasource["name"], 168 | "to": dest_service, 169 | "label": "datasource", 170 | } 171 | ) 172 | 173 | # In case is PG 174 | if datasource["type"] == "postgres": 175 | if dest_service is None: 176 | # Creates a database node in the external service 177 | nodes.append( 178 | { 179 | "id": "ext-src-" 180 | + dest_service 181 | + "~db~" 182 | + datasource["database"], 183 | "service_type": "ext-pg", 184 | "type": "database", 185 | "label": datasource["database"], 186 | } 187 | ) 188 | # Creates an edge between the database and the datasource 189 | edges.append( 190 | { 191 | "from": "grafana~" 192 | + service_name 193 | + "~datasource~" 194 | + datasource["name"], 195 | "to": "ext-src-" 196 | + dest_service 197 | + "~db~" 198 | + datasource["database"], 199 | "label": "datasource", 200 | } 201 | ) 202 | else: 203 | # Creates an edge between the database and the datasource 204 | edges.append( 205 | { 206 | "from": "grafana~" 207 | + service_name 208 | + "~datasource~" 209 | + datasource["name"], 210 | "to": "pg~" 211 | + dest_service 212 | + "~database~" 213 | + datasource["database"], 214 | "label": "datasource", 215 | } 216 | ) 217 | return nodes, edges, service_map, datasources_map 218 | 219 | 220 | def explore_grafana_dashboards( 221 | service_name, base_url, auth, avnadmin_password, datasources_map 222 | ): 223 | """Parsing Grafana dashboards""" 224 | 225 | nodes = [] 226 | edges = [] 227 | 228 | dashboards = requests.get( 229 | base_url + "/api/search?dash-folder", 230 | auth=auth, 231 | ) 232 | 233 | for dashboard in json.loads(dashboards.text): 234 | # Creates a node for the dashboard 235 | nodes.append( 236 | { 237 | "id": "grafana~" 238 | + service_name 239 | + "~dashboard~" 240 | + dashboard["title"], 241 | "service_type": "grafana", 242 | "type": "dashboard", 243 | "label": dashboard["title"], 244 | } 245 | ) 246 | # Creates an edge between service name and dashboard 247 | edges.append( 248 | { 249 | "from": "grafana~" 250 | + service_name 251 | + "~dashboard~" 252 | + dashboard["title"], 253 | "to": service_name, 254 | "label": "dashboard", 255 | } 256 | ) 257 | # gets the dashboard details 258 | 259 | dashboard_details = requests.get( 260 | base_url + "/api/dashboards/uid/" + dashboard["uid"], 261 | auth=("avnadmin", avnadmin_password), 262 | ) 263 | 264 | dash_details = json.loads(dashboard_details.text) 265 | # Adds edge between dashboard and creator 266 | edges.append( 267 | { 268 | "from": "grafana~" 269 | + service_name 270 | + "~dashboard~" 271 | + dashboard["title"], 272 | "to": "grafana~" 273 | + service_name 274 | + "~user~" 275 | + dash_details["meta"]["createdBy"], 276 | "type": "dashboard-creator", 277 | "label": "dashboard-creator", 278 | } 279 | ) 280 | 281 | # A dashboard can have rows defined or not 282 | if dash_details["dashboard"].get("rows") is not None: 283 | for row in dash_details["dashboard"]["rows"]: 284 | # Looks for panels in the dashboard 285 | for panel in row["panels"]: 286 | 287 | if isinstance(panel["datasource"], str): 288 | datasource = panel["datasource"] 289 | # Creates an edge between the dashboard and datasource 290 | edges.append( 291 | { 292 | "from": "grafana~" 293 | + service_name 294 | + "~dashboard~" 295 | + dashboard["title"], 296 | "to": "grafana~" 297 | + service_name 298 | + "~datasource~" 299 | + datasource, 300 | "label": "dashboard datasource", 301 | } 302 | ) 303 | else: 304 | datasource = panel["datasource"]["uid"] 305 | # Creates an edge between the dashboard and datasource 306 | edges.append( 307 | { 308 | "from": "grafana~" 309 | + service_name 310 | + "~dashboard~" 311 | + dashboard["title"], 312 | "to": "grafana~" 313 | + service_name 314 | + "~datasource~" 315 | + datasources_map[datasource], 316 | "label": "dashboard datasource", 317 | } 318 | ) 319 | # tobedone explore all columns in a dashboard panel 320 | else: 321 | for panel in dash_details["dashboard"]["panels"]: 322 | if isinstance(panel["datasource"], str): 323 | datasource = panel["datasource"] 324 | # Creates an edge between the dashboard and datasource 325 | edges.append( 326 | { 327 | "from": "grafana~" 328 | + service_name 329 | + "~dashboard~" 330 | + dashboard["title"], 331 | "to": "grafana~" 332 | + service_name 333 | + "~datasource~" 334 | + datasource, 335 | "label": "dashboard datasource", 336 | } 337 | ) 338 | elif isinstance(panel["datasource"], dict): 339 | datasource = panel["datasource"]["uid"] 340 | # Creates an edge between the dashboard and datasource 341 | edges.append( 342 | { 343 | "from": "grafana~" 344 | + service_name 345 | + "~dashboard~" 346 | + dashboard["title"], 347 | "to": "grafana~" 348 | + service_name 349 | + "~datasource~" 350 | + datasources_map[datasource], 351 | "label": "dashboard datasource", 352 | } 353 | ) 354 | return nodes, edges 355 | -------------------------------------------------------------------------------- /src/integration.py: -------------------------------------------------------------------------------- 1 | """Parsing service integration""" 2 | 3 | 4 | def explore_integrations(self, service_name, service_type, project): 5 | """Parsing service intergration""" 6 | 7 | nodes = [] 8 | edges = [] 9 | 10 | integrations = self.get_service_integrations( 11 | service=service_name, project=project 12 | ) 13 | for integration in integrations: 14 | if integration["enabled"] is True: 15 | 16 | edges.append( 17 | { 18 | "from": integration["source_service"], 19 | "to": integration["dest_service"], 20 | "main_type": "integration", 21 | "integration_type": integration["integration_type"], 22 | "label": integration["integration_type"], 23 | "integration_id": integration["service_integration_id"], 24 | } 25 | ) 26 | 27 | return nodes, edges 28 | -------------------------------------------------------------------------------- /src/kafka.py: -------------------------------------------------------------------------------- 1 | """Parsing Apache Kafka® services""" 2 | 3 | import re 4 | from src import kafka_connect 5 | 6 | 7 | def explore_kafka(self, service, service_name, project, service_map): 8 | """Explores an Apache Kafka service""" 9 | nodes = [] 10 | edges = [] 11 | host = service["service_uri_params"]["host"] 12 | kafka = self.get_service(project=project, service=service_name) 13 | 14 | new_nodes, new_edges, topic_list = explore_kafka_topics( 15 | self, service_name, project 16 | ) 17 | 18 | nodes = nodes + new_nodes 19 | edges = edges + new_edges 20 | 21 | new_nodes, new_edges = explore_kafka_users(self, service_name, project) 22 | 23 | nodes = nodes + new_nodes 24 | edges = edges + new_edges 25 | 26 | new_nodes, new_edges = explore_kafka_acls( 27 | self, service_name, project, topic_list 28 | ) 29 | 30 | nodes = nodes + new_nodes 31 | edges = edges + new_edges 32 | 33 | # tbd parse schemas 34 | 35 | # If the service has Kafka connect, we can explore it as well 36 | if kafka["user_config"]["kafka_connect"] is True: 37 | ( 38 | _, 39 | _, 40 | new_nodes, 41 | new_edges, 42 | service_map, 43 | ) = kafka_connect.explore_kafka_connect( 44 | self, None, service_name, project, service_map 45 | ) 46 | nodes = nodes + new_nodes 47 | edges = edges + new_edges 48 | 49 | return host, service, nodes, edges, service_map 50 | 51 | 52 | def explore_kafka_topics(self, service_name, project): 53 | """get Kafka topics""" 54 | nodes = [] 55 | edges = [] 56 | topic_list = [] 57 | 58 | topics = self.list_service_topics(service=service_name, project=project) 59 | 60 | # Exploring Topics 61 | for topic in topics: 62 | 63 | topic_infos = self.get_service_topic( 64 | project=project, service=service_name, topic=topic["topic_name"] 65 | ) 66 | 67 | nodes.append( 68 | { 69 | "id": "kafka~" 70 | + service_name 71 | + "~topic~" 72 | + topic["topic_name"], 73 | "service_type": "kafka", 74 | "type": "topic", 75 | "cleanup_policy": topic["cleanup_policy"], 76 | "label": topic["topic_name"], 77 | "partitions": topic_infos.get("partitions"), 78 | "min_insync_replicas": topic_infos.get("min_insync_replicas"), 79 | "replication": topic_infos.get("replication"), 80 | "retention_bytes": topic_infos.get("retention_bytes"), 81 | "retention_hours": topic_infos.get("retention_hours"), 82 | "state": topic_infos.get("state"), 83 | "config": topic_infos.get("config"), 84 | } 85 | ) 86 | edges.append( 87 | { 88 | "from": "kafka~" 89 | + service_name 90 | + "~topic~" 91 | + topic["topic_name"], 92 | "to": service_name, 93 | "label": "topic", 94 | } 95 | ) 96 | topic_list.append(topic["topic_name"]) 97 | 98 | new_nodes, new_edges = explore_kafka_topic_partitions( 99 | service_name, topic["topic_name"], topic_infos["partitions"] 100 | ) 101 | nodes = nodes + new_nodes 102 | edges = edges + new_edges 103 | 104 | for tag in topic["tags"]: 105 | nodes.append( 106 | { 107 | "id": "tag~id~" + tag["key"] + "~value~" + tag["value"], 108 | "service_type": "tag", 109 | "type": "tag", 110 | "label": tag["key"] + "=" + tag["value"], 111 | } 112 | ) 113 | edges.append( 114 | { 115 | "from": "kafka~" 116 | + service_name 117 | + "~topic~" 118 | + topic["topic_name"], 119 | "to": "tag~id~" + tag["key"] + "~value~" + tag["value"], 120 | "label": "tag", 121 | } 122 | ) 123 | return nodes, edges, topic_list 124 | 125 | 126 | def explore_kafka_users(self, service_name, project): 127 | """Get Kafka users""" 128 | nodes = [] 129 | edges = [] 130 | kafka = self.get_service(project=project, service=service_name) 131 | 132 | # Exploring Users 133 | for user in kafka["users"]: 134 | 135 | nodes.append( 136 | { 137 | "id": "kafka~" + service_name + "~user~" + user["username"], 138 | "service_type": "kafka", 139 | "type": "user", 140 | "user_type": user["type"], 141 | "label": user["username"], 142 | } 143 | ) 144 | edges.append( 145 | { 146 | "from": "kafka~" + service_name + "~user~" + user["username"], 147 | "to": service_name, 148 | "label": "user", 149 | } 150 | ) 151 | return nodes, edges 152 | 153 | 154 | def explore_kafka_acls(self, service_name, project, topic_list): 155 | """Getting Kafka ACLs""" 156 | nodes = [] 157 | edges = [] 158 | kafka = self.get_service(project=project, service=service_name) 159 | 160 | # Exploring ACLs 161 | for acl in kafka["acl"]: 162 | # Create node for ACL 163 | nodes.append( 164 | { 165 | "id": "kafka~" + service_name + "~acl~" + acl["id"], 166 | "service_type": "kafka", 167 | "type": "topic-acl", 168 | "permission": acl["permission"], 169 | "label": acl["id"], 170 | "topic": acl["topic"], 171 | "username": acl["username"], 172 | } 173 | ) 174 | # Create edge between ACL and username 175 | edges.append( 176 | { 177 | "from": "kafka~" + service_name + "~user~" + acl["username"], 178 | "to": "kafka~" + service_name + "~acl~" + acl["id"], 179 | "label": "user", 180 | } 181 | ) 182 | # Map topics that an ACL shows 183 | for topic in topic_list: 184 | strtomatch = acl["topic"] 185 | if strtomatch == "*": 186 | strtomatch = ".*" 187 | # Checking if the ACL string matches a topic 188 | # ACL strings are defined with Java RegExp 189 | # and we're parsing them with Python, 190 | # maybe something to improve here? 191 | if re.match(strtomatch, topic): 192 | edges.append( 193 | { 194 | "from": "kafka~" + service_name + "~acl~" + acl["id"], 195 | "to": "kafka~" + service_name + "~topic~" + topic, 196 | "label": "topic-acl", 197 | } 198 | ) 199 | return nodes, edges 200 | 201 | 202 | def explore_kafka_topic_partitions(service_name, topic_name, partitions): 203 | """Getting Kafka topic partitions""" 204 | nodes = [] 205 | edges = [] 206 | 207 | # Exploring partitions 208 | for partition in partitions: 209 | # Create node for partition 210 | 211 | nodes.append( 212 | { 213 | "id": "kafka~" 214 | + service_name 215 | + "~topic~" 216 | + topic_name 217 | + "~partition~" 218 | + str(partition["partition"]), 219 | "service_type": "kafka", 220 | "type": "partition", 221 | "earliest_offset": partition["earliest_offset"], 222 | "isr": partition["isr"], 223 | "latest_offset": partition["latest_offset"], 224 | "size": partition["size"], 225 | "label": "Partition " + str(partition["partition"]), 226 | } 227 | ) 228 | 229 | # Create edge between partition and topic 230 | edges.append( 231 | { 232 | "from": "kafka~" 233 | + service_name 234 | + "~topic~" 235 | + topic_name 236 | + "~partition~" 237 | + str(partition["partition"]), 238 | "to": "kafka~" + service_name + "~topic~" + topic_name, 239 | "label": "partition", 240 | } 241 | ) 242 | 243 | new_nodes, new_edges = explore_kafka_topic_partitions_consumer_groups( 244 | service_name, 245 | topic_name, 246 | partition["partition"], 247 | partition["consumer_groups"], 248 | ) 249 | nodes = nodes + new_nodes 250 | edges = edges + new_edges 251 | return nodes, edges 252 | 253 | 254 | def explore_kafka_topic_partitions_consumer_groups( 255 | service_name, topic_name, partition, consumers 256 | ): 257 | """Getting Kafka topic partitions consumer group info""" 258 | nodes = [] 259 | edges = [] 260 | 261 | # Exploring partitions 262 | for consumer in consumers: 263 | # Create node for partition 264 | nodes.append( 265 | { 266 | "id": "kafka~" 267 | + service_name 268 | + "~topic~" 269 | + topic_name 270 | + "~partition~" 271 | + str(partition) 272 | + "~consumer_group~" 273 | + consumer["group_name"], 274 | "offset": consumer["offset"], 275 | "label": consumer["group_name"], 276 | "type": "consumer_group", 277 | "service_type": "kafka", 278 | } 279 | ) 280 | # Create edge between partition and username 281 | edges.append( 282 | { 283 | "from": "kafka~" 284 | + service_name 285 | + "~topic~" 286 | + topic_name 287 | + "~partition~" 288 | + str(partition) 289 | + "~consumer_group~" 290 | + consumer["group_name"], 291 | "to": "kafka~" 292 | + service_name 293 | + "~topic~" 294 | + topic_name 295 | + "~partition~" 296 | + str(partition), 297 | "label": "Consumer group:" + consumer["group_name"], 298 | } 299 | ) 300 | return nodes, edges 301 | -------------------------------------------------------------------------------- /src/mysql.py: -------------------------------------------------------------------------------- 1 | """Parsing MySQL service""" 2 | 3 | import pymysql 4 | 5 | 6 | def explore_mysql(service, service_name, service_map): 7 | """Explores an MySQL service""" 8 | nodes = [] 9 | edges = [] 10 | 11 | host = service["connection_info"]["mysql_params"][0]["host"] 12 | port = service["connection_info"]["mysql_params"][0]["port"] 13 | 14 | # Get the avnadmin password 15 | # this is in case someone creates the service 16 | # and then changes avnadmin password 17 | avnadmin_pwd = list( 18 | filter(lambda x: x["username"] == "avnadmin", service["users"]) 19 | )[0]["password"] 20 | 21 | service_conn_info = service["service_uri_params"] 22 | 23 | try: 24 | conn = pymysql.connect( 25 | host=service_conn_info["host"], 26 | port=int(service_conn_info["port"]), 27 | database=service_conn_info["dbname"], 28 | user="avnadmin", 29 | password=avnadmin_pwd, 30 | connect_timeout=2, 31 | ) 32 | except pymysql.Error as err: 33 | conn = None 34 | print("Error connecting to: " + service_name + str(err)) 35 | nodes, edges = create_connection_error_node(service_name) 36 | 37 | if conn is not None: 38 | cur = conn.cursor() 39 | 40 | # Getting databases 41 | 42 | new_nodes, new_edges = explore_mysql_databases(cur, service_name) 43 | nodes = nodes + new_nodes 44 | edges = edges + new_edges 45 | 46 | # Getting tables 47 | 48 | new_nodes, new_edges = explore_mysql_tables(cur, service_name) 49 | nodes = nodes + new_nodes 50 | edges = edges + new_edges 51 | 52 | # Get users 53 | 54 | new_nodes, new_edges = explore_mysql_users(cur, service_name) 55 | nodes = nodes + new_nodes 56 | edges = edges + new_edges 57 | 58 | # Get User Priviledges 59 | 60 | # tobedone get user priviledges to tables 61 | 62 | # Get Columns 63 | 64 | new_nodes, new_edges = explore_mysql_columns(cur, service_name) 65 | nodes = nodes + new_nodes 66 | edges = edges + new_edges 67 | 68 | return host, port, nodes, edges, service_map 69 | 70 | 71 | def create_connection_error_node(service_name): 72 | """Creates an error node in case of connection errors""" 73 | nodes = [] 74 | edges = [] 75 | nodes.append( 76 | { 77 | "id": "mysql~" + service_name + "~connection-error", 78 | "service_type": "mysql", 79 | "type": "connection-error", 80 | "label": "connection-error", 81 | } 82 | ) 83 | edges.append( 84 | { 85 | "from": service_name, 86 | "to": "mysql~" + service_name + "~connection-error", 87 | "label": "connection-error", 88 | } 89 | ) 90 | return nodes, edges 91 | 92 | 93 | def explore_mysql_databases(cur, service_name): 94 | """Retrieves MySQL databases""" 95 | 96 | nodes = [] 97 | edges = [] 98 | 99 | cur.execute( 100 | """ 101 | select catalog_name, schema_name 102 | from information_schema.schemata; 103 | """ 104 | ) 105 | 106 | databases = cur.fetchall() 107 | for database in databases: 108 | # print(database) 109 | nodes.append( 110 | { 111 | "id": "mysql~" + service_name + "~database~" + database[1], 112 | "service_type": "mysql", 113 | "type": "database", 114 | "label": database[1], 115 | } 116 | ) 117 | edges.append( 118 | { 119 | "from": service_name, 120 | "to": "mysql~" + service_name + "~database~" + database[1], 121 | "label": "database", 122 | } 123 | ) 124 | return nodes, edges 125 | 126 | 127 | def explore_mysql_tables(cur, service_name): 128 | """Retrieves MySQL tables""" 129 | 130 | nodes = [] 131 | edges = [] 132 | 133 | cur.execute( 134 | """ 135 | select TABLE_SCHEMA,TABLE_NAME, TABLE_TYPE 136 | from information_schema.tables 137 | where table_schema not in 138 | ('information_schema','sys','performance_schema','mysql'); 139 | """ 140 | ) 141 | 142 | tables = cur.fetchall() 143 | for table in tables: 144 | nodes.append( 145 | { 146 | "id": "mysql~" 147 | + service_name 148 | + "~database~" 149 | + table[0] 150 | + "~table~" 151 | + table[1], 152 | "service_type": "mysql", 153 | "type": "table", 154 | "label": table[1], 155 | "table_type": table[2], 156 | } 157 | ) 158 | edges.append( 159 | { 160 | "from": "mysql~" + service_name + "~database~" + table[0], 161 | "to": "mysql~" 162 | + service_name 163 | + "~database~" 164 | + table[0] 165 | + "~table~" 166 | + table[1], 167 | "label": "table", 168 | } 169 | ) 170 | return nodes, edges 171 | 172 | 173 | def explore_mysql_users(cur, service_name): 174 | """Retrieves MySQL users""" 175 | 176 | nodes = [] 177 | edges = [] 178 | 179 | cur.execute( 180 | """ 181 | select USER, HOST, ATTRIBUTE 182 | from information_schema.USER_ATTRIBUTES; 183 | """ 184 | ) 185 | 186 | users = cur.fetchall() 187 | # print(users) 188 | for user in users: 189 | nodes.append( 190 | { 191 | "id": "mysql~" + service_name + "~user~" + user[0], 192 | "service_type": "mysql", 193 | "type": "user", 194 | "label": user[0], 195 | } 196 | ) 197 | edges.append( 198 | { 199 | "from": "mysql~" + service_name + "~user~" + user[0], 200 | "to": service_name, 201 | "label": "user", 202 | } 203 | ) 204 | return nodes, edges 205 | 206 | 207 | def explore_mysql_columns(cur, service_name): 208 | """Retrieves MySQL columns""" 209 | 210 | nodes = [] 211 | edges = [] 212 | 213 | cur.execute( 214 | """ 215 | select TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE 216 | from information_schema.columns where table_schema 217 | not in ('information_schema', 'sys','mysql','performance_schema'); 218 | """ 219 | ) 220 | 221 | columns = cur.fetchall() 222 | for column in columns: 223 | nodes.append( 224 | { 225 | "id": "mysql~" 226 | + service_name 227 | + "~database~" 228 | + column[0] 229 | + "~table~" 230 | + column[1] 231 | + "~column~" 232 | + column[2], 233 | "service_type": "mysql", 234 | "type": "table column", 235 | "data_type": column[4], 236 | "is_nullable": column[3], 237 | "label": column[2], 238 | } 239 | ) 240 | edges.append( 241 | { 242 | "from": "mysql~" 243 | + service_name 244 | + "~database~" 245 | + column[0] 246 | + "~table~" 247 | + column[1] 248 | + "~column~" 249 | + column[2], 250 | "to": "mysql~" 251 | + service_name 252 | + "~database~" 253 | + column[0] 254 | + "~table~" 255 | + column[1], 256 | "label": "column", 257 | } 258 | ) 259 | return nodes, edges 260 | -------------------------------------------------------------------------------- /src/opensearch.py: -------------------------------------------------------------------------------- 1 | """Parsing OpenSearch service""" 2 | 3 | 4 | def explore_opensearch(self, service, service_name, project, service_map): 5 | """Explores an OpenSearch service""" 6 | nodes = [] 7 | edges = [] 8 | 9 | host = service["service_uri_params"]["host"] 10 | port = service["service_uri_params"]["port"] 11 | 12 | # Exploring Users 13 | 14 | new_nodes, new_edges, users = explore_opensearch_users( 15 | self, service_name, project 16 | ) 17 | nodes = nodes + new_nodes 18 | edges = edges + new_edges 19 | 20 | # Getting indexes 21 | 22 | new_nodes, new_edges, indexes = explore_opensearch_indexes( 23 | self, service_name, project 24 | ) 25 | nodes = nodes + new_nodes 26 | edges = edges + new_edges 27 | 28 | # tobedone parse more stuff 29 | # Getting ACLs 30 | 31 | new_nodes, new_edges = explore_opensearch_acls( 32 | self, service_name, project, users, indexes 33 | ) 34 | nodes = nodes + new_nodes 35 | edges = edges + new_edges 36 | 37 | # tobedone: check how to parse everything when ACLs are set 38 | return host, port, nodes, edges, service_map 39 | 40 | 41 | def explore_opensearch_users(self, service_name, project): 42 | """Explores an OpenSearch users""" 43 | nodes = [] 44 | edges = [] 45 | 46 | opensearch = self.get_service(project=project, service=service_name) 47 | 48 | # Exploring Users 49 | for user in opensearch["users"]: 50 | 51 | nodes.append( 52 | { 53 | "id": "opensearch~" 54 | + service_name 55 | + "~user~" 56 | + user["username"], 57 | "service_type": "opensearch", 58 | "type": "user", 59 | "user_type": user["type"], 60 | "label": user["username"], 61 | } 62 | ) 63 | edges.append( 64 | { 65 | "from": "opensearch~" 66 | + service_name 67 | + "~user~" 68 | + user["username"], 69 | "to": service_name, 70 | "label": "user", 71 | } 72 | ) 73 | return nodes, edges, opensearch["users"] 74 | 75 | 76 | def explore_opensearch_indexes(self, service_name, project): 77 | """Explores an OpenSearch indexes""" 78 | nodes = [] 79 | edges = [] 80 | 81 | indexes = self.get_service_indexes(project=project, service=service_name) 82 | for index in indexes: 83 | 84 | # CReating node for index 85 | nodes.append( 86 | { 87 | "id": "opensearch~" 88 | + service_name 89 | + "~index~" 90 | + index["index_name"], 91 | "service_type": "opensearch", 92 | "type": "index", 93 | "label": index["index_name"], 94 | "health": index["health"], 95 | "replicas": index["number_of_replicas"], 96 | "shards": index["number_of_shards"], 97 | } 98 | ) 99 | # Creating edge between index and service 100 | edges.append( 101 | { 102 | "from": "opensearch~" 103 | + service_name 104 | + "~index~" 105 | + index["index_name"], 106 | "to": service_name, 107 | "label": "index", 108 | } 109 | ) 110 | return nodes, edges, indexes 111 | 112 | 113 | def explore_opensearch_acls(self, service_name, project, users, indexes): 114 | """Explores an OpenSearch ACLs""" 115 | nodes = [] 116 | edges = [] 117 | 118 | acls = self.list_service_elasticsearch_acl_config( 119 | project=project, service=service_name 120 | ) 121 | 122 | # If ACLs are not enabled create an edge between each user and each index 123 | if not acls["elasticsearch_acl_config"]["enabled"]: 124 | for user in users: 125 | for index in indexes: 126 | edges.append( 127 | { 128 | "from": "opensearch~" 129 | + service_name 130 | + "~index~" 131 | + index["index_name"], 132 | "to": "opensearch~" 133 | + service_name 134 | + "~user~" 135 | + user["username"], 136 | "label": "visibility_index", 137 | } 138 | ) 139 | return nodes, edges 140 | -------------------------------------------------------------------------------- /src/pg.py: -------------------------------------------------------------------------------- 1 | """Parsing PostgreSQL services""" 2 | 3 | import psycopg2 4 | from src import sql 5 | 6 | 7 | def build_conn_string(avnadmin_pwd, service_conn_info): 8 | """Builds conntection string""" 9 | connstr = ( 10 | "postgres://avnadmin:" 11 | + avnadmin_pwd 12 | + "@" 13 | + service_conn_info["host"] 14 | + ":" 15 | + service_conn_info["port"] 16 | + "/" 17 | + service_conn_info["dbname"] 18 | + "?sslmode=" 19 | + service_conn_info["sslmode"] 20 | ) 21 | return connstr 22 | 23 | 24 | def explore_pg(self, service, service_name, project, service_map): 25 | """Explores an PG service""" 26 | nodes = [] 27 | edges = [] 28 | 29 | service = self.get_service(project=project, service=service_name) 30 | 31 | # Get the avnadmin password 32 | # this is in case someone creates the service 33 | # and then changes avnadmin password 34 | avnadmin_pwd = list( 35 | filter(lambda x: x["username"] == "avnadmin", service["users"]) 36 | )[0]["password"] 37 | 38 | service_conn_info = service["connection_info"]["pg_params"][0] 39 | # Build the connection string 40 | connstr = build_conn_string(avnadmin_pwd, service_conn_info) 41 | 42 | try: 43 | conn = psycopg2.connect(connstr, connect_timeout=2) 44 | except psycopg2.Error as err: 45 | conn = None 46 | print("Error connecting to: " + service_name + str(err)) 47 | nodes.append( 48 | { 49 | "id": "pg~" + service_name + "~connection-error", 50 | "service_type": "pg", 51 | "type": "connection-error", 52 | "label": "connection-error", 53 | } 54 | ) 55 | edges.append( 56 | { 57 | "from": service_name, 58 | "to": "pg~" + service_name + "~connection-error", 59 | "label": "connection-error", 60 | } 61 | ) 62 | 63 | if conn is not None: 64 | cur = conn.cursor() 65 | 66 | new_nodes, new_edges = explore_pg_database(cur, service_name) 67 | nodes = nodes + new_nodes 68 | edges = edges + new_edges 69 | 70 | new_nodes, new_edges = explore_pg_tablespaces(cur, service_name) 71 | nodes = nodes + new_nodes 72 | edges = edges + new_edges 73 | 74 | new_nodes, new_edges = explore_pg_tables(cur, service_name) 75 | nodes = nodes + new_nodes 76 | edges = edges + new_edges 77 | 78 | new_nodes, new_edges = explore_pg_users(cur, service_name) 79 | nodes = nodes + new_nodes 80 | edges = edges + new_edges 81 | 82 | new_nodes, new_edges = explore_pg_grants(cur, service_name) 83 | nodes = nodes + new_nodes 84 | edges = edges + new_edges 85 | 86 | new_nodes, new_edges = explore_pg_views(cur, service_name) 87 | nodes = nodes + new_nodes 88 | edges = edges + new_edges 89 | 90 | new_nodes, new_edges = explore_pg_columns(cur, service_name) 91 | nodes = nodes + new_nodes 92 | edges = edges + new_edges 93 | 94 | cur.close() 95 | conn.close() 96 | return ( 97 | service_conn_info["host"], 98 | service_conn_info["port"], 99 | nodes, 100 | edges, 101 | service_map, 102 | ) 103 | 104 | 105 | def explore_pg_database(cur, service_name): 106 | """Retrieves info about a PG database""" 107 | 108 | nodes = [] 109 | edges = [] 110 | 111 | cur.execute("SELECT datname FROM pg_database;") 112 | 113 | databases = cur.fetchall() 114 | for database in databases: 115 | # print(database) 116 | nodes.append( 117 | { 118 | "id": "pg~" + service_name + "~database~" + database[0], 119 | "service_type": "pg", 120 | "type": "database", 121 | "label": database[0], 122 | } 123 | ) 124 | edges.append( 125 | { 126 | "from": service_name, 127 | "to": "pg~" + service_name + "~database~" + database[0], 128 | "label": "database", 129 | } 130 | ) 131 | return nodes, edges 132 | 133 | 134 | def explore_pg_tablespaces(cur, service_name): 135 | """Retrieves info about a PG tablespace""" 136 | 137 | nodes = [] 138 | edges = [] 139 | 140 | cur.execute( 141 | """ 142 | select catalog_name, schema_name, schema_owner 143 | from information_schema.schemata; 144 | """ 145 | ) 146 | 147 | namespaces = cur.fetchall() 148 | for namespace in namespaces: 149 | nodes.append( 150 | { 151 | "id": "pg~" + service_name + "~schema~" + namespace[1], 152 | "service_type": "pg", 153 | "type": "schema", 154 | "label": namespace[1], 155 | } 156 | ) 157 | edges.append( 158 | { 159 | "from": "pg~" + service_name + "~database~" + namespace[0], 160 | "to": "pg~" + service_name + "~schema~" + namespace[1], 161 | "label": "schema", 162 | } 163 | ) 164 | return nodes, edges 165 | 166 | 167 | def explore_pg_tables(cur, service_name): 168 | """Retrieves info about a PG tables""" 169 | 170 | nodes = [] 171 | edges = [] 172 | 173 | cur.execute( 174 | """ 175 | SELECT schemaname, tablename, tableowner 176 | FROM pg_tables where tableowner <> 'postgres'; 177 | """ 178 | ) 179 | 180 | tables = cur.fetchall() 181 | for table in tables: 182 | nodes.append( 183 | { 184 | "id": "pg~" 185 | + service_name 186 | + "~schema~" 187 | + table[0] 188 | + "~table_view~" 189 | + table[1], 190 | "service_type": "pg", 191 | "type": "table", 192 | "label": table[1], 193 | } 194 | ) 195 | edges.append( 196 | { 197 | "from": "pg~" + service_name + "~schema~" + table[0], 198 | "to": "pg~" 199 | + service_name 200 | + "~schema~" 201 | + table[0] 202 | + "~table_view~" 203 | + table[1], 204 | "label": "table", 205 | } 206 | ) 207 | return nodes, edges 208 | 209 | 210 | def explore_pg_users(cur, service_name): 211 | """Retrieves info about a PG users""" 212 | 213 | nodes = [] 214 | edges = [] 215 | 216 | cur.execute("SELECT * FROM pg_user;") 217 | 218 | users = cur.fetchall() 219 | # print(users) 220 | for user in users: 221 | nodes.append( 222 | { 223 | "id": "pg~" + service_name + "~user~" + user[0], 224 | "service_type": "pg", 225 | "type": "user", 226 | "label": user[0], 227 | } 228 | ) 229 | edges.append( 230 | { 231 | "from": "pg~" + service_name + "~user~" + user[0], 232 | "to": service_name, 233 | "label": "user", 234 | } 235 | ) 236 | return nodes, edges 237 | 238 | 239 | def explore_pg_grants(cur, service_name): 240 | """Retrieves info about a PG grants to users""" 241 | 242 | nodes = [] 243 | edges = [] 244 | cur.execute( 245 | """ 246 | SELECT grantee, table_schema, table_name, 247 | privilege_type,is_grantable 248 | FROM information_schema.role_table_grants; 249 | """ 250 | ) 251 | 252 | role_table_grants = cur.fetchall() 253 | 254 | for role_table_grant in role_table_grants: 255 | edges.append( 256 | { 257 | "from": "pg~" + service_name + "~user~" + role_table_grant[0], 258 | "to": "pg~" 259 | + service_name 260 | + "~schema~" 261 | + role_table_grant[1] 262 | + "~table_view~" 263 | + role_table_grant[2], 264 | "label": "grant", 265 | "privilege_type": role_table_grant[3], 266 | "is_grantable": role_table_grant[4], 267 | } 268 | ) 269 | return nodes, edges 270 | 271 | 272 | def explore_pg_columns(cur, service_name): 273 | """Retrieves info about a PG columns""" 274 | 275 | nodes = [] 276 | edges = [] 277 | 278 | cur.execute( 279 | """ 280 | select table_catalog, table_schema, table_name, column_name, 281 | data_type, is_nullable from information_schema.columns 282 | where table_schema not in ('information_schema', 'pg_catalog'); 283 | """ 284 | ) 285 | 286 | columns = cur.fetchall() 287 | for column in columns: 288 | nodes.append( 289 | { 290 | "id": "pg~" 291 | + service_name 292 | + "~schema~" 293 | + column[1] 294 | + "~table_view~" 295 | + column[2] 296 | + "~column~" 297 | + column[3], 298 | "service_type": "pg", 299 | "type": "table column", 300 | "data_type": column[4], 301 | "is_nullable": column[5], 302 | "label": column[3], 303 | } 304 | ) 305 | edges.append( 306 | { 307 | "from": "pg~" 308 | + service_name 309 | + "~schema~" 310 | + column[1] 311 | + "~table_view~" 312 | + column[2] 313 | + "~column~" 314 | + column[3], 315 | "to": "pg~" 316 | + service_name 317 | + "~schema~" 318 | + column[1] 319 | + "~table_view~" 320 | + column[2], 321 | "label": "column", 322 | } 323 | ) 324 | return nodes, edges 325 | 326 | 327 | def explore_pg_views(cur, service_name): 328 | 329 | nodes = [] 330 | edges = [] 331 | 332 | cur.execute( 333 | """ 334 | select table_catalog, table_schema, table_name, view_definition, 335 | check_option, is_updatable, is_insertable_into, 336 | is_trigger_updatable, is_trigger_deletable, is_trigger_insertable_into 337 | from information_schema.views 338 | where table_schema not in ('information_schema', 'pg_catalog'); 339 | """ 340 | ) 341 | 342 | views = cur.fetchall() 343 | for view in views: 344 | nodes.append( 345 | { 346 | "id": "pg~" 347 | + service_name 348 | + "~schema~" 349 | + view[1] 350 | + "~table_view~" 351 | + view[2], 352 | "service_type": "pg", 353 | "type": "view", 354 | "view_definition": view[3], 355 | "check_option": view[4], 356 | "is_updatable": view[5], 357 | "is_insertable_into": view[6], 358 | "is_trigger_updatable": view[7], 359 | "is_trigger_deletable": view[8], 360 | "is_trigger_insertable_into": view[9], 361 | "label": view[2], 362 | } 363 | ) 364 | edges.append( 365 | { 366 | "from": "pg~" + service_name + "~schema~" + view[1], 367 | "to": "pg~" 368 | + service_name 369 | + "~schema~" 370 | + view[1] 371 | + "~table_view~" 372 | + view[2], 373 | "label": "view", 374 | } 375 | ) 376 | 377 | new_nodes, new_edges = sql.explore_sql( 378 | view[3], service_name, view[1], view[2], "pg" 379 | ) 380 | nodes = nodes + new_nodes 381 | edges = edges + new_edges 382 | return nodes, edges 383 | 384 | # new_nodes, new_edges = sql.explore_sql( 385 | # "create view testview as select a, b from test", 386 | # "cavallo", 387 | # "serpente", 388 | # "kafka", 389 | # ) 390 | -------------------------------------------------------------------------------- /src/pg_store_tbl.sql: -------------------------------------------------------------------------------- 1 | create table metadata_parser_nodes( 2 | id text, 3 | json_content jsonb, 4 | primary key (id)); 5 | 6 | create table metadata_parser_edges( 7 | source_id text, 8 | destination_id text, 9 | json_content jsonb, 10 | primary key (source_id, destination_id), 11 | constraint fk_source_id foreign key (source_id) references metadata_parser_nodes(id), 12 | constraint fk_destination_id foreign key (destination_id) references metadata_parser_nodes(id) 13 | ); 14 | 15 | CREATE INDEX metadata_parser_nodes_idx ON metadata_parser_nodes USING GIN (json_content); 16 | 17 | CREATE INDEX metadata_parser_edges_idx ON metadata_parser_edges USING GIN (json_content); 18 | 19 | COMMIT; 20 | -------------------------------------------------------------------------------- /src/pyvis_display.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | from networkx.drawing.nx_pydot import write_dot 3 | from networkx.readwrite.gml import write_gml 4 | from pyvis.network import Network 5 | 6 | colors = {} 7 | colors["service"] = "#ff0000" 8 | colors["topic"] = "#00ff00" 9 | colors["topic-acl"] = "#003300" 10 | colors["database"] = "#0000ff" 11 | colors["schema"] = "#000077" 12 | colors["table"] = "#000033" 13 | colors["view"] = "#000033" 14 | colors["table column"] = "#000011" 15 | colors["user"] = "#0000AA" 16 | colors["flink application"] = "#AA0000" 17 | colors["flink job"] = "#660000" 18 | colors["flink application version"] = "#330000" 19 | colors["external_endpoint"] = "#cc0000" 20 | colors["kafka-connect"] = "#dd0000" 21 | colors["external-postgresql"] = "#0000ff" 22 | colors["external-postgresql-schema"] = "#0000ff" 23 | colors["external-postgresql-table"] = "#0000ff" 24 | colors["dashboard"] = "#0000cc" 25 | colors["datasource"] = "#0000cc" 26 | colors["index"] = "#0000cc" 27 | colors["connection-error"] = "#0000cc" 28 | colors["tag"] = "#0000cc" 29 | colors["backup"] = "#cccccc" 30 | colors["consumer_group"] = "#0000cc" 31 | colors["partition"] = "#0000cc" 32 | colors["service_nodes"] = "#0000cc" 33 | colors["service_nodes"] = "#0000cc" 34 | colors["reference"] = "#0000cc" 35 | colors["sql_reference"] = "#0000cc" 36 | 37 | 38 | sizes = {} 39 | 40 | sizes["service"] = 30 41 | sizes["topic"] = 15 42 | sizes["topic-acl"] = 15 43 | sizes["database"] = 20 44 | sizes["schema"] = 15 45 | sizes["table"] = 15 46 | sizes["view"] = 15 47 | sizes["user"] = 15 48 | sizes["flink application"] = 15 49 | sizes["flink job"] = 25 50 | sizes["flink application version"] = 10 51 | sizes["table column"] = 10 52 | sizes["external_endpoint"] = 30 53 | sizes["kafka-connect"] = 25 54 | sizes["external-postgresql"] = 30 55 | sizes["external-postgresql-schema"] = 15 56 | sizes["external-postgresql-table"] = 15 57 | sizes["dashboard"] = 15 58 | sizes["datasource"] = 15 59 | sizes["index"] = 15 60 | sizes["connection-error"] = 15 61 | sizes["tag"] = 10 62 | sizes["backup"] = 10 63 | sizes["consumer_group"] = 10 64 | sizes["partition"] = 10 65 | sizes["service_node"] = 10 66 | sizes["reference"] = 10 67 | sizes["sql_reference"] = 10 68 | 69 | 70 | images = {} 71 | images["service"] = "img/monitor.png" 72 | images["topic"] = "img/table.png" 73 | images["topic-acl"] = "img/document.png" 74 | images["acl"] = "img/document.png" 75 | images["database"] = "img/database.png" 76 | images["schema"] = "img/document.png" 77 | images["table"] = "img/table.png" 78 | images["view"] = "img/table.png" 79 | images["user"] = "img/user.png" 80 | images["flink application"] = "img/table.png" 81 | images["flink job"] = "img/engineering.png" 82 | images["flink application version"] = "img/layout.png" 83 | images["table column"] = "img/layout.png" 84 | images["external_endpoint"] = "img/ext-monitor.png" 85 | images["kafka-connect"] = "img/engineering.png" 86 | images["external-postgresql"] = "img/service.png" 87 | images["external-postgresql-schema"] = "img/document.png" 88 | images["external-postgresql-table"] = "img/table.png" 89 | images["external-postgresql-database"] = "img/table.png" 90 | images["dashboard"] = "img/dashboard.png" 91 | images["datasource"] = "img/data-source.png" 92 | images["index"] = "img/table.png" 93 | images["connection-error"] = "img/warning.png" 94 | images["tag"] = "img/tag.png" 95 | images["backup"] = "img/database.png" 96 | images["consumer_group"] = "img/user.png" 97 | images["partition"] = "img/layout.png" 98 | images["service_node"] = "img/servers.png" 99 | images["reference"] = "img/reference.png" 100 | images["sql_reference"] = "img/sql_reference.png" 101 | 102 | 103 | def pyviz_graphy(nodes, edges): 104 | print() 105 | print("Calulating network graph") 106 | # g = Network(height='750px', width='100%') 107 | g = nx.DiGraph() 108 | for node in nodes: 109 | if isinstance(node, dict) and node["id"] is None: 110 | print(f"Ignoring node {node} - it has id None") 111 | continue 112 | img = ( 113 | images.get(node["type"]) 114 | if images.get(node["type"]) 115 | else "unknown.png" 116 | ) 117 | if node["type"] == "service": 118 | img = "img/services/" + node["service_type"] + ".svg" 119 | 120 | nodesize = sizes.get(node["type"]) if sizes.get(node["type"]) else 10 121 | nodecolor = ( 122 | colors.get(node["type"]) if colors.get(node["type"]) else "#cccccc" 123 | ) 124 | g.add_node( 125 | '"' + node["id"] + '"', 126 | color=nodecolor, 127 | title='"' 128 | + str(node) 129 | .replace(",", ",
") 130 | .replace("{", "{
") 131 | .replace("}", "
}") 132 | + '"', 133 | size=nodesize, 134 | label='"' + node["label"] + '"', 135 | shape="image", 136 | image=img, 137 | type=node["type"] if node["type"] else "NoNodeType", 138 | service_type=node["service_type"], 139 | id='"' + node["id"] + '"', 140 | ) 141 | 142 | if node["id"] == None: 143 | print(node) 144 | 145 | for edge in edges: 146 | # print(str(edge)) 147 | if edge["from"] is None or edge["to"] is None: 148 | print(f"Ignoring edge {edge} - one or both ends is None") 149 | continue 150 | g.add_edge( 151 | '"' + edge["from"] + '"', 152 | '"' + edge["to"] + '"', 153 | title='"' 154 | + str(edge) 155 | .replace(",", ",
") 156 | .replace("{", "{
") 157 | .replace("}", "
}") 158 | + '"', 159 | physics=False, 160 | ) 161 | if edge["from"] == None or edge["to"] == None: 162 | print(edge) 163 | 164 | print() 165 | print("Writing DOT file") 166 | try: 167 | write_dot(g, "graph_data.dot") 168 | except Exception as err: 169 | print(f"Error writing DOT file: {err.__class__.__name__} {err}") 170 | 171 | print("Writing GML file") 172 | try: 173 | write_gml(g, "graph_data.gml") 174 | except Exception as err: 175 | print(f"Error writing GML file: {err.__class__.__name__} {err}") 176 | 177 | print("Writing NX file") 178 | try: 179 | nt = Network(height="1000px", width="1000px", font_color="#000000") 180 | nt.from_nx(g) 181 | # nt.show_buttons() 182 | nt.show_buttons() 183 | nt.set_edge_smooth("continuous") 184 | nt.show("nx.html") 185 | # print (g.nodes) 186 | except Exception as err: 187 | print(f"Error writing NX file: {err.__class__.__name__} {err}") 188 | 189 | ## All images are taken from https://www.flaticon.com/ 190 | -------------------------------------------------------------------------------- /src/redis.py: -------------------------------------------------------------------------------- 1 | """Parsing Redis service""" 2 | 3 | 4 | def explore_redis(self, service, service_name, project, service_map): 5 | """Explores an Redis service""" 6 | nodes = [] 7 | edges = [] 8 | host = service["service_uri_params"]["host"] 9 | port = service["service_uri_params"]["port"] 10 | 11 | # Exploring Users and ACL 12 | for user in service["users"]: 13 | 14 | user_node_id = f"redis~{service_name}~user~{user['username']}" 15 | 16 | nodes.append( 17 | { 18 | "id": user_node_id, 19 | "service_type": "redis", 20 | "type": "user", 21 | "user_type": user["type"], 22 | "label": user["username"], 23 | } 24 | ) 25 | edges.append( 26 | {"from": user_node_id, "to": service_name, "label": "user"} 27 | ) 28 | 29 | user_acl_info = user["access_control"] 30 | acl_node_id = f"redis~user-acl~id~{user['username']}" 31 | 32 | nodes.append( 33 | { 34 | "id": acl_node_id, 35 | "service_type": "user-acl", 36 | "type": "acl", 37 | "label": user["username"] + "-user-acl", 38 | "access_control": user_acl_info, 39 | } 40 | ) 41 | edges.append({"from": user_node_id, "to": acl_node_id, "label": "acl"}) 42 | 43 | # need to finish Could query redis and add more parsed data from that 44 | return host, port, nodes, edges, service_map 45 | -------------------------------------------------------------------------------- /src/sql.py: -------------------------------------------------------------------------------- 1 | """Parsing SQL""" 2 | 3 | from sqllineage.runner import LineageRunner 4 | from sqllineage.core.models import SubQuery 5 | 6 | 7 | def explore_sql(sql_statement, service_name, schema, target, service_type): 8 | """Parsing SQL""" 9 | nodes = [] 10 | edges = [] 11 | 12 | if not sql_statement.upper().startswith("INSERT"): 13 | sql_statement = "INSERT INTO " + target + " AS " + sql_statement 14 | runner = LineageRunner( 15 | sql_statement, 16 | verbose=False, 17 | draw_options={ 18 | "host": "abc", 19 | "port": 123, 20 | "f": "sql.sql", 21 | }, 22 | ) 23 | 24 | for line in runner.get_column_lineage(exclude_subquery=False): 25 | prev_col_id = None 26 | 27 | for col in reversed(line): 28 | is_subquery = False 29 | if isinstance(col.parent, SubQuery): 30 | table_name = col.parent.alias 31 | is_subquery = True 32 | else: 33 | table_name = col.parent.raw_name 34 | column_name = col.raw_name 35 | 36 | if table_name is None: 37 | # This is the case where is a reference to the end column 38 | new_col_id = ( 39 | service_type 40 | + "~" 41 | + service_name 42 | + "~schema~" 43 | + schema 44 | + "~table_view~" 45 | + target 46 | + "~" 47 | + "column" 48 | + "~" 49 | + column_name 50 | ) 51 | else: 52 | new_col_id = ( 53 | service_type 54 | + "~" 55 | + service_name 56 | + "~schema~" 57 | + schema 58 | + "~table_view~" 59 | + table_name 60 | + "~column~" 61 | + column_name 62 | ) 63 | if is_subquery: 64 | nodes.append( 65 | { 66 | "id": new_col_id, 67 | "service_type": service_type, 68 | "type": "reference", 69 | "label": column_name, 70 | } 71 | ) 72 | nodes.append( 73 | { 74 | "id": new_col_id.split("~column~")[0], 75 | "service_type": service_type, 76 | "type": "sql_reference", 77 | "label": new_col_id.split("~column~")[0].split( 78 | "~table_view~" 79 | )[1], 80 | } 81 | ) 82 | edges.append( 83 | { 84 | "from": new_col_id, 85 | "to": new_col_id.split("~column~")[0], 86 | "type": "sql_reference", 87 | } 88 | ) 89 | edges.append( 90 | { 91 | "from": service_type 92 | + "~" 93 | + service_name 94 | + "~schema~" 95 | + schema 96 | + "~table_view~" 97 | + target, 98 | "to": new_col_id.split("~column~")[0], 99 | "type": "sql_reference", 100 | "label": target, 101 | } 102 | ) 103 | 104 | if prev_col_id is not None: 105 | edges.append( 106 | { 107 | "from": new_col_id, 108 | "to": prev_col_id, 109 | "type": "sql_reference", 110 | } 111 | ) 112 | 113 | prev_col_id = new_col_id 114 | return nodes, edges 115 | -------------------------------------------------------------------------------- /src/tag.py: -------------------------------------------------------------------------------- 1 | """Parsing service tags""" 2 | 3 | 4 | def explore_tags(self, service_name, service_type, project): 5 | """Parsing service tags""" 6 | 7 | nodes = [] 8 | edges = [] 9 | 10 | tags = self.list_service_tags(service=service_name, project=project) 11 | 12 | for key, value in tags["tags"].items(): 13 | nodes.append( 14 | { 15 | "id": "tag~id~" + key + "~value~" + value, 16 | "service_type": "tag", 17 | "type": "tag", 18 | "label": key + "=" + value, 19 | } 20 | ) 21 | edges.append( 22 | { 23 | "from": service_name, 24 | "to": "tag~id~" + key + "~value~" + value, 25 | "label": "tag", 26 | } 27 | ) 28 | return nodes, edges 29 | -------------------------------------------------------------------------------- /write_pg.py: -------------------------------------------------------------------------------- 1 | "Writes results to PG" 2 | 3 | import argparse 4 | import json 5 | from networkx.readwrite.gml import read_gml 6 | import psycopg2 7 | 8 | parser = argparse.ArgumentParser(description="Push data to PG Database.") 9 | parser.add_argument("uri", metavar="N", help="pass the PG URI") 10 | args = parser.parse_args() 11 | print(args.uri) 12 | connstr = args.uri 13 | 14 | try: 15 | conn = psycopg2.connect(connstr, connect_timeout=2) 16 | except psycopg2.Error as err: 17 | conn = None 18 | print(str(err)) 19 | 20 | cur = conn.cursor() 21 | cur.execute( 22 | """ 23 | select exists( 24 | select * from information_schema.tables where table_name=%s 25 | ) 26 | """, 27 | ("metadata_parser_nodes",), 28 | ) 29 | if cur.fetchone()[0]: 30 | cur.execute("TRUNCATE TABLE metadata_parser_nodes cascade;") 31 | conn.commit() 32 | else: 33 | cur.execute(open("src/pg_store_tbl.sql", "r").read()) 34 | 35 | G = read_gml("graph_data.gml") 36 | 37 | for node in G.nodes(): 38 | json_rep = json.loads( 39 | G.nodes[node] 40 | .get("title") 41 | .replace("
", "") 42 | .replace("'", '"') 43 | .replace("True", "true") 44 | .replace("False", "false") 45 | .replace("None", "[]")[1:-1] 46 | ) 47 | print(json_rep["id"]) 48 | cur.execute( 49 | "insert into metadata_parser_nodes values(%s, %s);", 50 | (json_rep["id"], json.dumps(json_rep)), 51 | ) 52 | conn.commit() 53 | 54 | 55 | for edge in G.edges(): 56 | json_rep = json.loads( 57 | G.edges[edge]["title"] 58 | .replace("
", "") 59 | .replace("'", '"') 60 | .replace("True", "true") 61 | .replace("False", "false") 62 | .replace("None", "[]")[1:-1] 63 | ) 64 | print(json.dumps(json_rep)) 65 | cur.execute( 66 | "insert into metadata_parser_edges values(%s, %s, %s);", 67 | ( 68 | json_rep["from"], 69 | json_rep["to"], 70 | json.dumps(json_rep), 71 | ), 72 | ) 73 | cur.execute( 74 | "insert into metadata_parser_edges values(%s, %s, %s);", 75 | ( 76 | json_rep["to"], 77 | json_rep["from"], 78 | json.dumps(json_rep), 79 | ), 80 | ) 81 | 82 | conn.commit() 83 | --------------------------------------------------------------------------------