├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Readme.md ├── config ├── __init__.py ├── links.yaml.template └── settings.py.template ├── dashboard ├── __init__.py ├── app.py ├── blueprints │ ├── __init__.py │ └── page │ │ ├── __init__.py │ │ ├── templates │ │ ├── 500.html │ │ ├── etl_dashboard.html │ │ ├── etl_details.html │ │ ├── index.html │ │ ├── login.html │ │ └── no_config.html │ │ └── views.py ├── dataproviders │ ├── __init__.py │ ├── airflow.py │ └── etl.py ├── model │ ├── __init__.py │ ├── influxdb_data.py │ ├── links_data.py │ ├── prometheus_data.py │ └── tables_data.py ├── models.py ├── plugins │ ├── __init__.py │ ├── athena_usage │ │ ├── README.md │ │ ├── __init__.py │ │ ├── athena_query_model.py │ │ ├── athena_summary_provider.py │ │ ├── athena_usage.png │ │ ├── query_dao.py │ │ └── templates │ │ │ └── athena_usage │ │ │ └── index.html │ ├── hello_world │ │ ├── README.md │ │ ├── __init__.py │ │ └── templates │ │ │ └── hello_world │ │ │ └── index.html │ ├── reports │ │ ├── README.md │ │ ├── __init__.py │ │ ├── reports.yaml.template │ │ ├── reports_data.py │ │ ├── reports_list.png │ │ └── templates │ │ │ └── reports │ │ │ └── index.html │ ├── s3_usage │ │ ├── README.md │ │ ├── __init__.py │ │ ├── refresher.py │ │ ├── screenshot.png │ │ ├── statsdb.py │ │ ├── templates │ │ │ └── s3_usage │ │ │ │ ├── index.html │ │ │ │ ├── initializing.html │ │ │ │ └── partial_storage_classes.html │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── s3stats.sql │ │ │ ├── test_refresher.py │ │ │ └── test_statsdb.py │ ├── streaming │ │ ├── README.md │ │ ├── __init__.py │ │ ├── streaming.png │ │ ├── streaming.yaml.template │ │ └── templates │ │ │ └── streaming │ │ │ └── index.html │ ├── table_descriptions │ │ ├── README.md │ │ ├── __init__.py │ │ ├── dataproviders.py │ │ ├── table_descriptions.png │ │ └── templates │ │ │ └── table_descriptions │ │ │ └── index.html │ └── tables │ │ ├── README.md │ │ ├── __init__.py │ │ ├── table_details.png │ │ ├── tables.yaml.template │ │ ├── tables_list.png │ │ └── templates │ │ └── tables │ │ ├── details.html │ │ └── index.html ├── service │ ├── influxdb_service.py │ ├── mysql.py │ └── prometheus_service.py ├── static │ ├── css │ │ ├── base.css │ │ ├── descriptions.css │ │ ├── etl_dashboard.css │ │ ├── reports.css │ │ ├── style.css.map │ │ ├── style.min.css │ │ ├── style.min.css.map │ │ └── table_dashboard.css │ ├── favicon.ico │ ├── images │ │ ├── graph_icon.svg │ │ └── ui_screen.png │ ├── js │ │ ├── colors.js │ │ ├── colors.js.map │ │ ├── highlight.js │ │ ├── main.js.map │ │ ├── popovers.js │ │ ├── popovers.js.map │ │ ├── render_vis.js │ │ ├── search.js │ │ ├── src │ │ │ ├── charts.js │ │ │ ├── colors.js │ │ │ ├── main.js │ │ │ ├── popovers.js │ │ │ ├── tooltips.js │ │ │ └── widgets.js │ │ ├── tooltips.js │ │ ├── tooltips.js.map │ │ ├── widgets.js │ │ └── widgets.js.map │ └── vendors │ │ └── @coreui │ │ └── coreui-plugin-chartjs-custom-tooltips │ │ └── js │ │ ├── custom-tooltips.min.js │ │ └── custom-tooltips.min.js.map ├── templates │ ├── layouts │ │ ├── base.html │ │ └── clean.html │ └── macros │ │ └── common.html ├── tests │ ├── Dockerfile │ ├── __init__.py │ ├── dataproviders │ │ ├── __init__.py │ │ └── test_airflow.py │ ├── docker-compose.yml │ └── mocks │ │ ├── __init__.py │ │ ├── airflow.sql │ │ └── database.py └── utils │ ├── __init__.py │ └── vis.py ├── examples └── extra.html └── requirements.txt /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | */.idea/* 3 | *.pyc 4 | settings.py -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: "3.6" 3 | 4 | services: 5 | - docker 6 | install: 7 | - docker build -t dashboard:tests -f dashboard/tests/Dockerfile . 8 | script: 9 | - docker-compose --file dashboard/tests/docker-compose.yml up --abort-on-container-exit && docker-compose --file dashboard/tests/docker-compose.yml down 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # VERSION 0.3.0 2 | 3 | ## Base 4 | 5 | ### Changes 6 | 7 | * [Do not take manual task success as task exection](https://github.com/Wikia/discreETLy/pull/25/files) 8 | * [Bump DiscreETLy to Python 3.7 and update loginpass](https://github.com/Wikia/discreETLy/pull/28) 9 | * [Replace Fandom/Wikia branding with `COMPANY_NAME`](https://github.com/Wikia/discreETLy/issues/31) 10 | * [Make dataprovider Airflow compatible with MySQL 5.7](https://github.com/Wikia/discreETLy/issues/33) 11 | 12 | ### Fixes 13 | 14 | * [Fix issue where ETL dashboard displayed too many icons when a DAG had some failing statuses](https://github.com/Wikia/discreETLy/pull/26) 15 | * [Fix non-displaying icons](https://github.com/Wikia/discreETLy/issues/32) 16 | 17 | ## Plugins 18 | 19 | ### Changes 20 | 21 | * Tables (tables managed by..): Add possibility of declaring dependency on table from another DAG [#22](https://github.com/Wikia/discreETLy/pull/23/files), [#21](https://github.com/Wikia/discreETLy/pull/21) 22 | 23 | ### Fixes 24 | 25 | * [S3 Usage: Fixed HTML title for initialization phase](https://github.com/Wikia/discreETLy/issues/34) 26 | * [S3 Usage: Move S3 database so the plugin can be used as non-root user](https://github.com/Wikia/discreETLy/pull/27) 27 | 28 | # VERSION 0.2.9 29 | 30 | ## Plugins 31 | 32 | ### Changes 33 | * Fix rendering of page with "Tables managed by ..." 34 | 35 | # VERSION 0.2.8 36 | 37 | ## Plugins 38 | 39 | ### Changes 40 | * Streaming tab now shows the lag of stream consumption on time-based graph (last day/week/month) 41 | 42 | # VERSION 0.2.7 43 | 44 | ## Base 45 | 46 | ### Fixes 47 | 48 | * Fixed Firefox autorefresh issue on Table Descriptions and S3 Usage tabs 49 | * Favicon added 50 | 51 | # VERSION 0.2.6 52 | 53 | ## Plugins 54 | 55 | ### Changes 56 | * New plugin added: Athena Usage [(documentation)](dashboard/plugins/athena_usage/README.md) 57 | 58 | # VERSION 0.2.5 59 | 60 | ## Base 61 | 62 | ### Changes 63 | 64 | * Plugin manifest is changed to include the `init` fnction 65 | * New utils: `Timer` for external system call measurment and `sizeof_fmt` for formatting human readable object sizes [(source)](https://stackoverflow.com/a/1094933/7098262) 66 | 67 | ## Plugins 68 | 69 | ### Changes 70 | * New plugin added: S3 Usage [(documentation)](dashboard/plugins/s3_usage/README.md) 71 | 72 | ### Fixes 73 | * Auto-refresh disabled on Table descriptions tab 74 | 75 | # VERSION 0.2.4 76 | 77 | This is a version with tiny changes to layout and functionality as some bugfixes. It does not introduce new features. 78 | 79 | ## Base 80 | 81 | #### Changes 82 | 83 | * The labels on main site are no longer `button` type so the cursor does not suggest they are clickable 84 | * The help button on top bar with breadcrumbs now reacts to `click` event rather than `hover` event 85 | * Changed hover behaviour when clicking clear button in search field (also changed color on hover) 86 | 87 | #### Fixes 88 | 89 | * Fixed link to github documentation on help button 90 | 91 | ## Plugins 92 | 93 | ### Tables descriptions 94 | 95 | #### Changes 96 | 97 | * By default all table details are now hidden when opening the tab 98 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-alpine 2 | 3 | WORKDIR /app 4 | 5 | COPY requirements.txt requirements.txt 6 | RUN apk add --update --no-cache mariadb-connector-c \ 7 | && apk add --no-cache --virtual .build-deps mariadb-dev gcc musl-dev gcc build-base libffi-dev cargo \ 8 | && pip install --upgrade pip \ 9 | && pip install -r requirements.txt \ 10 | && apk del .build-deps 11 | 12 | RUN mkdir /var/run/discreetly 13 | 14 | COPY . . 15 | 16 | CMD ["sh", "-c", "SECRET_KEY=`cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1` gunicorn --worker-class sync --log-level DEBUG --reload -b 0.0.0.0:8000 --graceful-timeout 5 --workers 2 --access-logfile - 'dashboard.app:create_app()'"] 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 FANDOM 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # DiscreETLy 2 | 3 | ## No longer maintained! 4 | 5 | DiscreETLy's development has been stopped and the repository is put into archived, read-only mode. 6 | We recommend looking into alternative Data Catalog solutions, like: 7 | - [Amudsen](https://github.com/amundsen-io/amundsen) 8 | - [DataHub](https://github.com/datahub-project/datahub) 9 | 10 | ## Legacy docs 11 | **DiscreETLy** was an add-on dashboard service on top of [Apache Airflow](https://github.com/apache/incubator-airflow). It is a user friendly UI showing status of particular DAGs. Moreover, it allows the users to map Tasks within a particular DAG to tables available in any system (relational and non-relational) via friendly yaml definition. **DiscreETLy** provides fuctionality for monitoring DAGs status as well as optional communication with services such as [Prometheus](https://prometheus.io/) or [InfluxDB](https://www.influxdata.com/). 12 | 13 | ![screenshot](https://raw.githubusercontent.com/Wikia/discreETLy/master/dashboard/static/images/ui_screen.png) 14 | 15 | ### Prerequisites 16 | 17 | Minimal setup required to run the dashboard requires `docker`. You can find docker installation instructions on 18 | [official docker website](https://docs.docker.com/install/). 19 | 20 | The minimal setup requires also access to Airflow MySQL instance (MySQL version should be >= 8 and allow analytical functions). 21 | 22 | ### Configuration 23 | 24 | Before running or deploying **DiscreETLy** a configuration file needs to be provided. The template for configuration 25 | file can be found in `config` folder: `settings.py.template`. Configuration is provided as a standard python file, 26 | which makes it easy to define and change utilizng Python APIs. The bare minimum configuration needed for the app to run requires definition of a secret key (stub provided in template document) and connection details for Airflow database (currently only MySQL is supported). 27 | 28 | Configuration options for InfluxDB and Prometheus are optional. If those services are not defined in configuration 29 | file they will be simply ignored while running the app. 30 | 31 | If environment is not specified, the application is run in DEBUG mode, so any errors will be reported on dashboard UI. If environment variable `ENV_VAR_PREFIX` is set to `PROD` or appropriate 32 | option is changed in `settings.py` file the application will serve `500` errors as defined in dashboard [template](dashboard/blueprints/page/templates/500.html). 33 | 34 | ### Views & Plugins 35 | 36 | The basic configuration file is enough to run the dashboard, however, in order to take 37 | full advantage of dashboard features and functionality there are some additional steps 38 | that need to be performed. By default only **ETL** tab presents valuable information, 39 | allowing users to monitor progress of particular Airflow DAGs and tasks. 40 | But you can easily enable plugins that configure new views. All plugins reside in 41 | [plugins directory](dashboard/plugins) and are enabled if the plugin name is present 42 | in the `ENABLED_PLUGINS` in the `settings.py`. 43 | 44 | You can find more details on what each plugin provides and how to configure it 45 | in the following docs: 46 | 47 | * [Tables](dashboard/plugins/tables/README.md) - tables list: status monitoring and 48 | records count 49 | * [Reports](dashboard/plugins/reports/README.md) - monitoring of the reports external 50 | to the Aiflow DAGs (like Tableu, Mode) 51 | * [Streaming](dashboard/plugins/streaming/README.md) - view on non-Airflow based 52 | streaming applications 53 | * [Table Descritpions](dashboard/plugins/table_descriptions/README.md) - table and 54 | columns description 55 | * [S3 Usage](dashboard/plugins/s3_usage/README.md) - browser of the aggregated metadata 56 | of files stored inside mutliple S3 buckets 57 | * [Athena Usage](dashboard/plugins/athena_usage/README.md) - summaries 58 | of users' queries data consumption to Athena 59 | * [Hello World](dashboard/plugins/hello_world/README.md) - DiscreETLy Developers Docs 60 | * Important links - update links in config/links.yaml 61 | 62 | 63 | ### Running locally 64 | 65 | See: https://hub.docker.com/r/fandom/discreetly/ 66 | 67 | Before running the container with app we first need to build it so it becomes available 68 | in our local docker repository. Run the following command from project's root directory. 69 | 70 | ```bash 71 | docker build -t : . 72 | ``` 73 | 74 | Once the image is build the application can be triggered by running: 75 | 76 | ```bash 77 | docker run -e = --rm --name -v :/app -p 8000:8000 : 78 | ``` 79 | 80 | Let's dissect this command option by option: 81 | 82 | - `-e` flag allows to set up different evnvironment varaibles required to e.g. configure the app. Most of those options can be hardcoded in configuration file, however, passing them through environment is recommended. For more detials see **[configuration](#configuration)** section of this README. 83 | - `--rm` removes the container after stopping it. It ensures that there is always a fresh version of conpfiguration and other features while running the app. 84 | - `-v` maps folders containing application from local environment to container. It ensures that if in development mode all changes applied to files on local file system are immediately reflected in container. 85 | - `-p` maps a port from container to `localhost` 86 | 87 | If some of the configuration options are already available through `settings.py` file the command for running the application can be significantly abbreviated (from project root folder): 88 | 89 | ```bash 90 | docker run --rm -v $(pwd):/app -p 8000:8000 fandom/discreetly:latest 91 | ``` 92 | 93 | Remember to use docker image name and version provided during `build` stage. 94 | 95 | Once the container is ready and running navigate to `localhost:8000` in a browser and enjoy. 96 | 97 | #### Testing 98 | 99 | In order to run the tests a docker image needs to be build first. The Dockerfile is available in `dashboard/tests/` folder. To build an image one can run the following command from project's root directory: 100 | 101 | ```bash 102 | docker build -t dashboard:tests -f dashboard/tests/Dockerfile . 103 | ``` 104 | 105 | Once the image is build the tests can be preformed by typing 106 | 107 | ```bash 108 | docker-compose --file dashboard/tests/docker-compose.yml up --abort-on-container-exit && docker-compose --file dashboard/tests/docker-compose.yml down 109 | ``` 110 | 111 | The output of this command shows a nicely formatted information of number of tests performed and success ratio (all tests are performed by using `pytest` package). 112 | 113 | If working iteratively rebuilding the image everytime some changes are made would be cumbersome. In order to avoid that one can pass additional parameter to subsequent runs (mapping of a local project folder to container destination): 114 | 115 | ```bash 116 | docker run --rm -v :/tmp/dashboard/ dashboard:tests 117 | ``` 118 | 119 | ## Credits 120 | DiscreETLy was maintained by [Fandom's](https://github.com/Wikia) Data Engineering team 121 | 122 | --- 123 | * Concept & Product Design: 124 | [JoannaBorun](https://github.com/JoannaBorun) 125 | * Development 126 | [esthomw](https://github.com/esthomw) 127 | [jcellary](https://github.com/jcellary) 128 | [szczeles](https://github.com/szczeles) 129 | [MikolajBalcerek](https://github.com/MikolajBalcerek) 130 | * QA 131 | [klistiano](https://github.com/klistiano) 132 | -------------------------------------------------------------------------------- /config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/config/__init__.py -------------------------------------------------------------------------------- /config/links.yaml.template: -------------------------------------------------------------------------------- 1 | - name: 'The best Athena dashboard ;)' 2 | url: 'https://github.com/Wikia/discreETLy' 3 | - name: 'Sroka - Python library for API access and data analysis in Product, BI, Revenue Operations (GAM, GA, Athena etc.)' 4 | url: 'hhttps://github.com/Wikia/sroka' -------------------------------------------------------------------------------- /config/settings.py.template: -------------------------------------------------------------------------------- 1 | # This is a configuration file for dashboard application. This file is a standard python file, so if needed 2 | # it supports all python operations, e.g. os package for working with operating system specific features. 3 | 4 | import os 5 | 6 | ENV_VAR_PREFIX = os.getenv('ENV_VAR_PREFIX', '') 7 | DEBUG = not 'PROD' in ENV_VAR_PREFIX 8 | 9 | # Airflow database 10 | # Define configuration options required to connect to your Airflow database instance. 11 | # This configuration file assumes those options are provided through environment variables. 12 | # Database host machine 13 | # AIRFLOW_DB_HOST = os.environ['AIRFLOW_DB_HOST'] 14 | # Database user 15 | # AIRFLOW_USERNAME = os.environ['AIRFLOW_USERNAME'] 16 | # User password 17 | # AIRFLOW_PASSWORD = os.environ['AIRFLOW_PASSWORD'] 18 | # Database instance name 19 | # AIRFLOW_DATABASE = os.environ['AIRFLOW_DATABASE'] 20 | 21 | # This variable holds DNS address definition, e.g 'https://airflow.fandom.com' 22 | # AIRFLOW_WEBSERVER_BASE_URL = os.environ['AIRFLOW_WEBSERVER_BASE_URL'] 23 | 24 | # InfluxDB 25 | # The dashboard supports retriving data from an InfluxDB instance. This part of configuration is entirely optional. 26 | # Leave this section commented out if you want to ignore InfluxDB functionality. 27 | # INFLUXDB_HOST = os.environ['INFLUXDB_HOST'] 28 | # INFLUXDB_USERNAME = os.environ['INFLUXDB_USERNAME'] 29 | # INFLUXDB_PASSWORD = os.environ['INFLUXDB_PASSWORD'] 30 | # INFLUXDB_DATABASE = os.environ['INFLUXDB_DATABASE'] 31 | 32 | # Prometheus 33 | # The dashboard supports retriving data from an Prometheus instance. This part of configuration is entirely optional. 34 | # Leave this section commented out if you want to ignore Prometheus functionality. 35 | # PROMETHEUS_HOST = os.environ['PROMETHEUS_HOST'] 36 | 37 | # OAuth 38 | # The dashboard supports authorization through Google OAuth. This part of configuration is entirely optional. 39 | # Leave this section commented out if you want to ignore OAuth functionality. 40 | # GOOGLE_CLIENT_ID = os.environ['OAUTH_CLIENT_ID'] 41 | # GOOGLE_CLIENT_SECRET = os.environ['OAUTH_SECRET'] 42 | # Use domain that is used for OAuth services at your company, e.g. 'fandom.com' 43 | # You can also use a list ['domain1.com', 'domain2.com'] 44 | # OAUTH_DOMAIN = 45 | 46 | # Application secret 47 | # Secret key used for making the application secure. If deployed on production or other publicly available service it is advisable 48 | # to provide secret. If tested locally the default can be used. 49 | SECRET_KEY = os.environ.get('SECRET_KEY') or '8FF5F14DC7DBF90FDE769C3BF7FDCFA16FEF3D1E' 50 | 51 | # Company name 52 | # This name will be displayed on the header of the dashboard 53 | COMPANY_NAME = 'ACME' 54 | 55 | # Plugins 56 | # DiscreETLy supports views/functionalities customization with plugin system. The following list 57 | # describes what plugins are enabled. They are stored in dashboard/plugins directory with the corresponding 58 | # documentation. 59 | ENABLED_PLUGINS= ['tables', 'hello_world'] 60 | 61 | # Refresh rate - time in seconds when a page gets refreshed and pulls new data 62 | REFRESH_RATE=300 63 | 64 | # Plugin-specific configuration: table_descriptions 65 | # The following lines enable AWS Glue Data catalogue as a source of the table descritpions. This dataprovider 66 | # requires AWS region, access key and secret key to be provided. 67 | #TABLE_DESCRIPTIONS_SERVICE = 'dashboard.plugins.table_descriptions.dataproviders.GlueDescriptionService' 68 | #TABLE_DESCRIPTIONS_SERVICE_PARAMS = { 69 | # 'region_name': 'us-east-1', 70 | # 'aws_access_key_id': '', 71 | # 'aws_secret_access_key': '' 72 | #} 73 | #TABLE_DESCRIPTIONS_TTL = 300 # 5 minutes is the default 74 | 75 | # The following lines configure s3_usage plugin. See plugin documentation for more information. 76 | #S3_USAGE_PARAMS = { 77 | # 'buckets_regexp': '^data-.*$', 78 | # 'aws_access_key_id': '......' 79 | # 'aws_secret_access_key': '....' 80 | #} 81 | 82 | # The following lines configure athena_usage plugin. See plugin documentation for more information. 83 | #ATHENA_USAGE_PARAMS = { 84 | # 'QUERIES_TABLE': '....', 85 | # 'region_name': '....' 86 | # 'aws_access_key_id': '....', 87 | # 'aws_secret_access_key': '....', 88 | #} 89 | 90 | 91 | -------------------------------------------------------------------------------- /dashboard/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/__init__.py -------------------------------------------------------------------------------- /dashboard/app.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures.thread import ThreadPoolExecutor 2 | 3 | import os 4 | import logging 5 | import importlib 6 | from datetime import datetime 7 | from typing import List 8 | 9 | from flask import Flask, session, redirect, request, url_for 10 | from flask_sslify import SSLify 11 | from werkzeug.debug import DebuggedApplication 12 | from authlib.flask.client import OAuth 13 | from loginpass import create_flask_blueprint, Google 14 | from werkzeug.contrib.cache import SimpleCache 15 | 16 | from dashboard.blueprints.page import page 17 | from dashboard.dataproviders import * 18 | from dashboard.service.mysql import MySQLClient 19 | from dashboard.service.influxdb_service import InfluxDbService 20 | from dashboard.model.influxdb_data import InfluxDBData 21 | from dashboard.model.prometheus_data import PrometheusData 22 | from dashboard.model.tables_data import TableDataProvider 23 | from dashboard.model.links_data import LinksDataProvider 24 | from dashboard.models import ExtraEtl 25 | from dashboard.utils import get_yaml_file_content 26 | 27 | TABLES_PATH = 'config/tables.yaml' 28 | LINKS_PATH = 'config/links.yaml' 29 | 30 | 31 | def setup_authentication(app): 32 | oauth = OAuth(app) 33 | 34 | def handle_redirects(remote, token, user_info): 35 | if 'hd' in user_info: 36 | if user_info['hd'] == app.config['OAUTH_DOMAIN'] \ 37 | or (isinstance(app.config['OAUTH_DOMAIN'], List) and user_info['hd'] in app.config['OAUTH_DOMAIN']): 38 | 39 | session['user'] = user_info 40 | return redirect(session.get('next', '/')) 41 | 42 | return redirect(url_for('page.login')) 43 | 44 | 45 | def ensure_user_is_authorized(app): 46 | if 'user' not in session and request.path not in ['/oauth/login', '/oauth/auth', '/login', '/healthcheck'] and not request.path.startswith('/static/'): 47 | session['next'] = request.url 48 | return redirect(url_for('page.login')) 49 | 50 | app.register_blueprint( 51 | create_flask_blueprint(Google, oauth, handle_redirects), 52 | url_prefix='/oauth') 53 | 54 | app.before_request(lambda: ensure_user_is_authorized(app)) 55 | 56 | 57 | def create_app(settings_override=None): 58 | """ 59 | Create a Flask application using the app factory pattern. 60 | :param settings_override: Override settings 61 | :return: Flask app 62 | """ 63 | app = Flask(__name__, instance_relative_config=True) 64 | 65 | def disable_varnish(response): 66 | response.cache_control.private = True 67 | return response 68 | app.after_request(disable_varnish) 69 | 70 | SSLify(app, skips=['healthcheck']) 71 | gunicorn_logger = logging.getLogger('gunicorn.error') 72 | app.logger.handlers = gunicorn_logger.handlers 73 | app.logger.setLevel(gunicorn_logger.level) 74 | 75 | app.config.from_object('config.settings') 76 | 77 | if 'GOOGLE_CLIENT_ID' in app.config: 78 | setup_authentication(app) 79 | 80 | app.register_blueprint(page) 81 | 82 | plugins = {} 83 | for plugin in app.config['ENABLED_PLUGINS']: 84 | module = importlib.import_module(f'dashboard.plugins.{plugin}') 85 | app.register_blueprint(module.plugin, url_prefix=module.base_path) 86 | if hasattr(module, 'init'): 87 | module.init(app) 88 | plugins[plugin] = {'tab_name': module.tab_name} 89 | 90 | app.airflow_data_provider = AirflowDBDataProvider(app.config, app.logger, MySQLClient(app.config, app.logger)) 91 | app.influx_client = InfluxDbService(app.config, app.logger) if app.config.get('INFLUXDB_HOST') else None 92 | app.influx_data_provider = InfluxDBData(app.influx_client, app.logger) if app.config.get('INFLUXDB_HOST') else None 93 | app.prometheus_data_provider = PrometheusData(app.config, app.logger) if app.config.get('PROMETHEUS_HOST') else None 94 | 95 | # Reading tables configs, setting variable to `None` if file is not present 96 | tables = get_yaml_file_content(TABLES_PATH) 97 | 98 | app.table_data_provider = TableDataProvider( 99 | app.airflow_data_provider, app.influx_data_provider, 100 | app.prometheus_data_provider, tables, app.logger, app.config) if tables else None 101 | 102 | links = get_yaml_file_content(LINKS_PATH) 103 | app.links_data_provider = LinksDataProvider(links) 104 | 105 | app.etl_data_provider = EtlDataProvider( 106 | app.config, app.airflow_data_provider, app.table_data_provider) 107 | 108 | app.async_request_executor = ThreadPoolExecutor(max_workers=3) 109 | app.cache = SimpleCache() 110 | 111 | if app.debug: 112 | app.wsgi_app = DebuggedApplication(app.wsgi_app, evalex=True) 113 | 114 | app.context_processor(lambda: {'now': datetime.now(), 'plugins': plugins}) 115 | 116 | return app 117 | 118 | if __name__ == '__main__': # non-gunicorn entry point for debugging 119 | create_app().run('0.0.0.0', 8000) 120 | -------------------------------------------------------------------------------- /dashboard/blueprints/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/blueprints/__init__.py -------------------------------------------------------------------------------- /dashboard/blueprints/page/__init__.py: -------------------------------------------------------------------------------- 1 | from dashboard.blueprints.page.views import page 2 | -------------------------------------------------------------------------------- /dashboard/blueprints/page/templates/500.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/clean.html' %} 2 | 3 | {% block title %}{{ config['COMPANY_NAME'] }} Data - internal server error{% endblock %} 4 | 5 | {% block body %} 6 | 7 |
8 | 9 |

OOPS, something went wrong! Please report the issue to Data Engineering team.

10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /dashboard/blueprints/page/templates/etl_dashboard.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} 2 | {% import 'macros/common.html' as common with context %} 3 | 4 | {% block title %}ETLs{% endblock %} 5 | 6 | {% block nav %} 7 |
  • Home
    • 8 |
  • ETL dashboard
    • 9 | 10 | {% endblock %} 11 | 12 | {% block body %} 13 | 14 | 15 |
    16 |
    17 | 23 |
    24 |
    25 |
    26 |

    Legend:

    27 |
    28 |
    29 |

    ◉ Success

    30 |
    31 |
    32 |

    ◉ In Progress

    33 |
    34 |
    35 |

    ◉ Failure

    36 |
    37 |
    38 |
    39 | 40 |
    41 |
    42 |
    43 |
    ETL name
    44 |
    Latest run (UTC)
    45 |
    ETL state
    46 |
    Duration (sec)
    47 |
    Success | Failure
    48 |
    49 | {% for etl in etls|sort(attribute='name') %} 50 |
    51 |
    52 |

    {{ etl.name }}

    53 |
    54 |
    55 |

    56 | {{ etl.schedule }} 57 |

    58 |
    59 |
    60 | 66 | {% else %} 67 | {{ (etl.tasks_completed / etl.tasks_total * 100)|int }}% ({{ etl.tasks_completed }} of {{ etl.tasks_total }}) 68 | {% endif %} 69 | 70 | 71 | Details 72 | 73 |
    74 |
    75 | {% if etl.duration_seconds %}

    {{ etl.duration_seconds|describe_seconds }}

    {% endif %} 76 |
    77 |
    78 |

    79 | {% for run in history[etl.name] %} 80 | ◉ 82 | {% endfor %} 83 |

    84 |
    85 |
    86 | {% endfor %} 87 |
    88 | 89 | 90 | 94 | 95 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/blueprints/page/templates/etl_details.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} 2 | {% import 'macros/common.html' as common with context %} 3 | 4 | {% block title %}ETL {{ name }} details{% endblock %} 5 | 6 | {% block nav %} 7 |
  • Home
    • 8 |
  • ETL dashboard
    • 9 |
  • {{ name }}
    • 10 | 11 | {% endblock %} 12 | 13 | {% block body %} 14 | 15 |

    Tables managed by {{ name }}

    16 | 17 |
    18 |
    19 |
    20 | 21 | 27 | 28 | {#

    Tables in progress

    #} 29 | {% endblock %} 30 | -------------------------------------------------------------------------------- /dashboard/blueprints/page/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} 2 | {% import 'macros/common.html' as common with context %} 3 | 4 | {% block title %}{{ config['COMPANY_NAME'] }} Data{% endblock %} 5 | 6 | {% block nav %} 7 |
  • Home
    • 8 | {% endblock %} 9 | 10 | {% block body %} 11 | 12 | 13 | 14 |
    15 |
    16 | 17 | {% set etl_failure = etl_summary | selectattr('state', 'eq', 'failed') | list | count > 0 %} 18 | 19 |
    20 | {% if etl_failure %} 21 |
    ETL status: FAILURE
    22 | {% else %} 23 |
    ETL status: OK
    24 | {% endif %} 25 | 26 |
    27 | {% if etl_failure %} 28 |
    List of failed ETLs
    29 |
      30 | {% for etl in etl_summary | selectattr('state', 'eq', 'failed') %} 31 |
    • {{ etl.name }}
      • 32 | {% endfor %} 33 |
    34 | {% endif %} 35 | 36 |
    Succeeded: 37 |

    {{ etl_summary | selectattr('state', 'eq', 'success') | list | count }}

    38 |
    39 |
    Running: 40 |

    {{ etl_summary | selectattr('state', 'eq', 'running') | list | count }}

    41 |
    42 | 43 |
    44 |
    45 | 46 | {% if tables %} 47 |
    48 | {% set tables_failure = tables | selectattr('state', 'eq', 'failed') | list | count > 0 %} 49 | 50 | 51 |
    52 |
    53 |
    54 | Tables daily load status 55 |
    56 |
    57 | {% if tables_failure %} 58 |

    There are tables that failed to load toady, check tables dashboard

    59 | {% endif %} 60 | {% set today_tables = tables| 61 | selectattr('state', 'eq', 'success')| 62 | selectattr('last_update', 'ne', none)| 63 | selectattr('last_update', 'gt', now.replace(hour=0, minute=0, second=0))| 64 | list| 65 | count %} 66 |
    Daily load progress: 67 |

    68 | {{ (today_tables/tables|count * 100)|int }}% 69 |

    70 |
    71 |

    Total tables: {{ tables|count }}

    72 |

    Tables updated today: {{ today_tables }}

    73 | {% set stale_tables = tables|selectattr('last_update', 'ne', none)|selectattr('last_update', 'lt', now.replace(hour=0, minute=0, second=0))|list %} 74 | {% if stale_tables|count > 0 %} 75 |

    Tables awaiting completion: 76 |

      77 | {% for table in stale_tables|batch(5)|first %} 78 |
    • {{ table.id }}
      • 79 | {% endfor %} 80 | {% if stale_tables|count > 5 %}
    • ...
      • {% endif %} {# we're showing 5 tables max #} 81 |
    82 |

    83 | {% endif %} 84 |
    85 |
    86 |
    87 | 88 |
    89 | {% endif %} 90 |
    91 | 92 |
    93 |

    Important links

    94 | 99 |
    100 | 101 |
    102 | 103 | 104 | 105 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/blueprints/page/templates/login.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/clean.html' %} 2 | 3 | {% block title %}data.wikia-services.com - login page{% endblock %} 4 | 5 | {% block body %} 6 | 7 |
    8 |

    Login to your {{ config.OAUTH_DOMAIN if config.OAUTH_DOMAIN is string else "/".join(config.OAUTH_DOMAIN) }} account to access the site

    9 |

    10 | 11 |

    12 |
    13 | {% endblock %} 14 | -------------------------------------------------------------------------------- /dashboard/blueprints/page/templates/no_config.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} 2 | 3 | {% block title %}{{ config['COMPANY_NAME'] }} Data - internal server error{% endblock %} 4 | 5 | {% block nav %} 6 | 7 |
  • Home
    • 8 | {% if filename=="tables.yaml" %} 9 |
  • Tables
    • 10 | {% elif filename=="reports.yaml" %} 11 |
  • Reports
    • 12 | {% else %} 13 |
  • Extra ETLS
    • 14 | {% endif %} 15 | 16 | {% endblock %} 17 | 18 | {% block body %} 19 | 20 |
    21 |

    OOPS, this page is empty. But you can help me display 22 | some data here if you create a valid {{ filename }} file, see the docs.

    23 |
    24 | 25 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/blueprints/page/views.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from flask import Blueprint 4 | from flask import current_app as app 5 | from flask import redirect, render_template, request, session, url_for 6 | 7 | import dataclasses as dc 8 | from dashboard.utils import clean_dag_id 9 | from dashboard.utils.vis import tree_diagram 10 | 11 | page = Blueprint('page', __name__, template_folder='templates') 12 | 13 | @page.route('/') 14 | def index(): 15 | dags_status_future = app.async_request_executor.submit( 16 | app.airflow_data_provider.get_dags_status) 17 | 18 | template_vars = {'etl_summary': dags_status_future.result()} 19 | if app.table_data_provider: 20 | template_vars['tables'] = app.async_request_executor.submit(app.table_data_provider.list, True).result() 21 | 22 | if app.links_data_provider: 23 | template_vars['links'] = app.async_request_executor.submit(app.links_data_provider.get_links).result() 24 | 25 | return render_template('index.html', **template_vars) 26 | 27 | 28 | @page.route('/login') 29 | def login(): 30 | if 'GOOGLE_CLIENT_ID' in app.config: 31 | return render_template('login.html') 32 | else: 33 | return redirect(url_for('page.index')) 34 | 35 | 36 | @page.route('/logout') 37 | def logout(): 38 | del session['user'] 39 | return redirect(url_for('page.index')) 40 | 41 | 42 | @page.route('/etl') 43 | def etl_dashboard(): 44 | etl_with_progress_future = app.async_request_executor.submit( 45 | app.etl_data_provider.etl_with_progress) 46 | history_future = app.async_request_executor.submit( 47 | app.airflow_data_provider.get_history, 14) 48 | 49 | return render_template('etl_dashboard.html', 50 | etls=etl_with_progress_future.result(), 51 | history=history_future.result()) 52 | 53 | 54 | @page.route('/etl//') 55 | def etl_details(dag_id, execution_date): 56 | name_without_version = clean_dag_id(dag_id) 57 | data = app.table_data_provider.get_tables_graph(dag_id, execution_date) 58 | return render_template('etl_details.html', name=name_without_version, data=data, raw=json.dumps( 59 | tree_diagram([{ 60 | 'id': v.id, 61 | 'name': v.name, 62 | 'success': v.get_graph_color(), 63 | 'parent': v.parent} for v in data], width=800, height=800, padding=8) 64 | )) 65 | 66 | 67 | @page.app_template_filter() 68 | def describe_seconds(seconds): 69 | m, s = divmod(int(seconds), 60) 70 | h, m = divmod(m, 60) 71 | result = "{}s".format(s) 72 | if m > 0: 73 | result = "{}m {}".format(m, result) 74 | if h > 0: 75 | result = "{}h {}".format(h, result) 76 | return result 77 | 78 | 79 | @page.route('/healthcheck') 80 | def healthcheck(): 81 | return "https://www.youtube.com/watch?v=I_izvAbhExY" 82 | 83 | 84 | @page.errorhandler(500) 85 | def internal_server_error(e): 86 | return render_template("500.html"), 200 87 | -------------------------------------------------------------------------------- /dashboard/dataproviders/__init__.py: -------------------------------------------------------------------------------- 1 | from .airflow import AirflowDBDataProvider 2 | from .etl import EtlDataProvider -------------------------------------------------------------------------------- /dashboard/dataproviders/airflow.py: -------------------------------------------------------------------------------- 1 | from dashboard.utils import clean_dag_id 2 | 3 | from dashboard.models import DagRun, TaskInstance 4 | 5 | 6 | class AirflowDBDataProvider: 7 | 8 | def __init__(self, config, logger, client): 9 | self.logger = logger 10 | self.config = config 11 | self.client = client 12 | 13 | def get_dag_state(self, dag_id, execution_date): 14 | result = self.client.query( 15 | f"select state from dag_run where dag_id = '{dag_id}' and execution_date = '{execution_date}'") 16 | return result[0]['state'] 17 | 18 | def get_history(self, days): 19 | SQL = f''' 20 | SELECT dag_id, date, state FROM ( 21 | SELECT 22 | @row_number:=CASE 23 | WHEN @dag_id_current = clean_dag_id 24 | THEN 25 | @row_number + 1 26 | ELSE 27 | 1 28 | END AS rownum, 29 | @dag_id_current:=clean_dag_id clean_dag_id, 30 | dag_id, 31 | execution_date as date, 32 | state 33 | FROM ( 34 | SELECT *, 35 | CASE 36 | WHEN substring_index(reverse(dag_id), "v_", 1) rlike '[0-9]+\.[0-9]' 37 | THEN substring(dag_id, 1, length(dag_id) -1 - locate("v_", reverse(dag_id))) 38 | ELSE dag_id 39 | END as clean_dag_id from dag_run) dag_run, (SELECT @dag_id_current:='',@row_number:=0) as t 40 | ORDER BY clean_dag_id, execution_date DESC 41 | ) dr WHERE rownum <= {days}''' 42 | data = self.client.query(SQL) 43 | 44 | dag_names = set(map(lambda row: clean_dag_id(row['dag_id']), data)) 45 | 46 | return {dag_name: reversed([DagRun(**row) for row in data if clean_dag_id(row['dag_id']) == dag_name]) 47 | for dag_name in dag_names} 48 | 49 | def get_last_successful_tasks(self): 50 | last_successful_task_end_date = ''' 51 | SELECT dag_id, task_id, max(end_date) as end_date 52 | FROM task_instance 53 | WHERE state = "success" AND end_date is not null 54 | GROUP BY dag_id, task_id 55 | ''' 56 | 57 | data = self.client.query(last_successful_task_end_date) 58 | result = {} 59 | 60 | for row in data: 61 | row['dag_name'] = clean_dag_id(row['dag_id']) 62 | key = row['dag_name'] + row['task_id'] 63 | if key in result and result[key].end_date > row['end_date']: 64 | continue # duplicate with dag old version 65 | result[key] = TaskInstance(**row) 66 | 67 | return list(result.values()) 68 | 69 | def get_newest_task_instances(self): 70 | newest_task_instances_sql = ''' 71 | SELECT 72 | dr.dag_id, 73 | dr.execution_date, 74 | dr_latest.state as dag_state, 75 | ti.task_id, 76 | ti.state as task_state, 77 | ti.duration, 78 | ti.start_date, 79 | ti.end_date 80 | FROM ( 81 | SELECT dag_id, 82 | MAX(execution_date) as execution_date 83 | FROM dag_run 84 | GROUP BY dag_id) dr 85 | JOIN dag_run dr_latest ON dr.dag_id = dr_latest.dag_id AND dr.execution_date = dr_latest.execution_date 86 | JOIN task_instance ti ON dr.dag_id = ti.dag_id AND dr.execution_date = ti.execution_date 87 | JOIN dag ON dag.dag_id = dr.dag_id AND is_active = 1 AND is_paused = 0'''.replace("\n", "") 88 | 89 | data = self.client.query(newest_task_instances_sql) 90 | result = {} 91 | 92 | for row in data: 93 | row['dag_name'] = clean_dag_id(row['dag_id']) 94 | key = row['dag_name'] + row['task_id'] 95 | if key in result and row['end_date'] and result[key].end_date > row['end_date']: 96 | continue # duplicate with dag old version 97 | 98 | if row['dag_name'] in self.config.get('TECHNICAL_ETLS', set()): 99 | continue # task instance from the technical ETL 100 | 101 | result[key] = TaskInstance(**row) 102 | 103 | return list(result.values()) 104 | 105 | def get_dag_tasks(self, dag_id, execution_date): 106 | data = self.client.query( 107 | f"""SELECT dag_id, execution_date, task_id, start_date, end_date, duration, state as task_state 108 | FROM task_instance WHERE dag_id='{dag_id}' AND execution_date='{execution_date}'""".replace("\n", "")) 109 | return [TaskInstance(**row) for row in data] 110 | 111 | 112 | def get_dags_status(self): 113 | ''' 114 | For each non-technical DAG returns the name and the state of last run 115 | :return: 116 | ''' 117 | 118 | latest_dags_status_sql = ''' 119 | SELECT 120 | dag_run.dag_id, 121 | dag_run.state 122 | FROM dag_run 123 | INNER JOIN (SELECT 124 | dag_id, 125 | MAX(execution_date) AS date 126 | FROM dag_run 127 | GROUP BY dag_id) mx 128 | ON 129 | dag_run.dag_id = mx.dag_id 130 | AND dag_run.execution_date = mx.date 131 | JOIN dag ON dag.dag_id = dag_run.dag_id AND is_active = 1 AND is_paused = 0'''.replace("\n", "") 132 | return [{ 133 | 'name': clean_dag_id(dag['dag_id']), 134 | 'state': dag['state']} 135 | for dag in self.client.query(latest_dags_status_sql) 136 | if clean_dag_id(dag['dag_id']) not in self.config.get('TECHNICAL_ETLS', set())] 137 | -------------------------------------------------------------------------------- /dashboard/dataproviders/etl.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from dashboard.models import EtlWithProgress 3 | 4 | 5 | class EtlDataProvider: 6 | 7 | def __init__(self, config, airflow_data_provider, tables_data_provider): 8 | self.config = config 9 | self.airflow_data_provider = airflow_data_provider 10 | self.tables_data_provider = tables_data_provider 11 | 12 | def _get_etl_with_progress(self, dag_id, tasks): 13 | tasks = [task for task in tasks if task.dag_name == dag_id] 14 | completed_tasks = set([task.task_id for task in tasks if task.is_done()]) 15 | tables = [table for table in self.tables_data_provider.get_tables() if table.dag_id == dag_id] if self.tables_data_provider else None 16 | 17 | if tasks[0].dag_state == 'running': 18 | try: 19 | duration = datetime.datetime.now() \ 20 | - min(filter(None, map(lambda t: t.start_date, tasks))) 21 | except ValueError as _: 22 | duration = datetime.timedelta(0) 23 | else: 24 | try: 25 | duration = max(filter(None, map(lambda t: t.end_date, tasks))) \ 26 | - min(filter(None, map(lambda t: t.start_date, tasks))) 27 | except ValueError as _: 28 | duration = datetime.timedelta(0) 29 | 30 | return EtlWithProgress( 31 | name= dag_id, 32 | schedule=tasks[0].execution_date, 33 | state=tasks[0].dag_state, 34 | tasks_completed=len(completed_tasks), 35 | tasks_total=len(tasks), 36 | tables_completed=sum([1 for table in tables if table.task_id in completed_tasks]) if tables else None, 37 | tables_total=len(tables) if tables else None, 38 | duration_seconds=duration.total_seconds(), 39 | dag_id=tasks[0].dag_id, 40 | last_execution_date=tasks[0].execution_date 41 | ) 42 | 43 | def etl_with_progress(self): 44 | tasks_newest_instance = self.airflow_data_provider.get_newest_task_instances() 45 | dags = set([task.dag_name for task in tasks_newest_instance]) - set(self.config.get('TECHNICAL_ETLS', set())) 46 | 47 | return [self._get_etl_with_progress(dag, tasks_newest_instance) for dag in dags] 48 | -------------------------------------------------------------------------------- /dashboard/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/model/__init__.py -------------------------------------------------------------------------------- /dashboard/model/influxdb_data.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from dashboard.service.influxdb_service import InfluxDbService 3 | 4 | class InfluxDBData: 5 | def __init__(self, influx, logger): 6 | self.influx = influx 7 | self.logger = logger 8 | 9 | def get_influx_stats(self, table, days): 10 | query = self.__get_table_sql(table, days) 11 | return list(self.influx.query(query).get_points()) 12 | 13 | def __get_table_sql(self, table, days): 14 | period_filter = f" AND period_id = '{table.period.id}' " if table.period else "" 15 | query = f'SELECT * FROM "emr_stats_{table.db}_{table.name}" ' \ 16 | f'WHERE time >= now() - {days}d {period_filter}' \ 17 | f'ORDER BY time ASC' 18 | return query 19 | 20 | def get_influx_data(self, days, period_mapping): 21 | influx_stats = {} 22 | query = 'SELECT * FROM /emr_stats_.*/ ' \ 23 | 'WHERE time >= now() - {days}d'.format(days=days) 24 | result_set = self.influx.query(query) 25 | measurements = result_set.raw['series'] 26 | mapper = lambda row: (datetime.datetime.strptime(row['time'], '%Y-%m-%dT%H:%M:%SZ'), row['records']) 27 | 28 | for measurement_name in [x['name'] for x in measurements]: 29 | measurement_data = list(result_set.get_points(measurement=measurement_name)) 30 | base_table_id = measurement_name[10:] 31 | if measurement_data: 32 | if measurement_data[0].get('period_id') is None: 33 | influx_stats[base_table_id] = [mapper(row) for row in measurement_data] 34 | else: 35 | for row in measurement_data: 36 | period_id = row['period_id'] 37 | table_id = f'{base_table_id}_{period_mapping[int(period_id)]}' 38 | influx_stats.setdefault(table_id, []).append(mapper(row)) 39 | return influx_stats 40 | -------------------------------------------------------------------------------- /dashboard/model/links_data.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class LinksDataProvider: 4 | def __init__(self, links): 5 | self.links = links if links else [] 6 | 7 | def get_links(self): 8 | return self.links 9 | -------------------------------------------------------------------------------- /dashboard/model/prometheus_data.py: -------------------------------------------------------------------------------- 1 | from dashboard.service.prometheus_service import PrometheusService 2 | 3 | ALERT_PREFIX = 'emr_stats_' 4 | 5 | 6 | class PrometheusData: 7 | def __init__(self, config, logger): 8 | self.prometheus = PrometheusService(config, logger) 9 | 10 | def alerts(self): 11 | response = self.prometheus.query(f'ALERTS{{alertname=~"{ALERT_PREFIX}.*"}}') 12 | alerts = [result['metric']['alertname'] for result in response['data']['result']] 13 | return [{'db_table': alert.rsplit('__', 1)[0][len(ALERT_PREFIX):], 'name': alert} for alert in alerts] 14 | 15 | def alert_link(self, alert_name): 16 | return self.prometheus.alert_link(alert_name) 17 | -------------------------------------------------------------------------------- /dashboard/model/tables_data.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from concurrent.futures import ThreadPoolExecutor 3 | from typing import Dict 4 | 5 | from dashboard.models import Period, Table 6 | from dashboard.utils import clean_dag_id, handle_resource, simple_state 7 | from dataclasses import dataclass 8 | 9 | 10 | @dataclass 11 | class TableSize: 12 | name: str 13 | size: int 14 | 15 | 16 | @dataclass 17 | class GraphVertex: 18 | id: str 19 | name: str 20 | state: str = 'unknown' 21 | tooltip: str = '' 22 | parent: str = None 23 | 24 | def get_graph_color(self): 25 | map = { 26 | 'success': 1, 27 | 'running': 2, 28 | 'failed': 3 29 | } 30 | if self.state in map: 31 | return map[self.state] 32 | return 0 # unknown 33 | 34 | @dataclass 35 | class DagTableProgress: 36 | dag_name: str 37 | tables_done: int 38 | tables_all: int 39 | 40 | 41 | HISTOGRAM_DAYS_NUM = 7 42 | DETAILED_CHART_DAYS_NUM = 30 43 | 44 | 45 | class TableDataProvider: 46 | 47 | @staticmethod 48 | def _format_table_id(table): 49 | id = table['db'] + '.' + table['name'] 50 | if 'period' in table: 51 | id += '.' + table['period']['name'] 52 | return id 53 | 54 | @staticmethod 55 | def _process_input_tables(tables): 56 | output_tables = {} 57 | for table in tables: 58 | table['id'] = TableDataProvider._format_table_id(table) 59 | if 'period' in table: 60 | table['period'] = Period(**table['period']) 61 | output_tables[table['id']] = Table(**table) 62 | return output_tables 63 | 64 | def __init__(self, airflow, influx, prometheus, tables, logger, config): 65 | self.airflow = airflow 66 | self.influx = influx 67 | self.prometheus = prometheus 68 | self.tables = TableDataProvider._process_input_tables(tables) 69 | self.logger = logger 70 | self.config = config 71 | 72 | def get_tables(self): 73 | return list(self.tables.values()) 74 | 75 | def history(self, table): 76 | return self.__get_detailed_view_data(self.tables[table], DETAILED_CHART_DAYS_NUM) 77 | 78 | def get_tables_by_dag(self, dag_name) -> Dict[str, Table]: 79 | return {id: table for id, table in self.tables.items() if table.dag_id == dag_name} 80 | 81 | def get_tables_graph(self, dag_id, execution_date): 82 | name_without_version = clean_dag_id(dag_id) 83 | dag_tables = self.get_tables_by_dag(name_without_version) 84 | dag_progress = {d.task_id: d for d in self.airflow.get_dag_tasks(dag_id, execution_date)} 85 | 86 | 87 | # root node - dag name 88 | yield GraphVertex(id='main', name=name_without_version, 89 | state=self.airflow.get_dag_state(dag_id, execution_date)) 90 | 91 | # tables 92 | for table in dag_tables.values(): 93 | yield GraphVertex( 94 | id=table.id, 95 | name=table.name + (' ({})'.format(table.period.name) if table.period else ''), 96 | state=dag_progress[table.task_id].task_state, 97 | tooltip='Finished at: {}, duration: {}'.format( 98 | dag_progress[table.task_id].end_date, 99 | dag_progress[table.task_id].duration 100 | ), 101 | # workaround for this entire method not being able to reference table managed by other DAG in table.uses 102 | # see https://github.com/Wikia/discreETLy/issues/22 103 | parent='main' if table.uses is None or table.uses not in dag_tables.keys() else table.uses 104 | ) 105 | 106 | @handle_resource('influx') 107 | def __get_detailed_view_data(self, table, days): 108 | influx_points = self.influx.get_influx_stats(table, days) 109 | return [(x['time'][:days], x['records']) for x in influx_points if x['records'] is not None] 110 | 111 | def __get_period_mapping(self): 112 | return {table.period.id: table.period.name for table in self.get_tables() 113 | if table.is_rollup()} 114 | 115 | @handle_resource('prometheus', {}) 116 | def get_alerts(self): 117 | return self.prometheus.alerts() 118 | 119 | @handle_resource('prometheus') 120 | def active_alerts_for_table(self, db, table, alerts): 121 | return [{'name': alert['name'], 'link': self.prometheus.alert_link(alert['name'])} 122 | for alert in alerts if alert['db_table'] == f'{db}_{table}'] 123 | 124 | def list_unordered(self, gather_counts): 125 | # Get data for all rows in one query to avoid multiple db lookups 126 | newest_task_instances = self.airflow.get_newest_task_instances() 127 | last_updates = self.airflow.get_last_successful_tasks() 128 | 129 | if gather_counts: 130 | emr_stats = self.influx.get_influx_data(days=HISTOGRAM_DAYS_NUM, period_mapping=self.__get_period_mapping()) if self.influx else {} 131 | alerts = self.get_alerts() 132 | else: 133 | emr_stats = {} 134 | alerts = {} 135 | 136 | for table in self.tables.values(): 137 | table_id_underscore = table.id.replace('.', '_') 138 | if not table.streaming: 139 | # lookup the current row 140 | last_task_data = next((x for x in newest_task_instances if x.dag_name == table.dag_id 141 | and x.task_id == table.task_id), None) 142 | if last_task_data is None: 143 | self.logger.warn(f'Missing task_instances entry for table {table}') 144 | continue 145 | 146 | last_update = next((x.end_date for x in last_updates if x.dag_name == table.dag_id 147 | and x.task_id == table.task_id), None) 148 | 149 | state = simple_state(last_task_data.task_state) 150 | duration = last_task_data.duration 151 | else: 152 | duration = None 153 | # Todo edge cases in names 154 | table_stats = emr_stats.get(table_id_underscore) 155 | if table_stats: 156 | last_update = table_stats[-1][0] 157 | state = 'success' 158 | else: 159 | last_update = None 160 | state = 'success' 161 | 162 | row_counts = None 163 | if gather_counts and emr_stats and table_id_underscore in emr_stats: 164 | row_counts = sorted( 165 | emr_stats[table_id_underscore][-HISTOGRAM_DAYS_NUM:], reverse=True, key=lambda tup: [0]) 166 | 167 | yield {'id': table.id, 168 | 'name': table.name, 169 | 'dag_id': table.dag_id, 170 | 'task_id': table.task_id, 171 | 'glue': table.glue, 172 | 'last_update': last_update, 173 | 'counts': row_counts, 174 | 'load_duration_seconds': duration, 175 | 'active_alerts': self.active_alerts_for_table(table.db, table.name, alerts), 176 | 'state': state} 177 | 178 | def list(self, gather_counts=True): 179 | return sorted(self.list_unordered(gather_counts), key=lambda t: (t['state'], not t['active_alerts'], t['id'])) 180 | 181 | def get_tables_with_size(self): 182 | with open('/app/config/sizes_static.csv') as csvfile: 183 | for row in csv.reader(csvfile): 184 | yield TableSize(row[0], int(row[2])) 185 | -------------------------------------------------------------------------------- /dashboard/models.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from dataclasses import dataclass 3 | from datetime import datetime 4 | from typing import Optional 5 | 6 | @dataclasses.dataclass 7 | class TaskInstance: 8 | dag_id: str 9 | task_id: str 10 | end_date: datetime 11 | execution_date: datetime = None 12 | start_date: datetime = None 13 | task_state: str = None 14 | dag_name: Optional[str] = None 15 | dag_state: Optional[str] = None 16 | duration: Optional[float] = None 17 | 18 | def is_done(self): 19 | return self.task_state == "success" 20 | 21 | 22 | @dataclasses.dataclass 23 | class DagRun: 24 | dag_id: str 25 | date: str 26 | state: str 27 | 28 | 29 | @dataclass 30 | class EtlWithProgress: 31 | name: str 32 | link_title: Optional[str] = None # only for streams 33 | link_href: Optional[str] = None # only for streams 34 | schedule: Optional[str] = None # only for DAGs 35 | state: Optional[str] = None # only for DAGs 36 | tasks_completed: Optional[int] = None # only for DAGs 37 | tasks_total: Optional[int] = None # only for DAGs 38 | tables_completed: Optional[int] = None # only for DAGs 39 | tables_total: Optional[int] = None # only for DAGs 40 | duration_seconds: Optional[int] = None # only for DAGs 41 | dag_id: Optional[str] = None # only for DAGs 42 | last_execution_date: Optional[datetime] = None # only for DAGs 43 | 44 | @dataclass 45 | class Period: 46 | id: int 47 | name: str 48 | 49 | @dataclass 50 | class Table: 51 | id: str 52 | db: str 53 | name: str 54 | glue: bool = False 55 | streaming: bool = False 56 | uses: Optional[str] = None 57 | period: Optional[Period] = None 58 | dag_id: Optional[str] = None 59 | task_id: Optional[str] = None 60 | 61 | def get_parent(self): 62 | if self.uses is None: 63 | return 'main' 64 | return self.uses 65 | 66 | def is_rollup(self): 67 | return self.period is not None 68 | 69 | @dataclass 70 | class ExtraEtl: 71 | name: str 72 | link_title: str 73 | link_href: str 74 | table: str 75 | active_alerts: Optional[list] = None 76 | -------------------------------------------------------------------------------- /dashboard/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/__init__.py -------------------------------------------------------------------------------- /dashboard/plugins/athena_usage/README.md: -------------------------------------------------------------------------------- 1 | # Athena Usage plugin 2 | 3 | Athena Usage shows summarised daily, weekly and monthly data usage in Athena queries for an `executing_user`. 4 | This plugin aims to help users track their queries' efficiency. 5 | 6 | Relies on accessing `QUERIES_TABLE`, storing queries' info. 7 | 8 | This plugin configuration is stored as `ATHENA_USAGE_PARAMS`: 9 | ``` 10 | ATHENA_USAGE_PARAMS = { 11 | 'QUERIES_TABLE': '....', 12 | 'region_name': '....', 13 | 'aws_access_key_id': '....', 14 | 'aws_secret_access_key': '....'} 15 | ``` 16 | * `QUERIES_TABLE` - name of the table where queries' metadata are stored 17 | * `region_name` - AWS region where dynamodb is stored 18 | * `aws_access_key_id` and `aws_secret_access_key` - credentials to be used to communicate 19 | with AWS API 20 | 21 | ![athena usage](athena_usage.png) 22 | -------------------------------------------------------------------------------- /dashboard/plugins/athena_usage/__init__.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | 3 | from flask import current_app as app 4 | from flask import Blueprint 5 | from flask import render_template 6 | 7 | 8 | from .query_dao import QueryDao 9 | from .athena_summary_provider import AthenaSummaryProvider 10 | 11 | base_path = '/athena_usage' 12 | tab_name = 'Athena Usage' 13 | plugin = Blueprint('athena_usage', __name__, template_folder='templates') 14 | 15 | 16 | def init(app): 17 | dynamodb = boto3.client('dynamodb', 18 | aws_access_key_id=app.config['ATHENA_USAGE_PARAMS']['aws_access_key_id'], 19 | aws_secret_access_key=app.config['ATHENA_USAGE_PARAMS']['aws_secret_access_key'], 20 | region_name=app.config['ATHENA_USAGE_PARAMS']['region_name']) 21 | 22 | app.athena_summary_provider = AthenaSummaryProvider(app.config['ATHENA_USAGE_PARAMS'], dynamodb, app.logger) 23 | 24 | 25 | @plugin.route('/') 26 | def index(): 27 | summary_users_dict, summary_specials_dict = app.athena_summary_provider.summary_user_timespan_size 28 | 29 | # sort both summaries alphabetically 30 | summary_users_dict, summary_specials_dict = sorted(summary_users_dict.items()), (sorted(summary_specials_dict.items())) 31 | 32 | return render_template('athena_usage/index.html', 33 | summary_specials_timespan_size=summary_specials_dict, 34 | summary_user_timespan_size=summary_users_dict) 35 | 36 | 37 | -------------------------------------------------------------------------------- /dashboard/plugins/athena_usage/athena_query_model.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | 4 | TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S' 5 | 6 | DATE_FORMAT = '%Y-%m-%d' 7 | 8 | class QueryState(Enum): 9 | QUEUED = 'QUEUED' 10 | RUNNING = 'RUNNING' 11 | SUCCEEDED = 'SUCCEEDED' 12 | FAILED = 'FAILED' 13 | CANCELLED = 'CANCELLED' 14 | 15 | 16 | @dataclass(eq=True, frozen=True) 17 | class AthenaQuery: 18 | start_date: str = None 19 | start_timestamp: str = None 20 | query_execution_id: str = None 21 | query_state: str = None 22 | executing_user: str = None 23 | data_scanned: int = 0 24 | query_sql: str = None 25 | 26 | 27 | class Timespan(Enum): 28 | """ Timespan values in seconds: int """ 29 | MONTH: int = 30 * 24 * 60 * 60 30 | WEEK: int = 7 * 24 * 60 * 60 31 | DAY: int = 24 * 60 * 60 32 | -------------------------------------------------------------------------------- /dashboard/plugins/athena_usage/athena_summary_provider.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from typing import DefaultDict, Tuple 3 | 4 | from dashboard.utils import sizeof_fmt 5 | from .query_dao import * 6 | 7 | 8 | class AthenaSummaryProvider: 9 | """ 10 | This class communicates with QueryDao in order to prepare summaries of athena usage 11 | """ 12 | 13 | ALL_USERS_KEY = 'All users' 14 | 15 | def __init__(self, config: dict, dynamodb, logger): 16 | self.logger = logger 17 | self._query_dao = QueryDao(config, dynamodb, self.logger) 18 | 19 | @property 20 | def _summary_user_timespan_size_B(self) -> Tuple[DefaultDict[str, DefaultDict[str, int]], DefaultDict[str, DefaultDict[str, int]]]: 21 | """ 22 | This gets a username dict containing a sum of size of user's queries in 23 | three timespans: last month, week, and a day, 24 | and the same-structured dict containing special summaries, 25 | e.g. sums for all users under ALL_USERS_KEY 26 | 27 | Sizes are given in Bytes or default db's 'data_scanned' unit 28 | :return: Tuple[Dict[username] -> Dict['month' | 'week' | 'day'] -> sum(sizes in Bytes), 29 | Dict[special_summary] -> Dict['month' | 'week' | 'day'] -> sum(sizes in Bytes)] 30 | """ 31 | # looking only 30 days back since it's the most we are going to present 32 | all_queries_set = self._query_dao.queries_last_30_days 33 | 34 | # prepare a dict of username: dict(daily, weekly, monthly usage) 35 | summary_user_dict = defaultdict(lambda: defaultdict(lambda: 0)) 36 | 37 | # prepare a dict of special_summary: dict(daily, weekly, monthly usage) 38 | # e.g. special_summaries use: sum for all users, grouping by role, by team.. 39 | special_summaries_dict = defaultdict(lambda: defaultdict(lambda: 0)) 40 | 41 | for query in all_queries_set: 42 | # convert queries str start_timestamp to float for easier comparisons 43 | query_timestamp_from_date = time.mktime(datetime.strptime(query.start_timestamp, TIMESTAMP_FORMAT).timetuple()) 44 | 45 | normalized_name = self._normalize_username(query.executing_user) 46 | 47 | # check if a query is less than a month old 48 | if time.time() - query_timestamp_from_date <= Timespan.MONTH.value: 49 | summary_user_dict[normalized_name]['month'] += query.data_scanned 50 | special_summaries_dict[self.ALL_USERS_KEY]['month'] += query.data_scanned 51 | 52 | # check if a query is less than a week old. 53 | if time.time() - query_timestamp_from_date <= Timespan.WEEK.value: 54 | summary_user_dict[normalized_name]['week'] += query.data_scanned 55 | special_summaries_dict[self.ALL_USERS_KEY]['week'] += query.data_scanned 56 | 57 | # check if a query is less than a day old 58 | if time.time() - query_timestamp_from_date <= Timespan.DAY.value: 59 | summary_user_dict[normalized_name]['day'] += query.data_scanned 60 | special_summaries_dict[self.ALL_USERS_KEY]['day'] += query.data_scanned 61 | 62 | return summary_user_dict, special_summaries_dict 63 | 64 | @property 65 | def summary_user_timespan_size(self) -> Tuple[DefaultDict[str, DefaultDict[str, str]], DefaultDict[str, DefaultDict[str, str]]]: 66 | """ 67 | This gets a username dict containing a sum of size of user's queries in 68 | three timespans: last month, week, and a day, 69 | and the same-structured dict containing special summaries, 70 | e.g. sums for all users under ALL_USERS_KEY 71 | 72 | Sizes are given as a 'size:int unit:str':str in the appropriate unit 73 | :return: Tuple[Dict[username] -> Dict['month' | 'week' | 'day'] -> sum(str(size unit)), 74 | Dict[special_summary] -> Dict['month' | 'week' | 'day'] -> sum(str(size unit))] 75 | """ 76 | 77 | return tuple(map(self._convert_summary_dict_bytes_to_smart_unit, self._summary_user_timespan_size_B)) 78 | 79 | def _convert_summary_dict_bytes_to_smart_unit(self, summary_dict: DefaultDict[str, DefaultDict[str, int]]) -> DefaultDict[str, DefaultDict[str, str]]: 80 | """ 81 | This method converts 82 | Dict[name] -> Dict['month' | 'week' | 'day'] -> sum(sizes in Bytes) 83 | to use smart size unit 84 | :return: Dict[username] -> Dict['month' | 'week' | 'day'] -> sum(str(size unit)) 85 | """ 86 | for name in summary_dict.keys(): 87 | for timespan in summary_dict[name].keys(): 88 | summary_dict[name][timespan] = sizeof_fmt(int(summary_dict[name][timespan])) 89 | return summary_dict 90 | 91 | def _normalize_username(self, raw_name: str) -> str: 92 | """ Normalize usernames to count both @name and name as the same """ 93 | return raw_name[1:] if raw_name[0] == '@' else raw_name 94 | -------------------------------------------------------------------------------- /dashboard/plugins/athena_usage/athena_usage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/athena_usage/athena_usage.png -------------------------------------------------------------------------------- /dashboard/plugins/athena_usage/query_dao.py: -------------------------------------------------------------------------------- 1 | from calendar import timegm 2 | from datetime import datetime, timedelta 3 | import time 4 | from typing import Set 5 | import functools 6 | from math import ceil 7 | 8 | from boto3.dynamodb.types import TypeDeserializer 9 | 10 | from .athena_query_model import AthenaQuery, DATE_FORMAT, Timespan, \ 11 | TIMESTAMP_FORMAT 12 | 13 | 14 | class QueryDao: 15 | """ This class is responsible for querying dynamoDB """ 16 | 17 | def __init__(self, config, dynamodb, logger): 18 | self._config = config 19 | self._dynamodb = dynamodb 20 | self.logger = logger 21 | self._query_cache: Set[AthenaQuery] = None 22 | self._last_cached_time = None 23 | 24 | @property 25 | def queries_last_30_days(self) -> Set[AthenaQuery]: 26 | """ 27 | This returns last 30 days of queries made, without making additional 28 | requests if not needed. 29 | 30 | :return: set of AthenaQueries containing the request made in the last 30 days 31 | """ 32 | # TODO: this caching is very much inefficient due to not being shared between workers 33 | # TODO: sqllite or other db solution is required 34 | # if no previous queries are cached 35 | if self._query_cache is None: 36 | self._query_cache = self._get_finished_queries_for_days_back(30) 37 | self._last_cached_time = time.time() 38 | else: 39 | # just need to get missing queries from _last_cached_time to now 40 | _time_passed_since_cached = self._last_cached_time - time.time() 41 | # convert time passed to days: int with rounding up to not miss a day to query 42 | days_passed_since_cached = ceil(_time_passed_since_cached / Timespan.DAY.value) 43 | 44 | new_queries = self._get_finished_queries_for_days_back(days_passed_since_cached) 45 | 46 | self._query_cache.update(new_queries) 47 | self._last_cached_time = time.time() 48 | 49 | # remove unnecessary, old queries from cache 50 | self._remove_unnecessary_cached_queries() 51 | 52 | return self._query_cache 53 | 54 | def _remove_unnecessary_cached_queries(self): 55 | """ 56 | Removes cached queries older than a month from memory 57 | in order to prevent cache swelling up in size 58 | """ 59 | self._query_cache = set(filter(lambda x: time.time() - timegm(time.strptime(x.start_timestamp, TIMESTAMP_FORMAT)) < Timespan.MONTH.value, 60 | self._query_cache)) 61 | 62 | def _get_finished_queries_for_days_back(self, timespan_days: int) -> Set[AthenaQuery]: 63 | """ 64 | This gets queries made from days: int back to today 65 | :parameter timespan_days: int number of days to go back 66 | :return: Set of AthenaQueries containing the request made in that timespan 67 | """ 68 | 69 | # prepare a list of str dates of days going from now to number timespan_days: int back 70 | days_back_dates_str = [datetime.strftime(datetime.now() - timedelta(days=days_back), DATE_FORMAT) for days_back in range(timespan_days + 1)] 71 | 72 | self.logger.debug(f"athena_usage starting querying {self._config['QUERIES_TABLE']} table " 73 | f"for {timespan_days+1} days") 74 | self.logger.debug(f"athena_usage: querying for list of days: {days_back_dates_str}") 75 | 76 | # TODO: this map function call can be threaded for speed improvements 77 | results: Set[AthenaQuery] = set().union(*map(self._query_specific_date, days_back_dates_str)) 78 | 79 | self.logger.debug(f"athena_usage finished querying {self._config['QUERIES_TABLE']} table " 80 | f"for {timespan_days+1} days") 81 | return results 82 | 83 | def _query_specific_date(self, date) -> Set[AthenaQuery]: 84 | """ 85 | Private method to query for all queries 86 | made on a particular date with data_scanned > 0 87 | 88 | :param date: str date of DATE_FORMAT 89 | :return: Set of AthenaQuery, results of this operation 90 | """ 91 | # low level client type query is used instead of resource 92 | # due to higher level resource query not allowing to filter 93 | # data_scanned > 0 94 | 95 | # start_timestamp is additionally queried for 'true' 24h days calculation 96 | # instead of a 'day' being 24h +- 11.59h 97 | query_finished_queries_for_days_function_call = functools.partial( 98 | self._dynamodb.query, 99 | TableName=self._config['QUERIES_TABLE'], 100 | ProjectionExpression='data_scanned, executing_user, start_timestamp', 101 | KeyConditionExpression='start_date = :date', 102 | ExpressionAttributeValues={':date': {'S': f'{date}'}, 103 | ':num': {'N': '0'}}, 104 | FilterExpression='#ds > :num', 105 | ExpressionAttributeNames={'#ds': 'data_scanned'} 106 | ) 107 | 108 | response = query_finished_queries_for_days_function_call() 109 | data = response['Items'] 110 | 111 | # check for paginated results 112 | while response.get('LastEvaluatedKey'): 113 | response = query_finished_queries_for_days_function_call( 114 | ExclusiveStartKey=response['LastEvaluatedKey']) 115 | data.extend(response['Items']) 116 | 117 | # data that comes is a list in elements of format: 118 | # 'start_timestamp': {'S': '2019-01-23 18:47:04'}, .. 119 | # so it needs to be deserialized while taking into account _dynamodb's ('S'..) types 120 | 121 | python_data = [{field_name: TypeDeserializer().deserialize(type_value_dict) 122 | for field_name, type_value_dict in data_entry.items()} 123 | for data_entry in data] 124 | 125 | return set(AthenaQuery(start_date=date, **item) for item in python_data) 126 | 127 | -------------------------------------------------------------------------------- /dashboard/plugins/athena_usage/templates/athena_usage/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} {% import 'macros/common.html' as common with context %} 2 | 3 | {% block title %}Athena Usage Dashboard{% endblock %} 4 | 5 | {% block nav %} 6 | 7 |
  • Home
    • 8 |
  • Athena Usage
    • 9 | 10 | {% endblock %} 11 | 12 | 13 | {% block body %} 14 | 15 | 16 |

    Athena Usage Dashboard

    17 | 18 |
    19 |
    20 |
    Username
    21 |
    Daily
    22 |
    Weekly
    23 |
    Monthly
    24 |
    25 | {% for name, usages_timespan_dict in summary_specials_timespan_size + summary_user_timespan_size %} 26 |
    27 |
    28 | {{ name }} 29 |
    30 |
    31 | {{ usages_timespan_dict['day'] }} 32 |
    33 |
    34 | {{ usages_timespan_dict['week'] }} 35 |
    36 |
    37 | {{ usages_timespan_dict['month'] }} 38 |
    39 |
    40 | {% endfor %} 41 |
    42 | 43 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/plugins/hello_world/README.md: -------------------------------------------------------------------------------- 1 | # Hello World plugin 2 | 3 | This *dummy* plugin describes how to contribute to DiscreETLy by creating new plugins. 4 | Enable it in `settings.py` to find more! -------------------------------------------------------------------------------- /dashboard/plugins/hello_world/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint 2 | from flask import render_template 3 | 4 | base_path = '/hello_world' 5 | tab_name = 'Hello World!' 6 | plugin = Blueprint('hello_world', __name__, template_folder='templates') 7 | 8 | @plugin.route('/') 9 | def index(): 10 | return render_template('hello_world/index.html') -------------------------------------------------------------------------------- /dashboard/plugins/hello_world/templates/hello_world/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} {% import 'macros/common.html' as common with context %} 2 | 3 | {% block title %}Hello World{% endblock %} 4 | 5 | {% block nav %} 6 | 7 |
  • Home
    • 8 |
  • Hello world!
    • 9 | 10 | {% endblock %} 11 | {% block body %} 12 | 13 |

    Hello World! DiscreETLy Developers Documentation

    14 | 15 | Contribute to DiscreETLy and adjust the views and data providers to your needs using 16 | plugins system! All plugins reside in dashboard/plugins and creating 17 | a new one is very easy: 18 | 19 | 
    
20 | # 1. Copy hello_world plugin contents
21 | cp -r dashboard/plugins/hello_world dashboard/plugins/$PLUGIN_NAME
22 | 
23 | # 2. Fix the templates namespace
24 | mv  dashboard/plugins/$PLUGIN_NAME/templates/hello_world dashboard/plugins/$PLUGIN_NAME/templates/$PLUGIN_NAME 
25 | 
26 | # 3. Adjust the code and template
27 | vi dashboard/plugins/$PLUGIN_NAME/__init__.py
28 | vi dashboard/plugins/$PLUGIN_NAME/templates/$PLUGIN_NAME/index.html
29 | 
30 | # 4. Enable the plugin in settings.py
31 | vi config/settings.py
32 |
    33 | 34 |

    Plugin Manifest

    35 | 36 | Each plugin must provide 3 variables in the main scope as the manifest: 37 | 38 |
      39 |
    • base_path - the URL part that all plugin routes are relative to
      • 40 |
    • tab_name - user-friendly name of the plugin to put on navigation bar
      • 41 |
    • plugin - implementation of flask.Blueprint to be created
      • 42 |
    43 | 44 | Also, if the plugin needs to perform some work after start of the application (and not after 45 | first user's reques), define a function init inside that gets application object 46 | (with generic modules like app.config and app.logger) 47 | 48 |

    Core data providers and helpers

    49 | 50 | In your plugin you can use a number of services/helpers provided by the core. 51 | All these objects are members of the `app` object, so import the current application 52 | first: from flask import current_app as app and then you can use: 53 | 54 |
      55 |
    • app.airflow_data_provider - provides Airflow DAGs list, task statuses, ... (retrieved from the DB)
      • 56 |
    • app.influx_data_provider - queries InfluxDB for the records count for the tables
      • 57 |
    • app.prometheus_data_provider - lists currently fired alerts on Prometheus
      • 58 |
    • app.airflow_data_provider - provides tables list with status, alerts, records counts, ...
      • 59 |
    • app.etl_data_provider - formats app.airflow_data_provider with custom logic (like calculating DAG duration)
      • 60 |
    • app.async_request_executor - ThreadPoolExecutor to query data providers in parallel
      • 61 |
    • app.cache - simple in-memory cache
      • 62 |
    63 | 64 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/plugins/reports/README.md: -------------------------------------------------------------------------------- 1 | # Reports View 2 | 3 | The dashboard allows to monitor sets of tables that constitute to a report that is maintained by DE team or stakeholders. 4 | 5 | The definition of the set of tables and general report metadata should be places as 6 | `reports.yaml` file in `config` folder. 7 | 8 | Please, refer to [reports.yaml.template](reports.yaml.template) to learn more about particular options that 9 | need to be provided. Reports plugin requires valid [tables.yaml](../tables/tables.yaml.template) to be 10 | provided to match tables from `reports.yaml` with corresponding DAGs and tasks. 11 | 12 | ![reports list](reports_list.png) -------------------------------------------------------------------------------- /dashboard/plugins/reports/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint 2 | from flask import render_template 3 | from flask import current_app as app 4 | from dashboard.utils import get_yaml_file_content 5 | from .reports_data import ReportsDataProvider 6 | 7 | base_path = '/reports' 8 | tab_name = 'Reports' 9 | plugin = Blueprint('reports', __name__, template_folder='templates') 10 | 11 | REPORTS_PATH = 'config/reports.yaml' 12 | 13 | @plugin.route('/') 14 | def index(): 15 | reports = get_yaml_file_content(REPORTS_PATH) 16 | if not reports: 17 | return render_template('no_config.html', filename='reports.yaml') 18 | 19 | report_data_provider = ReportsDataProvider(app.table_data_provider.list(), reports['reports']) 20 | return render_template('reports/index.html', reports=report_data_provider.reports) -------------------------------------------------------------------------------- /dashboard/plugins/reports/reports.yaml.template: -------------------------------------------------------------------------------- 1 | reports: 2 | - id: sample_id 3 | definition: 4 | name: Sample report 5 | owner: John Rambo 6 | processing_times: '04:00 UTC' 7 | tables: 8 | - id: some_db.some_table 9 | name: Descriptive Name 10 | -------------------------------------------------------------------------------- /dashboard/plugins/reports/reports_data.py: -------------------------------------------------------------------------------- 1 | import json 2 | from dataclasses import dataclass 3 | from collections import defaultdict 4 | from dashboard.utils.vis import tree_diagram 5 | 6 | 7 | class ReportsDataProvider: 8 | 9 | def __init__(self, tables, reports): 10 | 11 | self.__tables = tables 12 | self.__reports = reports 13 | self.COLORS = { 14 | "success": 1, 15 | "pending": 2, 16 | "failure": 3 17 | } 18 | 19 | @property 20 | def reports(self): 21 | 22 | statuses = {table['id']: table['state'] for table in self.__tables} 23 | 24 | output_reports = [] 25 | 26 | for report in self.__reports: 27 | 28 | id_mapping = {table['id']: index+2 29 | for index, table in enumerate(report['definition']['tables'])} 30 | 31 | tables = [{'id': id_mapping[table['id']], 32 | 'name': table['name'], 33 | 'parent': id_mapping.get(table.get('parent', None)), 34 | 'success': self.COLORS.get(statuses[table['id']], "black") 35 | } 36 | for table in report['definition']['tables']] 37 | 38 | parent_updated = [{**table, **{'parent': 1}} 39 | if table['parent'] is None else table 40 | for table in tables] 41 | 42 | output_reports.append({ 43 | 'id': report['id'], 44 | 'name': report['definition']['name'], 45 | 'data': json.dumps( 46 | tree_diagram([{'id': 1, 'tid': None, 'name': report['definition']['name'], 'success': 0}] + parent_updated, provide_links=True) 47 | ) 48 | }) 49 | 50 | return output_reports 51 | -------------------------------------------------------------------------------- /dashboard/plugins/reports/reports_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/reports/reports_list.png -------------------------------------------------------------------------------- /dashboard/plugins/reports/templates/reports/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} {% import 'macros/common.html' as common with context %} 2 | 3 | {% block title %}Reports{% endblock %} 4 | 5 | {% block nav %} 6 | 7 |
  • Home
    • 8 |
  • Reports dashboard
    • 9 | 10 | {% endblock %} {% block body %} 11 | 12 | 13 |
    14 | 15 | 16 |
    17 |
    18 | 24 |
    25 |
    26 |
    27 |

    Legend:

    28 |
    29 |
    30 |

    ◉ Success

    31 |
    32 |
    33 |

    ◉ In Progress

    34 |
    35 |
    36 |

    ◉ Failure

    37 |
    38 |
    39 |

    ◉ Root Node

    40 |
    41 |
    42 |
    43 |
    44 |
    45 | {% for report in reports %} 46 | 54 |
    55 |
    56 |
    57 |
    58 |
    59 | {% endfor %} 60 |
    61 |
    62 |
    63 | 64 | 65 | 66 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/README.md: -------------------------------------------------------------------------------- 1 | # S3 Usage plugin 2 | 3 | This plugin provides metadata on data stored in multiple S3 buckets in AWS. The main 4 | purpose is to provide a view on: 5 | * inefficiently stored data (multiple small files), 6 | * size of each table/partition, 7 | * temporary data using S3 space. 8 | 9 | The plugin configration is stored as `S3_USAGE_PARAMS` as a dictionary with the 10 | following parameters: 11 | * `buckets_regexp` - regular expression to match the buckets 12 | * `aws_access_key_id` and `aws_secret_access_key` - credentials to be used to communicate 13 | with AWS API 14 | * `ttl` (optional) - time to live of cached sizes in seconds (default: 24 hours) 15 | 16 | From the technical perspective: after the application starts, plugin starts `S3StatsRefreshTask` 17 | that checks for buckets matching the provided regular expression. For each bucket all keys 18 | are listed and the metadata are stored in aggregated context (in "directory" context) in 19 | SQLite database inside Docker container. Tab view uses REST API to retrieve the requested 20 | data from SQLite database and present them in form of the directory tree and subburst 21 | diagram. 22 | 23 | ![s3 usage](screenshot.png) -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/__init__.py: -------------------------------------------------------------------------------- 1 | import html 2 | from flask import Blueprint, render_template, jsonify, request, abort 3 | from flask import current_app as app 4 | from .refresher import S3StatsRefresher 5 | from .statsdb import S3Stats 6 | from dashboard.utils import sizeof_fmt 7 | 8 | base_path = '/s3_usage' 9 | tab_name = 'S3 Usage' 10 | plugin = Blueprint('s3_usage', __name__, template_folder='templates') 11 | 12 | def init(app): 13 | S3StatsRefresher(app.logger, app.config['S3_USAGE_PARAMS']).start() 14 | app.s3stats = S3Stats(app.logger) 15 | 16 | @plugin.route('/') 17 | def index(): 18 | if not app.s3stats.is_ready(): 19 | return render_template('s3_usage/initializing.html') 20 | 21 | return render_template('s3_usage/index.html', 22 | storage_classes=app.s3stats.get_usage_by_storage_class("")) 23 | 24 | @plugin.route('/api/buckets') 25 | def list_buckets(): 26 | order = request.args.get('order') 27 | return jsonify(app.s3stats.get_jstree_buckets(order)) 28 | 29 | @plugin.route('/api/directories') 30 | def list_directories(): 31 | base = request.args.get('id') 32 | order = request.args.get('order') 33 | return jsonify(app.s3stats.get_jstree_directories(base, order)) 34 | 35 | @plugin.route('/api/sunburst') 36 | def prepare_sunburst(): 37 | size = 3 38 | base_path = request.args.get('base') 39 | return jsonify(app.s3stats.get_vega_sunburst(base_path, size)) 40 | 41 | @plugin.route('/ajax/storage_classes') 42 | def render_storage_classes(): 43 | path = request.args.get('path') 44 | return render_template('s3_usage/partial_storage_classes.html', 45 | storage_classes=app.s3stats.get_usage_by_storage_class(path)) -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/refresher.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import sqlite3 3 | import fcntl 4 | import os 5 | import time 6 | import threading 7 | import re 8 | from collections import defaultdict 9 | from concurrent.futures import ThreadPoolExecutor 10 | from dataclasses import dataclass 11 | from dashboard.utils import Timer 12 | 13 | STATS_DB_PATH = '/var/run/discreetly/s3_stats.db' 14 | STATS_DB_TMP_PATH = '/var/run/discreetly/s3_stats.db.tmp' 15 | LOCK_PATH = '/var/run/discreetly/s3_stats.db.lock' 16 | 17 | class S3StatsRefresher: 18 | def __init__(self, logger, params): 19 | self.logger = logger 20 | self.params = params 21 | self.validate_params() 22 | 23 | def validate_params(self): 24 | if not 'ttl' in self.params: 25 | self.params['ttl'] = 60 * 60 * 24 # 1 day 26 | def run_refresher_task(self): 27 | try: 28 | S3StatsRefreshTask(self.logger, **(self.params)).run() 29 | next_run_after = self.params['ttl'] 30 | except Exception as e: 31 | next_run_after = 300 32 | self.logger.error(f"S3StatsRefreshTask failed, will retry in 5 minutes", exc_info=e) 33 | 34 | threading.Timer(next_run_after, self.run_refresher_task).start() 35 | 36 | def start(self): 37 | threading.Timer(1, self.run_refresher_task).start() 38 | 39 | class S3StatsRefreshTask: 40 | CREATE_STMT = 'CREATE TABLE stats (path VARCHAR, files INTEGER, size_standard INTEGER, size_ia INTEGER, size_glacier INTEGER, depth INTEGER, is_leaf INTEGER)' 41 | INSERT_STMT = 'INSERT INTO stats VALUES (?, ?, ?, ?, ?, ?, ?)' 42 | 43 | def __init__(self, logger, buckets_regexp, aws_access_key_id, aws_secret_access_key, ttl): 44 | self.s3 = boto3.client('s3', 45 | aws_access_key_id=aws_access_key_id, 46 | aws_secret_access_key=aws_secret_access_key) 47 | self.s3_aggregator = S3Aggregator(self.s3) 48 | self.buckets_regexp = buckets_regexp 49 | self.logger = logger 50 | self.ttl = ttl 51 | 52 | def needs_refresh(self): 53 | try: 54 | return time.time() - os.stat(STATS_DB_PATH).st_mtime > self.ttl 55 | except FileNotFoundError: 56 | return True 57 | 58 | def list_buckets(self): 59 | return [bucket['Name'] for bucket in self.s3.list_buckets()['Buckets'] 60 | if re.match(self.buckets_regexp, bucket['Name'])] 61 | 62 | def run(self): 63 | with open(LOCK_PATH, 'w') as lock: 64 | self.logger.debug("Trying to acquire the S3Stats lock") 65 | fcntl.flock(lock, fcntl.LOCK_EX) 66 | self.logger.debug("S3Stats Lock acquired") 67 | if not self.needs_refresh(): 68 | self.logger.info("No need to refresh S3Stats, exiting") 69 | return 70 | 71 | if os.path.exists(STATS_DB_TMP_PATH): 72 | os.remove(STATS_DB_TMP_PATH) 73 | self.sqlite = sqlite3.connect(STATS_DB_TMP_PATH) 74 | self.sqlite.execute(self.CREATE_STMT) 75 | self.sqlite.commit() 76 | 77 | buckets = self.list_buckets() 78 | self.logger.info(f"Downloading stats for buckets {buckets}") 79 | global_stats = KeyPrefix() 80 | for bucket in buckets: 81 | data = self.load_s3_bucket(bucket) 82 | bucket_stats = self.dump_s3_bucket(data, bucket) 83 | global_stats += bucket_stats 84 | 85 | self.sqlite.execute(self.INSERT_STMT, ("", global_stats.files, global_stats.size_standard, global_stats.size_ia, global_stats.size_glacier, 0, 0)) 86 | self.sqlite.commit() 87 | self.sqlite.close() 88 | os.rename(STATS_DB_TMP_PATH, STATS_DB_PATH) 89 | 90 | def insert_iterator(self, bucket, data): 91 | parents = set([path[:path.rfind('/')] for path in data.keys()]) 92 | for key in data: 93 | key_fixed = key.rstrip('/') 94 | is_leaf = not key in parents 95 | yield f"/{bucket}{key_fixed}", data[key].files, \ 96 | data[key].size_standard, data[key].size_ia, data[key].size_glacier, \ 97 | len(key_fixed.split('/')), int(is_leaf) 98 | 99 | def load_s3_bucket(self, bucket): 100 | with Timer(self.logger, f"Listing bucket {bucket}"): 101 | data = self.s3_aggregator.load(bucket) 102 | self.logger.info(f"Listed {len(data)} directories ({data['/'].files} files) in bucket {bucket}") 103 | return data 104 | 105 | def dump_s3_bucket(self, data, bucket): 106 | with Timer(self.logger, f"Dumping bucket {bucket} to DB"): 107 | self.sqlite.execute('BEGIN') 108 | self.sqlite.executemany(self.INSERT_STMT, self.insert_iterator(bucket, data)) 109 | self.sqlite.commit() 110 | return data['/'] 111 | 112 | @dataclass 113 | class KeyPrefix: 114 | files:int = 0 115 | size_standard:int = 0 116 | size_ia:int = 0 117 | size_glacier:int = 0 118 | 119 | def add_file(self, size, storage_class): 120 | self.files += 1 121 | if storage_class == 'STANDARD_IA': 122 | self.size_ia += size 123 | elif storage_class == 'GLACIER': 124 | self.size_glacier += size 125 | else: 126 | self.size_standard += size 127 | 128 | def __add__(self, other): 129 | merged = KeyPrefix() 130 | merged.files = self.files + other.files 131 | merged.size_standard = self.size_standard + other.size_standard 132 | merged.size_ia = self.size_ia + other.size_ia 133 | merged.size_glacier = self.size_glacier + other.size_glacier 134 | return merged 135 | 136 | class S3Aggregator: 137 | def __init__(self, s3): 138 | self.s3 = s3 139 | 140 | def load(self, bucket): 141 | db = defaultdict(KeyPrefix) 142 | 143 | for page in self.s3.get_paginator('list_objects_v2').paginate(Bucket=bucket): 144 | for obj in page.get('Contents', ()): 145 | key_parts = obj['Key'].split('/') 146 | for i in range(len(key_parts)): 147 | if key_parts[i] == '': 148 | key_parts[i] = '' 149 | for depth in range(len(key_parts)): 150 | path = '/' + '/'.join(key_parts[:depth]) 151 | db[path].add_file(obj['Size'], obj['StorageClass']) 152 | 153 | return db -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/s3_usage/screenshot.png -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/statsdb.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sqlite3 3 | import html 4 | from dashboard.utils import sizeof_fmt 5 | from .refresher import STATS_DB_PATH 6 | from dashboard.utils import Timer 7 | 8 | class S3Stats: 9 | def __init__(self, logger): 10 | self.db = S3StatsDB(logger) 11 | 12 | def is_ready(self): 13 | return self.db.is_ready() 14 | 15 | def get_usage_by_storage_class(self, path): 16 | data = self.db.query(path, children_only=False, depth=len(path.split('/'))-1) 17 | if len(data) == 0: 18 | return {} 19 | 20 | return { 21 | sclass: { 22 | 'size': sizeof_fmt(data[0][sclass]), 23 | 'percent': data[0][sclass] / data[0]['size'] 24 | } for sclass in ['size_standard', 'size_ia', 'size_glacier'] 25 | } 26 | 27 | def describe(self, node, strip=''): 28 | return f"{html.escape(node['path'][len(strip):])} ({node['files']} files, {sizeof_fmt(node['size'])})" 29 | 30 | def get_jstree_buckets(self, order): 31 | total = self.db.query('', 0)[0] 32 | buckets = self.db.query('', 1, order, children_only=True) 33 | 34 | return { 35 | "text" : "All buckets " + self.describe(total), "state": {"opened": True}, "id": "root", 36 | "children" : [{ "id": bucket['path'], "text" : self.describe(bucket, '/'), "children" : True } 37 | for bucket in buckets]} 38 | 39 | def get_vega_sunburst(self, base, size): 40 | base_depth = len(base.split('/')) - 1 41 | data = {} 42 | for idx, row in enumerate(self.db.query(base, depth=(base_depth, base_depth + size), order='alphabetically')): 43 | path = row['path'] 44 | path_split = path.split('/') 45 | name = path_split[-1] if path_split[-1] != '' else 'all buckets' 46 | data[path] = { 47 | 'id': idx+1, 48 | 'size': row['size'], 49 | 'name': name, 50 | 'size_fmt': sizeof_fmt(row['size']) } 51 | parent_dir = '/'.join(path_split[:-1]) 52 | if path != '' and parent_dir in data: 53 | data[path]['parent'] = data[parent_dir]['id'] 54 | data[parent_dir]['size'] -= data[path]['size'] 55 | 56 | return list(data.values()) 57 | 58 | def get_jstree_directories(self, base, order): 59 | return [ 60 | { "id": directory['path'], 61 | "text" : self.describe(directory, strip=base + '/'), 62 | "children" : directory['is_leaf'] == 0 } 63 | for directory in self.db.query(base, order=order, children_only=True) 64 | ] 65 | 66 | class S3StatsDB: 67 | ORDER_DEFINITION = { 68 | 'alphabetically': 'path asc', 69 | 'avg_file_size_desc': 'size/files asc', 70 | 'files_desc': 'files desc', 71 | 'size_desc': 'size desc' 72 | } 73 | def __init__(self, logger): 74 | self.logger = logger 75 | 76 | def is_ready(self): 77 | return os.path.exists(STATS_DB_PATH) 78 | 79 | def format_path_condition(self, base, children_only): 80 | if children_only: 81 | return f"path like '{base}/%'" 82 | else: 83 | return f"(path like '{base}/%' or path = '{base}')" 84 | 85 | def format_depth_condition(self, base, depth): 86 | if depth is None: 87 | depth = len(base.split('/')) 88 | if type(depth) is tuple: 89 | return f'depth between {depth[0]} and {depth[1]}' 90 | else: 91 | return f'depth = {depth}' 92 | 93 | def query(self, base, depth=None, order="size_desc", children_only=False): 94 | sqlite = sqlite3.connect(STATS_DB_PATH) 95 | sqlite.row_factory = sqlite3.Row 96 | order = self.ORDER_DEFINITION.get(order, self.ORDER_DEFINITION['size_desc']) 97 | query = f"SELECT *, (size_standard + size_ia + size_glacier) as size " \ 98 | f"FROM stats " \ 99 | f"WHERE {self.format_path_condition(base, children_only)} " \ 100 | f"AND {self.format_depth_condition(base, depth)} " \ 101 | f"order by {order}" 102 | with Timer(self.logger, f"S3 stats query {query}"): 103 | try: 104 | cursor = sqlite.cursor() 105 | cursor.execute(query) 106 | return cursor.fetchall() 107 | finally: 108 | cursor.close() 109 | sqlite.close() 110 | -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/templates/s3_usage/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} {% import 'macros/common.html' as common with context %} 2 | 3 | {% block autorefresh %}{% endblock %} 4 | 5 | {% block title %}S3 Usage{% endblock %} 6 | 7 | {% block css %} 8 | 9 | {% endblock %} 10 | 11 | 12 | 13 | {% block nav %} 14 |
  • Home
    • 15 |
  • S3 Usage
    • 16 | {% endblock %} 17 | 18 | {% block body %} 19 |

    S3 usage

    20 |
    21 |
    22 | Sort tree by: 23 |
    24 |
    25 | 26 | 27 |
    28 |
    29 | 30 | 31 |
    32 |
    33 | 34 | 35 |
    36 |
    37 | 38 | 39 |
    40 |
    41 |
    42 |
    43 |
    44 | Selected path: all buckets, storage classes distribution: 45 |
    46 | {% include 's3_usage/partial_storage_classes.html' with context %} 47 |
    48 |
    49 |
    50 |
    51 | {% endblock %} 52 | 53 | 54 | {% block js %} 55 | 56 | 83 | 84 | 85 | 165 | {% endblock %} 166 | -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/templates/s3_usage/initializing.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} {% import 'macros/common.html' as common with context %} 2 | 3 | {% block title %}S3 Usage{% endblock %} 4 | 5 | {% block nav %} 6 | 7 |
  • Home
    • 8 |
  • S3 Usage
    • 9 | 10 | {% endblock %} 11 | {% block body %} 12 | 13 |

    S3 Usage

    14 | 15 |

    Module is initializing...

    16 | 17 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/templates/s3_usage/partial_storage_classes.html: -------------------------------------------------------------------------------- 1 | {% if storage_classes.size_standard.percent > 0 %} 2 |
    STANDARD: {{ storage_classes.size_standard.size}}
    3 | {% endif %} 4 | {% if storage_classes.size_ia.percent > 0 %} 5 |
    INFREQUENT ACCES: {{ storage_classes.size_ia.size}}
    6 | {% endif %} 7 | {% if storage_classes.size_glacier.percent > 0 %} 8 |
    GLACIER: {{ storage_classes.size_glacier.size}}
    9 | {% endif %} 10 | -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/s3_usage/tests/__init__.py -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/tests/s3stats.sql: -------------------------------------------------------------------------------- 1 | BEGIN TRANSACTION; 2 | CREATE TABLE stats (path VARCHAR, files INTEGER, size_standard INTEGER, size_ia INTEGER, size_glacier INTEGER, depth INTEGER, is_leaf INTEGER); 3 | INSERT INTO stats VALUES('/discreetly-bucket1',3,3145728,0,0,1,1); 4 | INSERT INTO stats VALUES('/discreetly-bucket1/table',3,3145728,0,0,2,0); 5 | INSERT INTO stats VALUES('/discreetly-bucket1/table/year=2019',3,3145728,0,0,3,0); 6 | INSERT INTO stats VALUES('/discreetly-bucket1/table/year=2019/month=01',3,3145728,0,0,4,0); 7 | INSERT INTO stats VALUES('/discreetly-bucket1/table/year=2019/month=01/day=01',1,1048576,0,0,5,1); 8 | INSERT INTO stats VALUES('/discreetly-bucket1/table/year=2019/month=01/day=02',1,1048576,0,0,5,1); 9 | INSERT INTO stats VALUES('/discreetly-bucket1/table/year=2019/month=01/day=03',1,1048576,0,0,5,1); 10 | INSERT INTO stats VALUES('/discreetly-bucket2',2,2097152,0,0,1,1); 11 | INSERT INTO stats VALUES('/discreetly-bucket2/dir1',1,1048576,0,0,2,1); 12 | INSERT INTO stats VALUES('/discreetly-bucket2/dir2',1,1048576,0,0,2,1); 13 | INSERT INTO stats VALUES('',5,5242880,0,0,0,0); 14 | COMMIT; -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/tests/test_refresher.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from dashboard.plugins.s3_usage.refresher import S3StatsRefreshTask, STATS_DB_PATH 3 | from dashboard.plugins.s3_usage.statsdb import S3StatsDB 4 | from moto import mock_s3 5 | import os 6 | import boto3 7 | import pytest 8 | 9 | @pytest.yield_fixture(scope="function") 10 | def fixtures(request): 11 | mock_s3().start() 12 | task = S3StatsRefreshTask( 13 | logger=logging, 14 | buckets_regexp="^my-bucket$", 15 | aws_access_key_id='', 16 | aws_secret_access_key='', 17 | ttl=60 18 | ) 19 | yield task, boto3.client("s3") 20 | 21 | if os.path.exists(STATS_DB_PATH): 22 | os.unlink(STATS_DB_PATH) 23 | mock_s3().stop() 24 | 25 | def test_no_matching_bucket_and_no_failure(fixtures): 26 | # given no buckets 27 | task = fixtures[0] 28 | 29 | # when 30 | task.run() 31 | 32 | # then 33 | assert len(task.list_buckets()) == 0 34 | assert len(S3StatsDB(logging).query("")) == 0 35 | 36 | def test_one_empty_bucket_case(fixtures): 37 | # given 38 | task = fixtures[0] 39 | s3client = fixtures[1] 40 | s3client.create_bucket(Bucket='my-bucket') 41 | 42 | # when 43 | task.run() 44 | 45 | # then 46 | stats = S3StatsDB(logging).query("")[0] 47 | assert stats['size'] == 0 48 | assert stats['files'] == 0 49 | 50 | 51 | def test_files_in_bucket_root(fixtures): 52 | # given 53 | task = fixtures[0] 54 | s3client = fixtures[1] 55 | s3client.create_bucket(Bucket='my-bucket') 56 | s3client.put_object(Body='hello', Bucket='my-bucket', Key='hello') 57 | s3client.put_object(Body='world', Bucket='my-bucket', Key='world') 58 | 59 | # when 60 | task.run() 61 | 62 | # then 63 | global_stats = S3StatsDB(logging).query("")[0] 64 | assert global_stats['size'] == len('hello') + len('world') 65 | assert global_stats['files'] == 2 66 | root_stats = S3StatsDB(logging).query("/my-bucket", depth=1)[0] 67 | assert root_stats['size'] == len('hello') + len('world') 68 | assert root_stats['files'] == 2 69 | 70 | def test_files_and_directories(fixtures): 71 | # given 72 | task = fixtures[0] 73 | s3client = fixtures[1] 74 | s3client.create_bucket(Bucket='my-bucket') 75 | s3client.put_object(Body='hello', Bucket='my-bucket', Key='greeting/hello') 76 | s3client.put_object(Body='world', Bucket='my-bucket', Key='greeting/world') 77 | s3client.put_object(Body='see_greeting!', Bucket='my-bucket', Key='clue') 78 | 79 | # when 80 | task.run() 81 | 82 | # then 83 | root_stats = S3StatsDB(logging).query("/my-bucket", depth=1)[0] 84 | assert root_stats['size'] == len('hello') + len('world') + len('see_greeting!') 85 | assert root_stats['files'] == 3 86 | 87 | greeting_stats = S3StatsDB(logging).query("/my-bucket/greeting", depth=2)[0] 88 | assert greeting_stats['size'] == len('hello') + len('world') 89 | assert greeting_stats['files'] == 2 90 | 91 | def test_different_storage_classes(fixtures): 92 | # given 93 | task = fixtures[0] 94 | s3client = fixtures[1] 95 | s3client.create_bucket(Bucket='my-bucket') 96 | s3client.put_object(Body='test', Bucket='my-bucket', Key='standard', StorageClass='STANDARD') 97 | s3client.put_object(Body='test', Bucket='my-bucket', Key='ia', StorageClass='STANDARD_IA') 98 | # can't test GLACIER (https://stackoverflow.com/a/41841229/7098262) 99 | 100 | # when 101 | task.run() 102 | 103 | # then 104 | global_stats = S3StatsDB(logging).query("")[0] 105 | assert global_stats['size'] == 2 * len('test') 106 | assert global_stats['size_standard'] == len('test') 107 | assert global_stats['size_ia'] == len('test') 108 | assert global_stats['files'] == 2 -------------------------------------------------------------------------------- /dashboard/plugins/s3_usage/tests/test_statsdb.py: -------------------------------------------------------------------------------- 1 | from dashboard.plugins.s3_usage.statsdb import S3Stats 2 | from dashboard.tests.mocks.database import SqliteService 3 | from dashboard.plugins.s3_usage.refresher import STATS_DB_PATH 4 | import pytest 5 | import logging 6 | 7 | @pytest.fixture(scope='session') 8 | def s3stats(request): 9 | sqlite = SqliteService(STATS_DB_PATH) 10 | SqliteService.migrate(sqlite.conn, 'dashboard/plugins/s3_usage/tests/s3stats.sql') 11 | return S3Stats(logging) 12 | 13 | def test_list_buckets(s3stats): 14 | # when 15 | stats = s3stats.get_jstree_buckets(order='size_desc') 16 | 17 | # then 18 | assert stats['text'] == 'All buckets (5 files, 5.0MiB)' 19 | assert len(stats['children']) == 2 20 | assert stats['children'][0]['text'] == 'discreetly-bucket1 (3 files, 3.0MiB)' 21 | assert stats['children'][0]['children'] == True 22 | assert stats['children'][1]['text'] == 'discreetly-bucket2 (2 files, 2.0MiB)' 23 | assert stats['children'][1]['children'] == True 24 | 25 | def test_list_buckets_change_order(s3stats): 26 | # when 27 | stats = s3stats.get_jstree_buckets(order='alphabetically') 28 | 29 | # then 30 | assert stats['text'] == 'All buckets (5 files, 5.0MiB)' 31 | assert len(stats['children']) == 2 32 | assert stats['children'][0]['text'] == 'discreetly-bucket1 (3 files, 3.0MiB)' 33 | assert stats['children'][0]['children'] == True 34 | assert stats['children'][1]['text'] == 'discreetly-bucket2 (2 files, 2.0MiB)' 35 | assert stats['children'][1]['children'] == True 36 | 37 | def test_list_directories_leafs(s3stats): 38 | # when 39 | stats = s3stats.get_jstree_directories(base='/discreetly-bucket2', order='size_desc') 40 | 41 | # then 42 | assert len(stats) == 2 43 | assert stats[0]['text'] == 'dir1 (1 files, 1.0MiB)' 44 | assert stats[0]['children'] == False 45 | assert stats[1]['text'] == 'dir2 (1 files, 1.0MiB)' 46 | assert stats[1]['children'] == False 47 | 48 | def test_list_directories_non_leafs(s3stats): 49 | # when 50 | stats = s3stats.get_jstree_directories(base='/discreetly-bucket1/table', order='size_desc') 51 | 52 | # then 53 | assert len(stats) == 1 54 | assert stats[0]['text'] == 'year=2019 (3 files, 3.0MiB)' 55 | assert stats[0]['children'] == True 56 | 57 | def test_sunburst_data_on_root(s3stats): 58 | # when 59 | stats = s3stats.get_vega_sunburst(base='', size=3) 60 | 61 | # then 62 | expected_response = [ 63 | {'id': 1, 'size': 0, 'name': 'all buckets', 'size_fmt': '5.0MiB'}, 64 | {'id': 2, 'size': 0, 'name': 'discreetly-bucket1', 'size_fmt': '3.0MiB', 'parent': 1}, 65 | {'id': 3, 'size': 0, 'name': 'table', 'size_fmt': '3.0MiB', 'parent': 2}, 66 | {'id': 4, 'size': 3145728, 'name': 'year=2019', 'size_fmt': '3.0MiB', 'parent': 3}, 67 | {'id': 5, 'size': 0, 'name': 'discreetly-bucket2', 'size_fmt': '2.0MiB', 'parent': 1}, 68 | {'id': 6, 'size': 1048576, 'name': 'dir1', 'size_fmt': '1.0MiB', 'parent': 5}, 69 | {'id': 7, 'size': 1048576, 'name': 'dir2', 'size_fmt': '1.0MiB', 'parent': 5} 70 | ] 71 | assert stats == expected_response 72 | 73 | def test_sunburst_data_on_non_root(s3stats): 74 | # when 75 | stats = s3stats.get_vega_sunburst(base='/discreetly-bucket1/table/year=2019', size=3) 76 | 77 | # then 78 | expected_response = [ 79 | {'id': 1, 'size': 0, 'name': 'year=2019', 'size_fmt': '3.0MiB'}, 80 | {'id': 2, 'size': 0, 'name': 'month=01', 'size_fmt': '3.0MiB', 'parent': 1}, 81 | {'id': 3, 'size': 1048576, 'name': 'day=01', 'size_fmt': '1.0MiB', 'parent': 2}, 82 | {'id': 4, 'size': 1048576, 'name': 'day=02', 'size_fmt': '1.0MiB', 'parent': 2}, 83 | {'id': 5, 'size': 1048576, 'name': 'day=03', 'size_fmt': '1.0MiB', 'parent': 2}] 84 | assert stats == expected_response 85 | -------------------------------------------------------------------------------- /dashboard/plugins/streaming/README.md: -------------------------------------------------------------------------------- 1 | # Streaming applications view 2 | 3 | The Streaming tab lists the applications that consume a data streams and put the records 4 | to the Data Warehouse. Each row contains a stream name, link to external monitoring screen 5 | (for example AWS Kinesis Monitoring tab) and status of the stream. Also a time series chart 6 | is presented, showing the lag of stream consumption (values are provided by LagProvider 7 | implementation and currently only InfluxDB is supported). 8 | 9 | Status is based on the alerts provided by Prometheus, but assigned to the tables that 10 | are filled by dumping the stream. The configuration file requires you to provide a list 11 | of entries with the following keys: 12 | - `name` - name of stream to be displayed on the dashboard 13 | - `link` - link to external monitoing site (can be empty) 14 | - `table` - name of the table that stores data dumped by the streaming application 15 | 16 | You can find more details in the [example file](streaming.yaml.template). For the plugin 17 | to work the file should be named `streaming.yaml` and needs to be put in `config` directory. 18 | Streaming plugin requires valid [tables.yaml](../tables/tables.yaml.template) to be 19 | provided to match alerts for tables mentioned in `streaming.yaml`. 20 | 21 | ![streaming](streaming.png) 22 | -------------------------------------------------------------------------------- /dashboard/plugins/streaming/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint 2 | from flask import render_template, jsonify 3 | from flask import current_app as app 4 | from dashboard.utils import get_yaml_file_content 5 | 6 | base_path = '/streaming' 7 | tab_name = 'Streaming' 8 | plugin = Blueprint('streaming', __name__, template_folder='templates') 9 | 10 | 11 | def init(app): 12 | lag_provider_class = app.config.get('STREAMING_LAG_PROVIDER', 'InfluxBasedLagProvider') 13 | class_instance = globals()[lag_provider_class] 14 | app.streaming_lag_data_provider = class_instance(app) 15 | 16 | @plugin.route('/') 17 | def index(): 18 | streams_definition = get_yaml_file_content('config/streaming.yaml') 19 | if not streams_definition: 20 | return render_template('no_config.html', filename='streaming.yaml') 21 | 22 | alerts = [table['id'] for table in app.table_data_provider.list() if table['active_alerts']] 23 | streams = [dict(active_alerts = stream['table'] in alerts, **stream) for stream in streams_definition] 24 | 25 | return render_template('streaming/index.html', streams=streams) 26 | 27 | 28 | @plugin.route('/api/lag///') 29 | def api_lag_value(stream, table, period): 30 | return jsonify(app.streaming_lag_data_provider.get_lag(stream, table, period)) 31 | 32 | class LagProvider: 33 | def __init__(self, app): 34 | self.app = app 35 | self.interval = app.config.get('STREAMING_BATCH_INTERVAL', '15m') 36 | 37 | def get_lag(self, stream, table, period): 38 | raise NotImplementedException() 39 | 40 | class InfluxBasedLagProvider(LagProvider): 41 | def get_lag(self, stream, table, period): 42 | return list(self.app.influx_client.query('SELECT mean(extra_lag)/1000 AS lag FROM emr_stats_{} WHERE time >= now() - {} GROUP BY time({})'.format( 43 | table.replace('.', '_'), period, self.interval)).get_points()) 44 | 45 | -------------------------------------------------------------------------------- /dashboard/plugins/streaming/streaming.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/streaming/streaming.png -------------------------------------------------------------------------------- /dashboard/plugins/streaming/streaming.yaml.template: -------------------------------------------------------------------------------- 1 | - name: friendly_name 2 | link: "https://console.aws.amazon.com/kinesis/home?region=us-east-1#/streams/details?streamName=myStream&tab=monitoring" 3 | table: dbname.tablename 4 | -------------------------------------------------------------------------------- /dashboard/plugins/streaming/templates/streaming/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} {% import 'macros/common.html' as common with context %} 2 | 3 | {% block title %}Streaming{% endblock %} 4 | 5 | {% block nav %} 6 | 7 |
  • Home
    • 8 |
  • Streaming
    • 9 | 10 | {% endblock %} 11 | 12 | {% block body %} 13 | 14 | 15 | 16 |
    17 |
    18 | 19 |
    20 |
    Streaming name
    21 |
    Streaming table
    22 |
    Status
    23 |
    Lag [ 24 |
    25 | 26 | 27 |
    28 |
    29 | 30 | 31 |
    32 |
    33 | 34 | 35 |
    36 | ] 37 |
    38 |
    39 | {% for stream in streams %} 40 |
    41 |
    {{ stream.name }}
    42 |
    43 | {{ stream.table }} 44 |
    45 |
    46 | {% if stream.active_alerts %} 47 |

    ❌

    48 | {% else %} 49 |

    ✅

    50 | {% endif %} 51 |
    52 |
    53 |
    54 |
    55 |
    56 | {% endfor %} 57 |
    58 |
    59 | 60 | {% endblock %} 61 | 62 | {% block js %} 63 | 102 | 103 | {% endblock %} 104 | -------------------------------------------------------------------------------- /dashboard/plugins/table_descriptions/README.md: -------------------------------------------------------------------------------- 1 | # Table descriptions View 2 | 3 | The table descriptions tab displays table and column descriptions (comments). This can be useful for stakeholders to better understand your data structure and search for particular information. The default implementation takes them from AWS glue which stores comments added during table creation. Since not all tables have to have comments provided, this tab is fully optional. 4 | 5 | It's possible to set a custom data provider which reads the table descriptions from a different source than AWS glue. This is controlled by the `TABLE_DESCRIPTION_SERVICE` setting. 6 | 7 | Table descriptions plugin requires valid [tables.yaml](../tables/tables.yaml.template) to be 8 | provided to provide a subset of the tables to be displayed. 9 | 10 | ![table descriptions](table_descriptions.png) -------------------------------------------------------------------------------- /dashboard/plugins/table_descriptions/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint 2 | from flask import render_template 3 | from flask import current_app as app 4 | from dashboard.utils import load_data_provider 5 | 6 | base_path = '/table_descriptions' 7 | tab_name = 'Table descriptions' 8 | plugin = Blueprint('table_descriptions', __name__, template_folder='templates') 9 | 10 | DEFAULT_CACHE = 300 11 | def get_descriptions(): 12 | cache_ttl = app.config.get('TABLE_DESCRIPTIONS_TTL', DEFAULT_CACHE) 13 | descriptions = app.cache.get('table_descriptions') 14 | if descriptions is not None: 15 | return descriptions 16 | 17 | classname = app.config['TABLE_DESCRIPTIONS_SERVICE'] 18 | params = app.config['TABLE_DESCRIPTIONS_SERVICE_PARAMS'] 19 | params['logger'] = app.logger 20 | params['config'] = app.config 21 | params['tables'] = app.table_data_provider.tables 22 | 23 | descriptions = load_data_provider(classname, params).get_table_descriptions() 24 | app.cache.set('table_descriptions', descriptions, timeout=cache_ttl) 25 | return descriptions 26 | 27 | @plugin.route('/') 28 | def index(): 29 | if not 'TABLE_DESCRIPTIONS_SERVICE' in app.config or app.table_data_provider is None: 30 | return render_template('no_config.html', filename='TABLE_DESCRIPTIONS_SERVICE') 31 | 32 | return render_template('table_descriptions/index.html', tables=get_descriptions()) -------------------------------------------------------------------------------- /dashboard/plugins/table_descriptions/dataproviders.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from timeit import default_timer as timer 3 | 4 | import boto3 5 | 6 | from flask import current_app as app 7 | 8 | 9 | class GlueTimer: 10 | 11 | def __init__(self, logger): 12 | self.logger = logger 13 | 14 | def __enter__(self): 15 | self.start = timer() 16 | self.logger.debug(f"Glue description fetch started") 17 | return self 18 | 19 | def __exit__(self, *args): 20 | self.end = timer() 21 | self.interval = self.end - self.start 22 | self.logger.info(f"Glue description fetch took {self.interval:.2f} sec") 23 | 24 | class GlueDescriptionService: 25 | 26 | def __init__(self, logger, config, tables, region_name, aws_access_key_id, aws_secret_access_key): 27 | self.client = boto3.client('glue', 28 | region_name=region_name, 29 | aws_access_key_id=aws_access_key_id, 30 | aws_secret_access_key=aws_secret_access_key) 31 | self.logger = logger 32 | self.db_tables = self.get_db_tables(tables) 33 | 34 | 35 | @staticmethod 36 | def get_db_tables(tables): 37 | result = defaultdict(list) 38 | for table in tables.values(): 39 | result[table.db].append(table.name) 40 | return result 41 | 42 | def get_tables_for_db(self, db, tables): 43 | """ 44 | Get tables with their columns and descriptions for a given db 45 | :param db: Name of database 46 | :param tables: List of table names 47 | :return: List of table dicts (name = table name, columns = list of dicts with 'name' and 'comment' keys) 48 | """ 49 | try: 50 | glue_tables = self.client.get_tables(DatabaseName=db, MaxResults=1000) 51 | result_tables = [] 52 | for table in glue_tables['TableList']: 53 | columns = [{'name': column['Name'], 54 | 'description': column.get('Comment', ''), 55 | 'is_partition': False, 56 | 'type': column.get('Type')} 57 | for column in table['StorageDescriptor']['Columns']] 58 | 59 | columns.extend([{'name': column['Name'], 60 | 'description': column.get('Comment', ''), 61 | 'is_partition': True, 62 | 'type': column.get('Type')} 63 | for column in table.get('PartitionKeys', [])]) 64 | 65 | result_tables.append({'name': table['Name'], 66 | 'description': table.get('Parameters', {}).get('comment', ''), 67 | 'columns': columns}) 68 | 69 | result_tables = [t for t in result_tables if t['name'] in tables] 70 | return sorted(result_tables, key=lambda k: k['name']) 71 | except Exception as e: 72 | self.logger.warn('Exception caught while trying to get table descriptions', e) 73 | return [] 74 | 75 | 76 | def get_table_descriptions(self): 77 | """ 78 | Returns a dict of key = 'db_name.table_name' and 79 | value = dict with table description and list of column dicts with name, type and description fields 80 | """ 81 | futures = list() 82 | with GlueTimer(self.logger): 83 | for db, tables in self.db_tables.items(): 84 | futures.append((db, app.async_request_executor.submit(self.get_tables_for_db, db, tables))) 85 | 86 | result = dict() 87 | for db, tables in futures: 88 | for table in tables.result(): 89 | result['{}.{}'.format(db, table['name'])] = table 90 | 91 | return result 92 | -------------------------------------------------------------------------------- /dashboard/plugins/table_descriptions/table_descriptions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/table_descriptions/table_descriptions.png -------------------------------------------------------------------------------- /dashboard/plugins/table_descriptions/templates/table_descriptions/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} 2 | {% import 'macros/common.html' as common with context %} 3 | 4 | {% block autorefresh %}{% endblock %} 5 | 6 | {% block title %}Table {% endblock %} 7 | 8 | {% block nav %} 9 |
  • Home
    • 10 |
  • Tables dashboard
    • 11 | {% endblock %} 12 | 13 | {% block body %} 14 | 15 | 16 | 17 |
    18 | 24 |
    25 | Legend: Partition column 26 |
    27 |
    28 | 29 |
    30 |
    31 |
    32 | 33 | Database | Table name 34 |
    35 |
    Description
    36 |
    37 | {% for table_name, details in tables.items() %} 38 |
    39 |
    40 |
    41 | 42 | 43 | 44 | 45 | {{ table_name }} 46 | 47 |
    48 |
    {{ details.description }}
    49 |
    50 |
    51 |
    52 |
    Column
    53 |
    Type
    54 |
    Description
    55 |
    56 | {% for column in details.columns %} 57 |
    58 |
    {{ column.name }} {% if column.is_partition %}{% endif %}
    59 |
    {{ column.type }}
    60 |
    {{ column.description }}
    61 |
    62 | {% endfor %} 63 |
    64 |
    65 | {% endfor %} 66 |
    67 | 68 | {% endblock %} 69 | 70 | {% block charts %} 71 | 72 | 73 | 74 | 120 | {% endblock %} 121 | -------------------------------------------------------------------------------- /dashboard/plugins/tables/README.md: -------------------------------------------------------------------------------- 1 | # Tables View 2 | 3 | The dashboard allows the users to monitor the progress of particular Airflow DAGs and 4 | tasks, moreover, it can also show the status of tasks in relation to tables they populate 5 | with data. 6 | 7 | However, Airflow does not contain the mapping between tables (Hive, database) and that is 8 | why a mapping needs to be provided to **discreETLy**. A mapping can be defined in 9 | `tables.yaml` file available in `config` folder. Each mapping consists of a few pieces of information: 10 | 11 | - `name` - the name of the table in a database 12 | - `db` - a database definition (can be a namespace) 13 | - `uses` - a table that provides data for the task populating currently describe table (helps if there are dependencies between tables) 14 | - `dag_id` - id of a DAG that contains the task required for the mapping 15 | - `task_id` - id of the task that populates the table 16 | - (optional) 17 | 18 | Additionally you may want to provide cadence for how often the table is updated like so: 19 | ``` 20 | period: 21 | id: 1 22 | name: daily 23 | ``` 24 | Update `id` incrementally starting from 1 for each table redeclared with another cadence. 25 | This makes specifying dependency on a particular cadence possible, e.g.: `uses: dbname.frequently_updated.daily`. 26 | 27 | 28 | See [example file](tables.yaml.template) for more details on the data structure. 29 | 30 | The application will automatically ingest the definition of the tables and map them to particular tasks. 31 | 32 | ![tables list](tables_list.png) 33 | 34 | ![table detailed view](table_details.png) -------------------------------------------------------------------------------- /dashboard/plugins/tables/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint 2 | from flask import render_template 3 | from flask import current_app as app 4 | 5 | base_path = '/tables' 6 | tab_name = 'Tables' 7 | plugin = Blueprint('tables', __name__, template_folder='templates') 8 | 9 | @plugin.route('/') 10 | def index(): 11 | if not app.table_data_provider: 12 | return render_template('no_config.html', filename='tables.yaml') 13 | 14 | return render_template('tables/index.html', tables=app.table_data_provider.list()) 15 | 16 | 17 | @plugin.route('/') 18 | def details(table_name): 19 | history = app.table_data_provider.history(table_name) 20 | return render_template('tables/details.html', name=table_name, counts=history) -------------------------------------------------------------------------------- /dashboard/plugins/tables/table_details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/tables/table_details.png -------------------------------------------------------------------------------- /dashboard/plugins/tables/tables.yaml.template: -------------------------------------------------------------------------------- 1 | - name: popular_raw_table 2 | db: dbname 3 | dag_id: my_dag 4 | task_id: popular_raw_table_insert 5 | 6 | - name: fact_table 7 | db: dbname 8 | dag_id: my_dag 9 | task_id: fact_table_insert 10 | uses: dbname.popular_raw_table 11 | 12 | # declaring a table with a specific cadence 13 | - name: frequently_updated_table 14 | db: dbname 15 | dag_id: my_dag 16 | task_id: update_daily_frequently_updated_table 17 | period: 18 | id: 1 19 | name: daily 20 | 21 | - name: rollup_table_daily 22 | db: dbname 23 | dag_id: my_dag 24 | task_id: rollup_table_insert 25 | uses: dbname.frequently_updated_table.daily 26 | -------------------------------------------------------------------------------- /dashboard/plugins/tables/tables_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/plugins/tables/tables_list.png -------------------------------------------------------------------------------- /dashboard/plugins/tables/templates/tables/details.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} 2 | {% import 'macros/common.html' as common with context %} 3 | 4 | {% block title %}Table {{ name }} details{% endblock %} 5 | 6 | {% block nav %} 7 |
  • Home
    • 8 |
  • Tables dashboard
    • 9 |
  • Table {{ name }}
    • 10 | {% endblock %} 11 | 12 | {% block body %} 13 | 14 | {% endblock %} 15 | 16 | {% block charts %} 17 | 74 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/plugins/tables/templates/tables/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} 2 | {% import 'macros/common.html' as common with context %} 3 | 4 | {% block title %}Tables{% endblock %} 5 | 6 | {% block nav %} 7 |
  • Home
    • 8 |
  • Tables dashboard
    • 9 | {% endblock %} 10 | 11 | {% block body %} 12 | 13 | 14 | 15 |
    16 | 22 |
    23 | 24 |
    25 |
    26 |
    Database | Table name
    27 |
    Last update
    28 |
    Status
    29 |
    Record count
    30 |
    31 | {% for table in tables %} 32 |
    33 |
    34 |

    35 | {% if 'table_descriptions' in plugins %}{% endif %} 36 | {{ table.id }} 37 | {% if 'table_descriptions' in plugins %}{% endif %} 38 | {% if table.active_alerts %} 39 | {% for alert in table.active_alerts %} 40 | 41 | {% endfor %} 42 | {% else %} 43 | 44 | {% endif %} 45 |

    46 |
    47 |
    48 | {% if table.last_update %}{{ table.last_update.strftime('%Y-%m-%d %H:%M:%S') }}{% else %}Unknown{% endif %} 49 |
    50 |
    51 | 52 | {{ table.state }} 53 | 54 |
    55 |
    56 | {% if table.counts %} 57 | 58 | {% endif %} 59 |
    60 |
    61 | {% endfor %} 62 | 63 |
    64 | 65 | {% endblock %} 66 | 67 | {% block charts %} 68 | 122 | 123 | 124 | 125 | 126 | 132 | {% endblock %} -------------------------------------------------------------------------------- /dashboard/service/influxdb_service.py: -------------------------------------------------------------------------------- 1 | from influxdb import InfluxDBClient 2 | from dashboard.utils import Timer 3 | from influxdb.exceptions import InfluxDBClientError, InfluxDBServerError 4 | import inspect 5 | import backoff 6 | 7 | class InfluxDbService: 8 | 9 | def __init__(self, config, logger): 10 | self.cnx = self.create_connection(config) 11 | self.logger = logger 12 | 13 | def create_connection(self, config): 14 | return InfluxDBClient( 15 | config['INFLUXDB_HOST'], 16 | 8086, 17 | config['INFLUXDB_USERNAME'], 18 | config['INFLUXDB_PASSWORD'], 19 | config['INFLUXDB_DATABASE']) 20 | 21 | @backoff.on_exception(backoff.expo, (InfluxDBServerError, InfluxDBClientError), max_tries=4) 22 | def query(self, query): 23 | with Timer(self.logger, f"Influx query {query}"): 24 | return self.cnx.query(query) 25 | -------------------------------------------------------------------------------- /dashboard/service/mysql.py: -------------------------------------------------------------------------------- 1 | import MySQLdb 2 | import MySQLdb.cursors 3 | import _mysql_exceptions 4 | import inspect 5 | import queue 6 | import backoff 7 | from dashboard.utils import Timer 8 | 9 | class MySQLClient: 10 | 11 | def __init__(self, config, logger, pool_size=3): 12 | self.config = config 13 | self.logger = logger 14 | self.pool = queue.Queue() 15 | for _ in range(pool_size): 16 | self.pool.put(self.create_connection()) 17 | 18 | def create_connection(self): 19 | conn = MySQLdb.connect( 20 | host=self.config['AIRFLOW_DB_HOST'], 21 | user=self.config['AIRFLOW_USERNAME'], 22 | password=self.config['AIRFLOW_PASSWORD'], 23 | db=self.config['AIRFLOW_DATABASE'], 24 | cursorclass=MySQLdb.cursors.DictCursor, 25 | connect_timeout=3 26 | ) 27 | conn.autocommit(True) 28 | return conn 29 | 30 | @backoff.on_exception(backoff.expo, _mysql_exceptions.OperationalError, max_tries=4) 31 | def query(self, query): 32 | cursor = None 33 | conn = self.pool.get(True) 34 | try: 35 | with Timer(self.logger, f"MySQL query {query}"): 36 | cursor = conn.cursor() 37 | cursor.execute(query) 38 | return cursor.fetchall() 39 | except _mysql_exceptions.OperationalError: 40 | conn = self.create_connection() # recreate connection 41 | raise 42 | finally: 43 | if cursor is not None: 44 | cursor.close() 45 | self.pool.put(conn) 46 | -------------------------------------------------------------------------------- /dashboard/service/prometheus_service.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import urllib.parse 3 | from dashboard.utils import Timer 4 | 5 | 6 | class PrometheusService: 7 | 8 | def __init__(self, config, logger): 9 | self.hostname = config['PROMETHEUS_HOST'] 10 | self.logger = logger 11 | 12 | def query(self, query): 13 | with Timer(self.logger, f"Prometheus query {query}"): 14 | response = requests.get(f'http://{self.hostname}/api/v1/query', params=dict(query=query)) 15 | response.raise_for_status() 16 | response_json = response.json() 17 | return response_json 18 | 19 | def alert_link(self, alert_name): 20 | query_params = urllib.parse.urlencode({'g0.expr': 'ALERTS{{alertname="{}"}}'.format(alert_name), 'g0.tab':'0'}) 21 | return f'http://{self.hostname}/graph?{query_params}' 22 | -------------------------------------------------------------------------------- /dashboard/static/css/base.css: -------------------------------------------------------------------------------- 1 | img { 2 | border: .5px dashed transparent; 3 | border-radius: 10%; 4 | } 5 | 6 | img:hover { 7 | border: .5px dashed black; 8 | border-radius: 10%; 9 | } 10 | 11 | #tooling_help { 12 | float: left; 13 | } 14 | 15 | #tooling { 16 | display: inline-flex; 17 | justify-content: space-between; 18 | flex-direction: row; 19 | float: left; 20 | } 21 | 22 | #help { 23 | float: left; 24 | } 25 | 26 | #help>div { 27 | display: inline-block; 28 | padding: .5rem; 29 | margin-right: 1rem; 30 | } 31 | 32 | #search { 33 | display: flex; 34 | justify-content: space-between; 35 | width: 12rem; 36 | background-color: #fff; 37 | margin-bottom: .5rem; 38 | border-radius: .3rem; 39 | border: 1px solid #999; 40 | } 41 | 42 | #search input { 43 | padding: .5rem; 44 | border: none; 45 | width: 85%; 46 | } 47 | 48 | #search input:focus { 49 | outline: none; 50 | } 51 | 52 | #search p { 53 | font-size: 1.5rem; 54 | margin: 0; 55 | padding: 0; 56 | align-self: flex-end; 57 | vertical-align: middle; 58 | } 59 | 60 | #clear-search-container { 61 | visibility: hidden; 62 | width: 20%; 63 | height: 100%; 64 | text-align: center; 65 | background-color: #5f7a7b; 66 | border-left: 1px dotted black; 67 | } 68 | 69 | #clear-search-container:hover { 70 | background-color: black; 71 | color: white; 72 | } 73 | 74 | #clear-search { 75 | cursor: default; 76 | } 77 | 78 | #list-container { 79 | clear: both; 80 | } 81 | 82 | #table-container { 83 | clear: both; 84 | } 85 | 86 | row { 87 | margin: 0 !important; 88 | padding: .5rem; 89 | } 90 | 91 | .table-header { 92 | background-color: #5f7a7b; 93 | color: white; 94 | font-weight: bolder; 95 | padding-bottom: 1rem; 96 | padding-top: 1rem; 97 | padding-left: .5rem; 98 | padding-right: .5rem; 99 | margin: 0 !important; 100 | } 101 | 102 | /* end of custom formatting */ 103 | 104 | .app-header { 105 | background-color: #fff; 106 | color: #1a1a1a; 107 | border-bottom: 1px solid black; 108 | } 109 | 110 | .app-header a { 111 | font-weight: bolder; 112 | } 113 | 114 | .app-header a:visited { 115 | color: #006661; 116 | } 117 | 118 | .app-header a:hover { 119 | color: #006661; 120 | } 121 | 122 | .app-header a:active { 123 | color: #006661; 124 | } 125 | 126 | .app-header a:link { 127 | color: #006661; 128 | } 129 | 130 | .app-header ul.navbar-nav { 131 | flex-direction: row; 132 | } 133 | 134 | .breadcrumb { 135 | background-color: #fff; 136 | color: #006661; 137 | } 138 | 139 | .breadcrumb a:visited { 140 | color: #006661; 141 | } 142 | 143 | .breadcrumb a:hover { 144 | color: #006661; 145 | } 146 | 147 | .breadcrumb a:active { 148 | color: #006661; 149 | } 150 | 151 | .breadcrumb a:link { 152 | color: #006661; 153 | } 154 | 155 | .container-fluid { 156 | background-color: #fff; 157 | margin-bottom: 1.5rem; 158 | } 159 | 160 | .main { 161 | background-color: #fff; 162 | } 163 | 164 | .etl-success { 165 | background-color: #5df2ae; 166 | color: #002a32; 167 | } 168 | 169 | .etl-success * { 170 | background-color: #5df2ae; 171 | color: #002a32; 172 | } 173 | 174 | .etl-issue { 175 | background-color: #dfec24; 176 | color: #002a32; 177 | } 178 | 179 | .etl-issue * { 180 | background-color: #dfec24; 181 | color: #002a32; 182 | } 183 | 184 | .etl-danger { 185 | background-color: #ff6a64; 186 | } 187 | 188 | .breadcrumb { 189 | border-bottom: 0; 190 | } 191 | 192 | .breadcrumb-container { 193 | margin-bottom: 1rem; 194 | width: 100%; 195 | display: inline-block; 196 | border-bottom: .5px dashed black; 197 | } 198 | 199 | .breadcrumb-container ol { 200 | margin-bottom: 0; 201 | display: inline-block; 202 | } 203 | 204 | .breadcrumb-container ol li { 205 | display: inline-block; 206 | } 207 | 208 | .app-help { 209 | cursor: default; 210 | position: absolute; 211 | right: 1.5rem; 212 | margin-top: .3rem; 213 | display: inline-block; 214 | width: 2rem; 215 | height: 2rem; 216 | border: 1px solid black; 217 | border-radius: 50%; 218 | text-align: center; 219 | font-size: 1.2rem; 220 | font-weight: bold; 221 | } 222 | 223 | .app-help:hover { 224 | background-color: #002a32; 225 | color: #fff; 226 | } 227 | 228 | .help-modal { 229 | position: absolute; 230 | visibility: hidden; 231 | top: 6rem; 232 | right: 1.7rem; 233 | width: 30%; 234 | padding: .7rem; 235 | border: 1px solid black; 236 | border-radius: 1rem; 237 | background-color: white; 238 | } 239 | 240 | .status-box { 241 | display: inline-block; 242 | padding: .8rem; 243 | border-radius: 10% 244 | } 245 | 246 | @media (max-width:768px) { 247 | .container { 248 | max-width: 540px 249 | } 250 | 251 | .navbar-brand { 252 | top: auto !important; 253 | left: auto !important; 254 | margin: auto !important; 255 | position: relative !important; 256 | } 257 | 258 | .app-header { 259 | display: inline-block; 260 | height: 170px; 261 | } 262 | 263 | .app-body { 264 | margin-top: 100px; 265 | } 266 | 267 | .navbar-nav { 268 | display: block; 269 | margin: auto; 270 | width: 100hv; 271 | } 272 | 273 | .navbar-nav li { 274 | margin: 5px 0 !important; 275 | border-top: 1px solid black; 276 | } 277 | 278 | .loginrefresh { 279 | position: absolute; 280 | top: 2px; 281 | right: 10px; 282 | width: 60px; 283 | } 284 | 285 | .refresh-info { 286 | display: none !important; 287 | } 288 | } 289 | 290 | @media (min-width:768px) and (max-width:992px) { 291 | .container { 292 | max-width: 760px 293 | } 294 | 295 | .navbar-brand { 296 | top: auto !important; 297 | left: auto !important; 298 | margin: auto !important; 299 | position: relative !important; 300 | } 301 | 302 | .app-header { 303 | display: inline-block; 304 | height: 95px; 305 | } 306 | 307 | .navbar-nav { 308 | margin: auto; 309 | width: 340px; 310 | } 311 | 312 | .loginrefresh { 313 | position: absolute; 314 | top: 2px; 315 | right: 10px; 316 | width: 60px; 317 | } 318 | 319 | .refresh-info { 320 | display: none !important; 321 | } 322 | } 323 | -------------------------------------------------------------------------------- /dashboard/static/css/descriptions.css: -------------------------------------------------------------------------------- 1 | .table-data { 2 | margin-top: 1rem; 3 | margin-bottom: 1rem; 4 | margin-left: 0; 5 | margin-right: 0; 6 | background-color: #dee7e5; 7 | word-wrap: break-word; 8 | } 9 | 10 | #legend { 11 | vertical-align: middle; 12 | padding-left: 2rem; 13 | padding-top: .5rem; 14 | } 15 | 16 | .column-data:nth-child(odd) { 17 | background-color: #c7cfce; 18 | } 19 | 20 | .column-data:nth-child(even) { 21 | background-color: #ffffff; 22 | } 23 | 24 | .row { 25 | margin: 0 !important; 26 | padding: .5rem; 27 | clear: both; 28 | } 29 | 30 | .column-header-section { 31 | background-color: #c7cfce; 32 | padding: 1rem; 33 | } 34 | 35 | .column-data-section { 36 | display: none; 37 | padding-left: 3rem; 38 | padding-right: 3rem; 39 | padding-top: 1.5rem; 40 | padding-bottom: 1.5rem; 41 | } 42 | 43 | a.toggle:hover { 44 | text-decoration: none; 45 | } 46 | 47 | .table-columns-header { 48 | background-color: #5f7a7b; 49 | color: white; 50 | font-weight: bolder; 51 | } 52 | -------------------------------------------------------------------------------- /dashboard/static/css/etl_dashboard.css: -------------------------------------------------------------------------------- 1 | .status { 2 | width: 8rem; 3 | } 4 | 5 | .dot { 6 | font-size: 1.2em; 7 | line-height: .25em; 8 | } 9 | 10 | .etl-data { 11 | margin: 0 !important; 12 | padding: .5rem; 13 | } 14 | 15 | .etl-data div p { 16 | margin: 0 !important; 17 | padding-top: .5rem; 18 | } 19 | 20 | .etl-data:nth-child(odd) { 21 | background-color: #dee7e5; 22 | } 23 | 24 | .dot.success { 25 | color: #4dbd74; 26 | } 27 | 28 | .dot.failed { 29 | color: #f86c6b; 30 | } 31 | 32 | .dot.running { 33 | color: #ffc107; 34 | } -------------------------------------------------------------------------------- /dashboard/static/css/reports.css: -------------------------------------------------------------------------------- 1 | .panel-group { 2 | background-color: #ffffff; 3 | clear: both 4 | } 5 | 6 | .panel-heading { 7 | padding: .5rem; 8 | border-bottom: 1px solid black; 9 | } 10 | 11 | .panel-heading:last-of-type { 12 | border-bottom: none; 13 | } 14 | 15 | .panel-collapse { 16 | border-bottom: 1px solid black; 17 | height: 4rem; 18 | } 19 | 20 | .panel-collapse:last-of-type { 21 | border-bottom: none; 22 | } 23 | 24 | .dot.success { 25 | color: #4dbd74; 26 | } 27 | 28 | .dot.failed { 29 | color: #f86c6b; 30 | } 31 | 32 | .dot.running { 33 | color: #ffc107; 34 | } 35 | 36 | .dot.root { 37 | color: #000000; 38 | } -------------------------------------------------------------------------------- /dashboard/static/css/table_dashboard.css: -------------------------------------------------------------------------------- 1 | .table-data { 2 | margin: 0 !important; 3 | padding: .5rem; 4 | } 5 | 6 | .table-data:nth-child(odd) { 7 | background-color: #dee7e5; 8 | } 9 | 10 | .table-data div p { 11 | margin: 0 !important; 12 | padding-top: .5rem; 13 | } 14 | 15 | .last-update button { 16 | width: 14rem; 17 | } -------------------------------------------------------------------------------- /dashboard/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/static/favicon.ico -------------------------------------------------------------------------------- /dashboard/static/images/graph_icon.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /dashboard/static/images/ui_screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/static/images/ui_screen.png -------------------------------------------------------------------------------- /dashboard/static/js/colors.js: -------------------------------------------------------------------------------- 1 | /* global rgbToHex */ 2 | 3 | /** 4 | * -------------------------------------------------------------------------- 5 | * CoreUI Free Boostrap Admin Template (v2.0.0): colors.js 6 | * Licensed under MIT (https://coreui.io/license) 7 | * -------------------------------------------------------------------------- 8 | */ 9 | $('.theme-color').each(function () { 10 | var Color = $(this).css('backgroundColor'); 11 | $(this).parent().append("\n \n \n \n \n \n \n \n \n \n
    HEX:" + rgbToHex(Color) + "
    RGB:" + Color + "
    \n "); 12 | }); 13 | //# sourceMappingURL=colors.js.map -------------------------------------------------------------------------------- /dashboard/static/js/colors.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"sources":["src/colors.js"],"names":["$","each","Color","css","parent","append","rgbToHex"],"mappings":"AAAA;;AAGA;;;;;;AAOAA,EAAE,cAAF,EAAkBC,IAAlB,CAAuB,YAAY;AACjC,MAAMC,QAAQF,EAAE,IAAF,EAAQG,GAAR,CAAY,iBAAZ,CAAd;AACAH,IAAE,IAAF,EAAQI,MAAR,GAAiBC,MAAjB,oIAIqCC,SAASJ,KAAT,CAJrC,2HAQqCA,KARrC;AAYD,CAdD","sourcesContent":["/* global rgbToHex */\nimport $ from 'jquery'\n\n/**\n * --------------------------------------------------------------------------\n * CoreUI Free Boostrap Admin Template (v2.0.0): colors.js\n * Licensed under MIT (https://coreui.io/license)\n * --------------------------------------------------------------------------\n */\n\n$('.theme-color').each(function () {\n const Color = $(this).css('backgroundColor')\n $(this).parent().append(`\n \n \n \n \n \n \n \n \n \n
    HEX:${rgbToHex(Color)}
    RGB:${Color}
    \n `)\n})\n"],"file":"colors.js"} -------------------------------------------------------------------------------- /dashboard/static/js/highlight.js: -------------------------------------------------------------------------------- 1 | const highlightTable = (tid) => { 2 | document.getElementById(tid).style.backgroundColor = "pink"; 3 | window.scrollBy(0, -100); 4 | }; 5 | 6 | const checkHighlight = () => { 7 | if(window.location.hash) 8 | { highlightTable(window.location.hash.substr(1));}}; -------------------------------------------------------------------------------- /dashboard/static/js/popovers.js: -------------------------------------------------------------------------------- 1 | /** 2 | * -------------------------------------------------------------------------- 3 | * CoreUI Free Boostrap Admin Template (v2.0.0): popovers.js 4 | * Licensed under MIT (https://coreui.io/license) 5 | * -------------------------------------------------------------------------- 6 | */ 7 | $('[data-toggle="popover"]').popover(); 8 | $('.popover-dismiss').popover({ 9 | trigger: 'focus' 10 | }); 11 | //# sourceMappingURL=popovers.js.map -------------------------------------------------------------------------------- /dashboard/static/js/popovers.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"sources":["src/popovers.js"],"names":["$","popover","trigger"],"mappings":"AAEA;;;;;;AAOAA,EAAE,yBAAF,EAA6BC,OAA7B;AACAD,EAAE,kBAAF,EAAsBC,OAAtB,CAA8B;AAC5BC,WAAS;AADmB,CAA9B","sourcesContent":["import $ from 'jquery'\n\n/**\n * --------------------------------------------------------------------------\n * CoreUI Free Boostrap Admin Template (v2.0.0): popovers.js\n * Licensed under MIT (https://coreui.io/license)\n * --------------------------------------------------------------------------\n */\n\n$('[data-toggle=\"popover\"]').popover()\n$('.popover-dismiss').popover({\n trigger: 'focus'\n})\n"],"file":"popovers.js"} -------------------------------------------------------------------------------- /dashboard/static/js/render_vis.js: -------------------------------------------------------------------------------- 1 | const render = (element, spec) => { 2 | new vega.View(vega.parse(spec)) 3 | .renderer('svg') // set renderer (canvas or svg) 4 | .initialize(element) // initialize view within parent DOM container 5 | .hover() 6 | .run(); 7 | } -------------------------------------------------------------------------------- /dashboard/static/js/search.js: -------------------------------------------------------------------------------- 1 | class SearchProvider { 2 | constructor(fieldsClass) { 3 | this.fields = document.querySelectorAll(fieldsClass); 4 | this.search = document.getElementById('search'); 5 | this.searchClear = document.getElementById('clear-search-container'); 6 | } 7 | 8 | filterList(nodes, search) { 9 | nodes.forEach(node => { 10 | node.style.display = node.textContent.toLowerCase().includes(search.toLowerCase()) ? "" : 11 | "none" 12 | }) 13 | } 14 | 15 | main() { 16 | this.search.addEventListener('keyup', event => { 17 | if (typeof this.onSearchPhraseChanged === "function") { 18 | this.onSearchPhraseChanged(); 19 | } 20 | if(event.target.value) { this.searchClear.style.visibility = "visible" } else { this.searchClear.style.visibility = "hidden" }; 21 | this.filterList(this.fields, event.target.value) 22 | }) 23 | 24 | this.searchClear.addEventListener("click", _ => { 25 | this.search.firstElementChild.value = ""; 26 | this.searchClear.style.visibility = "hidden"; 27 | this.filterList(this.fields, ""); 28 | }); 29 | } 30 | } -------------------------------------------------------------------------------- /dashboard/static/js/src/charts.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable object-curly-newline */ 2 | /* global Chart */ 3 | import $ from 'jquery' 4 | 5 | /** 6 | * -------------------------------------------------------------------------- 7 | * CoreUI Free Boostrap Admin Template (v2.0.0): main.js 8 | * Licensed under MIT (https://coreui.io/license) 9 | * -------------------------------------------------------------------------- 10 | */ 11 | 12 | /* eslint-disable no-magic-numbers */ 13 | // random Numbers 14 | const random = () => Math.round(Math.random() * 100) 15 | 16 | // eslint-disable-next-line no-unused-vars 17 | const lineChart = new Chart($('#canvas-1'), { 18 | type: 'line', 19 | data: { 20 | labels : ['January', 'February', 'March', 'April', 'May', 'June', 'July'], 21 | datasets : [ 22 | { 23 | label: 'My First dataset', 24 | backgroundColor : 'rgba(220, 220, 220, 0.2)', 25 | borderColor : 'rgba(220, 220, 220, 1)', 26 | pointBackgroundColor : 'rgba(220, 220, 220, 1)', 27 | pointBorderColor : '#fff', 28 | data : [random(), random(), random(), random(), random(), random(), random()] 29 | }, 30 | { 31 | label: 'My Second dataset', 32 | backgroundColor : 'rgba(151, 187, 205, 0.2)', 33 | borderColor : 'rgba(151, 187, 205, 1)', 34 | pointBackgroundColor : 'rgba(151, 187, 205, 1)', 35 | pointBorderColor : '#fff', 36 | data : [random(), random(), random(), random(), random(), random(), random()] 37 | } 38 | ] 39 | }, 40 | options: { 41 | responsive: true 42 | } 43 | }) 44 | 45 | // eslint-disable-next-line no-unused-vars 46 | const barChart = new Chart($('#canvas-2'), { 47 | type: 'bar', 48 | data: { 49 | labels : ['January', 'February', 'March', 'April', 'May', 'June', 'July'], 50 | datasets : [ 51 | { 52 | backgroundColor : 'rgba(220, 220, 220, 0.5)', 53 | borderColor : 'rgba(220, 220, 220, 0.8)', 54 | highlightFill: 'rgba(220, 220, 220, 0.75)', 55 | highlightStroke: 'rgba(220, 220, 220, 1)', 56 | data : [random(), random(), random(), random(), random(), random(), random()] 57 | }, 58 | { 59 | backgroundColor : 'rgba(151, 187, 205, 0.5)', 60 | borderColor : 'rgba(151, 187, 205, 0.8)', 61 | highlightFill : 'rgba(151, 187, 205, 0.75)', 62 | highlightStroke : 'rgba(151, 187, 205, 1)', 63 | data : [random(), random(), random(), random(), random(), random(), random()] 64 | } 65 | ] 66 | }, 67 | options: { 68 | responsive: true 69 | } 70 | }) 71 | 72 | // eslint-disable-next-line no-unused-vars 73 | const doughnutChart = new Chart($('#canvas-3'), { 74 | type: 'doughnut', 75 | data: { 76 | labels: ['Red', 'Green', 'Yellow'], 77 | datasets: [{ 78 | data: [300, 50, 100], 79 | backgroundColor: ['#FF6384', '#36A2EB', '#FFCE56'], 80 | hoverBackgroundColor: ['#FF6384', '#36A2EB', '#FFCE56'] 81 | }] 82 | }, 83 | options: { 84 | responsive: true 85 | } 86 | }) 87 | 88 | // eslint-disable-next-line no-unused-vars 89 | const radarChart = new Chart($('#canvas-4'), { 90 | type: 'radar', 91 | data: { 92 | labels: ['Eating', 'Drinking', 'Sleeping', 'Designing', 'Coding', 'Cycling', 'Running'], 93 | datasets: [ 94 | { 95 | label: 'My First dataset', 96 | backgroundColor: 'rgba(220, 220, 220, 0.2)', 97 | borderColor: 'rgba(220, 220, 220, 1)', 98 | pointBackgroundColor: 'rgba(220, 220, 220, 1)', 99 | pointBorderColor: '#fff', 100 | pointHighlightFill: '#fff', 101 | pointHighlightStroke: 'rgba(220, 220, 220, 1)', 102 | data: [65, 59, 90, 81, 56, 55, 40] 103 | }, 104 | { 105 | label: 'My Second dataset', 106 | backgroundColor: 'rgba(151, 187, 205, 0.2)', 107 | borderColor: 'rgba(151, 187, 205, 1)', 108 | pointBackgroundColor: 'rgba(151, 187, 205, 1)', 109 | pointBorderColor: '#fff', 110 | pointHighlightFill: '#fff', 111 | pointHighlightStroke: 'rgba(151, 187, 205, 1)', 112 | data: [28, 48, 40, 19, 96, 27, 100] 113 | } 114 | ] 115 | }, 116 | options: { 117 | responsive: true 118 | } 119 | }) 120 | 121 | // eslint-disable-next-line no-unused-vars 122 | const pieChart = new Chart($('#canvas-5'), { 123 | type: 'pie', 124 | data: { 125 | labels: ['Red', 'Green', 'Yellow'], 126 | datasets: [{ 127 | data: [300, 50, 100], 128 | backgroundColor: ['#FF6384', '#36A2EB', '#FFCE56'], 129 | hoverBackgroundColor: ['#FF6384', '#36A2EB', '#FFCE56'] 130 | }] 131 | }, 132 | options: { 133 | responsive: true 134 | } 135 | }) 136 | 137 | // eslint-disable-next-line no-unused-vars 138 | const polarAreaChart = new Chart($('#canvas-6'), { 139 | type: 'polarArea', 140 | data: { 141 | labels: ['Red', 'Green', 'Yellow', 'Grey', 'Blue'], 142 | datasets: [{ 143 | data: [11, 16, 7, 3, 14], 144 | backgroundColor: ['#FF6384', '#4BC0C0', '#FFCE56', '#E7E9ED', '#36A2EB'] 145 | }] 146 | }, 147 | options: { 148 | responsive: true 149 | } 150 | }) 151 | -------------------------------------------------------------------------------- /dashboard/static/js/src/colors.js: -------------------------------------------------------------------------------- 1 | /* global rgbToHex */ 2 | import $ from 'jquery' 3 | 4 | /** 5 | * -------------------------------------------------------------------------- 6 | * CoreUI Free Boostrap Admin Template (v2.0.0): colors.js 7 | * Licensed under MIT (https://coreui.io/license) 8 | * -------------------------------------------------------------------------- 9 | */ 10 | 11 | $('.theme-color').each(function () { 12 | const Color = $(this).css('backgroundColor') 13 | $(this).parent().append(` 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
    HEX:${rgbToHex(Color)}
    RGB:${Color}
    24 | `) 25 | }) 26 | -------------------------------------------------------------------------------- /dashboard/static/js/src/main.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable object-shorthand */ 2 | /* global Chart, CustomTooltips, getStyle, hexToRgba */ 3 | import $ from 'jquery' 4 | 5 | /** 6 | * -------------------------------------------------------------------------- 7 | * CoreUI Free Boostrap Admin Template (v2.0.0): main.js 8 | * Licensed under MIT (https://coreui.io/license) 9 | * -------------------------------------------------------------------------- 10 | */ 11 | 12 | /* eslint-disable no-magic-numbers */ 13 | // Disable the on-canvas tooltip 14 | Chart.defaults.global.pointHitDetectionRadius = 1 15 | Chart.defaults.global.tooltips.enabled = false 16 | Chart.defaults.global.tooltips.mode = 'index' 17 | Chart.defaults.global.tooltips.position = 'nearest' 18 | Chart.defaults.global.tooltips.custom = CustomTooltips 19 | 20 | // eslint-disable-next-line no-unused-vars 21 | const cardChart1 = new Chart($('#card-chart1'), { 22 | type: 'line', 23 | data: { 24 | labels: ['January', 'February', 'March', 'April', 'May', 'June', 'July'], 25 | datasets: [ 26 | { 27 | label: 'My First dataset', 28 | backgroundColor: getStyle('--primary'), 29 | borderColor: 'rgba(255,255,255,.55)', 30 | data: [65, 59, 84, 84, 51, 55, 40] 31 | } 32 | ] 33 | }, 34 | options: { 35 | maintainAspectRatio: false, 36 | legend: { 37 | display: false 38 | }, 39 | scales: { 40 | xAxes: [{ 41 | gridLines: { 42 | color: 'transparent', 43 | zeroLineColor: 'transparent' 44 | }, 45 | ticks: { 46 | fontSize: 2, 47 | fontColor: 'transparent' 48 | } 49 | }], 50 | yAxes: [{ 51 | display: false, 52 | ticks: { 53 | display: false, 54 | min: 35, 55 | max: 89 56 | } 57 | }] 58 | }, 59 | elements: { 60 | line: { 61 | borderWidth: 1 62 | }, 63 | point: { 64 | radius: 4, 65 | hitRadius: 10, 66 | hoverRadius: 4 67 | } 68 | } 69 | } 70 | }) 71 | 72 | // eslint-disable-next-line no-unused-vars 73 | const cardChart2 = new Chart($('#card-chart2'), { 74 | type: 'line', 75 | data: { 76 | labels: ['January', 'February', 'March', 'April', 'May', 'June', 'July'], 77 | datasets: [ 78 | { 79 | label: 'My First dataset', 80 | backgroundColor: getStyle('--info'), 81 | borderColor: 'rgba(255,255,255,.55)', 82 | data: [1, 18, 9, 17, 34, 22, 11] 83 | } 84 | ] 85 | }, 86 | options: { 87 | maintainAspectRatio: false, 88 | legend: { 89 | display: false 90 | }, 91 | scales: { 92 | xAxes: [{ 93 | gridLines: { 94 | color: 'transparent', 95 | zeroLineColor: 'transparent' 96 | }, 97 | ticks: { 98 | fontSize: 2, 99 | fontColor: 'transparent' 100 | } 101 | }], 102 | yAxes: [{ 103 | display: false, 104 | ticks: { 105 | display: false, 106 | min: -4, 107 | max: 39 108 | } 109 | }] 110 | }, 111 | elements: { 112 | line: { 113 | tension: 0.00001, 114 | borderWidth: 1 115 | }, 116 | point: { 117 | radius: 4, 118 | hitRadius: 10, 119 | hoverRadius: 4 120 | } 121 | } 122 | } 123 | }) 124 | 125 | // eslint-disable-next-line no-unused-vars 126 | const cardChart3 = new Chart($('#card-chart3'), { 127 | type: 'line', 128 | data: { 129 | labels: ['January', 'February', 'March', 'April', 'May', 'June', 'July'], 130 | datasets: [ 131 | { 132 | label: 'My First dataset', 133 | backgroundColor: 'rgba(255,255,255,.2)', 134 | borderColor: 'rgba(255,255,255,.55)', 135 | data: [78, 81, 80, 45, 34, 12, 40] 136 | } 137 | ] 138 | }, 139 | options: { 140 | maintainAspectRatio: false, 141 | legend: { 142 | display: false 143 | }, 144 | scales: { 145 | xAxes: [{ 146 | display: false 147 | }], 148 | yAxes: [{ 149 | display: false 150 | }] 151 | }, 152 | elements: { 153 | line: { 154 | borderWidth: 2 155 | }, 156 | point: { 157 | radius: 0, 158 | hitRadius: 10, 159 | hoverRadius: 4 160 | } 161 | } 162 | } 163 | }) 164 | 165 | // eslint-disable-next-line no-unused-vars 166 | const cardChart4 = new Chart($('#card-chart4'), { 167 | type: 'bar', 168 | data: { 169 | labels: ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December', 'January', 'February', 'March', 'April'], 170 | datasets: [ 171 | { 172 | label: 'My First dataset', 173 | backgroundColor: 'rgba(255,255,255,.2)', 174 | borderColor: 'rgba(255,255,255,.55)', 175 | data: [78, 81, 80, 45, 34, 12, 40, 85, 65, 23, 12, 98, 34, 84, 67, 82] 176 | } 177 | ] 178 | }, 179 | options: { 180 | maintainAspectRatio: false, 181 | legend: { 182 | display: false 183 | }, 184 | scales: { 185 | xAxes: [{ 186 | display: false, 187 | barPercentage: 0.6 188 | }], 189 | yAxes: [{ 190 | display: false 191 | }] 192 | } 193 | } 194 | }) 195 | 196 | // eslint-disable-next-line no-unused-vars 197 | const mainChart = new Chart($('#main-chart'), { 198 | type: 'line', 199 | data: { 200 | labels: ['M', 'T', 'W', 'T', 'F', 'S', 'S', 'M', 'T', 'W', 'T', 'F', 'S', 'S', 'M', 'T', 'W', 'T', 'F', 'S', 'S', 'M', 'T', 'W', 'T', 'F', 'S', 'S'], 201 | datasets: [ 202 | { 203 | label: 'My First dataset', 204 | backgroundColor: hexToRgba(getStyle('--info'), 10), 205 | borderColor: getStyle('--info'), 206 | pointHoverBackgroundColor: '#fff', 207 | borderWidth: 2, 208 | data: [165, 180, 70, 69, 77, 57, 125, 165, 172, 91, 173, 138, 155, 89, 50, 161, 65, 163, 160, 103, 114, 185, 125, 196, 183, 64, 137, 95, 112, 175] 209 | }, 210 | { 211 | label: 'My Second dataset', 212 | backgroundColor: 'transparent', 213 | borderColor: getStyle('--success'), 214 | pointHoverBackgroundColor: '#fff', 215 | borderWidth: 2, 216 | data: [92, 97, 80, 100, 86, 97, 83, 98, 87, 98, 93, 83, 87, 98, 96, 84, 91, 97, 88, 86, 94, 86, 95, 91, 98, 91, 92, 80, 83, 82] 217 | }, 218 | { 219 | label: 'My Third dataset', 220 | backgroundColor: 'transparent', 221 | borderColor: getStyle('--danger'), 222 | pointHoverBackgroundColor: '#fff', 223 | borderWidth: 1, 224 | borderDash: [8, 5], 225 | data: [65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65] 226 | } 227 | ] 228 | }, 229 | options: { 230 | maintainAspectRatio: false, 231 | legend: { 232 | display: false 233 | }, 234 | scales: { 235 | xAxes: [{ 236 | gridLines: { 237 | drawOnChartArea: false 238 | } 239 | }], 240 | yAxes: [{ 241 | ticks: { 242 | beginAtZero: true, 243 | maxTicksLimit: 5, 244 | stepSize: Math.ceil(250 / 5), 245 | max: 250 246 | } 247 | }] 248 | }, 249 | elements: { 250 | point: { 251 | radius: 0, 252 | hitRadius: 10, 253 | hoverRadius: 4, 254 | hoverBorderWidth: 3 255 | } 256 | } 257 | } 258 | }) 259 | 260 | const brandBoxChartLabels = ['January', 'February', 'March', 'April', 'May', 'June', 'July'] 261 | 262 | const brandBoxChartOptions = { 263 | responsive: true, 264 | maintainAspectRatio: false, 265 | legend: { 266 | display: false 267 | }, 268 | scales: { 269 | xAxes: [{ 270 | display:false 271 | }], 272 | yAxes: [{ 273 | display:false 274 | }] 275 | }, 276 | elements: { 277 | point: { 278 | radius: 0, 279 | hitRadius: 10, 280 | hoverRadius: 4, 281 | hoverBorderWidth: 3 282 | } 283 | } 284 | } 285 | 286 | // eslint-disable-next-line no-unused-vars 287 | const brandBoxChart1 = new Chart($('#social-box-chart-1'), { 288 | type: 'line', 289 | data: { 290 | labels: brandBoxChartLabels, 291 | datasets: [{ 292 | label: 'My First dataset', 293 | backgroundColor: 'rgba(255,255,255,.1)', 294 | borderColor: 'rgba(255,255,255,.55)', 295 | pointHoverBackgroundColor: '#fff', 296 | borderWidth: 2, 297 | data: [65, 59, 84, 84, 51, 55, 40] 298 | }] 299 | }, 300 | options: brandBoxChartOptions 301 | }) 302 | 303 | // eslint-disable-next-line no-unused-vars 304 | const brandBoxChart2 = new Chart($('#social-box-chart-2'), { 305 | type: 'line', 306 | data: { 307 | labels: brandBoxChartLabels, 308 | datasets: [{ 309 | label: 'My First dataset', 310 | backgroundColor: 'rgba(255,255,255,.1)', 311 | borderColor: 'rgba(255,255,255,.55)', 312 | pointHoverBackgroundColor: '#fff', 313 | borderWidth: 2, 314 | data: [1, 13, 9, 17, 34, 41, 38] 315 | }] 316 | }, 317 | options: brandBoxChartOptions 318 | }) 319 | 320 | // eslint-disable-next-line no-unused-vars 321 | const brandBoxChart3 = new Chart($('#social-box-chart-3'), { 322 | type: 'line', 323 | data: { 324 | labels: brandBoxChartLabels, 325 | datasets: [{ 326 | label: 'My First dataset', 327 | backgroundColor: 'rgba(255,255,255,.1)', 328 | borderColor: 'rgba(255,255,255,.55)', 329 | pointHoverBackgroundColor: '#fff', 330 | borderWidth: 2, 331 | data: [78, 81, 80, 45, 34, 12, 40] 332 | }] 333 | }, 334 | options: brandBoxChartOptions 335 | }) 336 | 337 | // eslint-disable-next-line no-unused-vars 338 | const brandBoxChart4 = new Chart($('#social-box-chart-4'), { 339 | type: 'line', 340 | data: { 341 | labels: brandBoxChartLabels, 342 | datasets: [{ 343 | label: 'My First dataset', 344 | backgroundColor: 'rgba(255,255,255,.1)', 345 | borderColor: 'rgba(255,255,255,.55)', 346 | pointHoverBackgroundColor: '#fff', 347 | borderWidth: 2, 348 | data: [35, 23, 56, 22, 97, 23, 64] 349 | }] 350 | }, 351 | options: brandBoxChartOptions 352 | }) 353 | -------------------------------------------------------------------------------- /dashboard/static/js/src/popovers.js: -------------------------------------------------------------------------------- 1 | import $ from 'jquery' 2 | 3 | /** 4 | * -------------------------------------------------------------------------- 5 | * CoreUI Free Boostrap Admin Template (v2.0.0): popovers.js 6 | * Licensed under MIT (https://coreui.io/license) 7 | * -------------------------------------------------------------------------- 8 | */ 9 | 10 | $('[data-toggle="popover"]').popover() 11 | $('.popover-dismiss').popover({ 12 | trigger: 'focus' 13 | }) 14 | -------------------------------------------------------------------------------- /dashboard/static/js/src/tooltips.js: -------------------------------------------------------------------------------- 1 | import $ from 'jquery' 2 | 3 | /** 4 | * -------------------------------------------------------------------------- 5 | * CoreUI Free Boostrap Admin Template (v2.0.0): tooltips.js 6 | * Licensed under MIT (https://coreui.io/license) 7 | * -------------------------------------------------------------------------- 8 | */ 9 | 10 | $('[data-toggle="tooltip"]').tooltip() 11 | -------------------------------------------------------------------------------- /dashboard/static/js/tooltips.js: -------------------------------------------------------------------------------- 1 | /** 2 | * -------------------------------------------------------------------------- 3 | * CoreUI Free Boostrap Admin Template (v2.0.0): tooltips.js 4 | * Licensed under MIT (https://coreui.io/license) 5 | * -------------------------------------------------------------------------- 6 | */ 7 | $('[data-toggle="tooltip"]').tooltip(); 8 | //# sourceMappingURL=tooltips.js.map -------------------------------------------------------------------------------- /dashboard/static/js/tooltips.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"sources":["src/tooltips.js"],"names":["$","tooltip"],"mappings":"AAEA;;;;;;AAOAA,EAAE,yBAAF,EAA6BC,OAA7B","sourcesContent":["import $ from 'jquery'\n\n/**\n * --------------------------------------------------------------------------\n * CoreUI Free Boostrap Admin Template (v2.0.0): tooltips.js\n * Licensed under MIT (https://coreui.io/license)\n * --------------------------------------------------------------------------\n */\n\n$('[data-toggle=\"tooltip\"]').tooltip()\n"],"file":"tooltips.js"} -------------------------------------------------------------------------------- /dashboard/static/vendors/@coreui/coreui-plugin-chartjs-custom-tooltips/js/custom-tooltips.min.js: -------------------------------------------------------------------------------- 1 | function CustomTooltips(s){var e,t,a=this,o="above",n="below",l="chartjs-tooltip",i="no-transform",c="tooltip-body",r="tooltip-body-item",d="tooltip-body-item-color",p="tooltip-body-item-label",m="tooltip-body-item-value",h="tooltip-header",u="tooltip-header-item",v={DIV:"div",SPAN:"span",TOOLTIP:(this._chart.canvas.id||(e=function(){return(65536*(1+Math.random())|0).toString(16)},t="_canvas-"+(e()+e()),a._chart.canvas.id=t))+"-tooltip"},y=document.getElementById(v.TOOLTIP);if(y||((y=document.createElement("div")).id=v.TOOLTIP,y.className=l,this._chart.canvas.parentNode.appendChild(y)),0!==s.opacity){if(y.classList.remove(o,n,i),s.yAlign?y.classList.add(s.yAlign):y.classList.add(i),s.body){var f=s.title||[],N=document.createElement(v.DIV);N.className=h,f.forEach(function(e){var t=document.createElement(v.DIV);t.className=u,t.innerHTML=e,N.appendChild(t)});var b=document.createElement(v.DIV);b.className=c,s.body.map(function(e){return e.lines}).forEach(function(e,t){var a=document.createElement(v.DIV);a.className=r;var o=s.labelColors[t],n=document.createElement(v.SPAN);if(n.className=d,n.style.backgroundColor=o.backgroundColor,a.appendChild(n),1 {\n const _idMaker = () => {\n const _hex = 16\n const _multiplier = 0x10000\n return ((1 + Math.random()) * _multiplier | 0).toString(_hex)\n }\n const _canvasId = `_canvas-${_idMaker() + _idMaker()}`\n this._chart.canvas.id = _canvasId\n return _canvasId\n }\n\n const ClassName = {\n ABOVE : 'above',\n BELOW : 'below',\n CHARTJS_TOOLTIP : 'chartjs-tooltip',\n NO_TRANSFORM : 'no-transform',\n TOOLTIP_BODY : 'tooltip-body',\n TOOLTIP_BODY_ITEM : 'tooltip-body-item',\n TOOLTIP_BODY_ITEM_COLOR : 'tooltip-body-item-color',\n TOOLTIP_BODY_ITEM_LABEL : 'tooltip-body-item-label',\n TOOLTIP_BODY_ITEM_VALUE : 'tooltip-body-item-value',\n TOOLTIP_HEADER : 'tooltip-header',\n TOOLTIP_HEADER_ITEM : 'tooltip-header-item'\n }\n\n const Selector = {\n DIV : 'div',\n SPAN : 'span',\n TOOLTIP : `${this._chart.canvas.id || _setCanvasId()}-tooltip`\n }\n\n let tooltip = document.getElementById(Selector.TOOLTIP)\n\n if (!tooltip) {\n tooltip = document.createElement('div')\n tooltip.id = Selector.TOOLTIP\n tooltip.className = ClassName.CHARTJS_TOOLTIP\n this._chart.canvas.parentNode.appendChild(tooltip)\n }\n\n // Hide if no tooltip\n if (tooltipModel.opacity === 0) {\n tooltip.style.opacity = 0\n return\n }\n\n // Set caret Position\n tooltip.classList.remove(ClassName.ABOVE, ClassName.BELOW, ClassName.NO_TRANSFORM)\n if (tooltipModel.yAlign) {\n tooltip.classList.add(tooltipModel.yAlign)\n } else {\n tooltip.classList.add(ClassName.NO_TRANSFORM)\n }\n\n // Set Text\n if (tooltipModel.body) {\n const titleLines = tooltipModel.title || []\n\n const tooltipHeader = document.createElement(Selector.DIV)\n tooltipHeader.className = ClassName.TOOLTIP_HEADER\n\n titleLines.forEach((title) => {\n const tooltipHeaderTitle = document.createElement(Selector.DIV)\n tooltipHeaderTitle.className = ClassName.TOOLTIP_HEADER_ITEM\n tooltipHeaderTitle.innerHTML = title\n tooltipHeader.appendChild(tooltipHeaderTitle)\n })\n\n const tooltipBody = document.createElement(Selector.DIV)\n tooltipBody.className = ClassName.TOOLTIP_BODY\n\n const tooltipBodyItems = tooltipModel.body.map((item) => item.lines)\n tooltipBodyItems.forEach((item, i) => {\n const tooltipBodyItem = document.createElement(Selector.DIV)\n tooltipBodyItem.className = ClassName.TOOLTIP_BODY_ITEM\n\n const colors = tooltipModel.labelColors[i]\n\n const tooltipBodyItemColor = document.createElement(Selector.SPAN)\n tooltipBodyItemColor.className = ClassName.TOOLTIP_BODY_ITEM_COLOR\n tooltipBodyItemColor.style.backgroundColor = colors.backgroundColor\n\n tooltipBodyItem.appendChild(tooltipBodyItemColor)\n\n if (item[0].split(':').length > 1) {\n const tooltipBodyItemLabel = document.createElement(Selector.SPAN)\n tooltipBodyItemLabel.className = ClassName.TOOLTIP_BODY_ITEM_LABEL\n tooltipBodyItemLabel.innerHTML = item[0].split(': ')[0]\n\n tooltipBodyItem.appendChild(tooltipBodyItemLabel)\n\n const tooltipBodyItemValue = document.createElement(Selector.SPAN)\n tooltipBodyItemValue.className = ClassName.TOOLTIP_BODY_ITEM_VALUE\n tooltipBodyItemValue.innerHTML = item[0].split(': ').pop()\n\n tooltipBodyItem.appendChild(tooltipBodyItemValue)\n } else {\n const tooltipBodyItemValue = document.createElement(Selector.SPAN)\n tooltipBodyItemValue.className = ClassName.TOOLTIP_BODY_ITEM_VALUE\n tooltipBodyItemValue.innerHTML = item[0]\n\n tooltipBodyItem.appendChild(tooltipBodyItemValue)\n }\n\n tooltipBody.appendChild(tooltipBodyItem)\n })\n\n tooltip.innerHTML = ''\n\n tooltip.appendChild(tooltipHeader)\n tooltip.appendChild(tooltipBody)\n }\n\n const positionY = this._chart.canvas.offsetTop\n const positionX = this._chart.canvas.offsetLeft\n\n // Display, position, and set styles for font\n tooltip.style.opacity = 1\n tooltip.style.left = `${positionX + tooltipModel.caretX}px`\n tooltip.style.top = `${positionY + tooltipModel.caretY}px`\n}\n\nexport default CustomTooltips\n"]} -------------------------------------------------------------------------------- /dashboard/templates/layouts/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | {% block autorefresh %}{% endblock %} 12 | 13 | {% block title %}{% endblock %} 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | {% block css %}{% endblock %} 33 | 34 | 35 | 36 |
    37 | 38 | CoreUI Logo 40 | CoreUI Logo 42 |   {{ config['COMPANY_NAME'] }} Data 43 | 44 | 45 |
    46 | 56 |
    57 | 58 |
      59 |
    • 60 | 61 | Page refreshed at: {{ now.strftime('%H:%M:%S') }} UTC 62 |
      • 63 |
    • 64 | {% if 'user' in session %} 65 | 68 |
      69 |
      70 | Settings 71 |
      72 | 73 | Logout 74 |
      75 | {% endif %} 76 |
      • 77 |
    78 |
    79 |
    80 |
    81 |
    82 | 83 |
      84 | {% block nav %}{% endblock %} 85 | 86 |
    1. 87 |
      2. 88 |
    89 |
    90 | ? 91 |
    92 |
    93 |
    94 | {% block body %}{% endblock %} 95 |
    96 |
    97 | 98 | 99 |
    100 |

    ETLs - information about the status of all ETL processes (DAGs) available in airflow database. If tables are defined this view shows number of processed tables. If tables information 101 | is missing, it shows number of tasks completed.

    102 |

    Tables - information available only it tables mapping is defined in a yaml file. It shows current status of processing related to a particular table.

    103 |

    Plugins - any information provided through plugins system available in discreETLy with appropriate tab name.

    104 |

    For more information contact the team responsible for maintaining the dashboard or search the documentation.

    105 |
    106 |
    107 |
    108 |
    109 | CoreUI 110 | © 2018 creativeLabs. 111 |
    112 |
    113 | Powered by 114 | CoreUI 115 |
    116 |
    117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | {% block charts %}{% endblock %} 129 | 134 | 142 | {% block js %}{% endblock %} 143 | 144 | 145 | 146 | -------------------------------------------------------------------------------- /dashboard/templates/layouts/clean.html: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | {% block title %}{% endblock %} 17 | 18 | 19 | 20 | 21 |
    22 |
    23 |
    24 | {% block body %}{% endblock %} 25 |
    26 |
    27 |
    28 | 29 | 30 | -------------------------------------------------------------------------------- /dashboard/templates/macros/common.html: -------------------------------------------------------------------------------- 1 | 2 | {% macro status_button_color(status) -%} 3 | {% if status == 'running' %}warning 4 | {% elif status == 'success' %}success 5 | {% elif status == 'failed' %}danger 6 | {% else %}secondary 7 | {% endif %} 8 | {%- endmacro %} 9 | -------------------------------------------------------------------------------- /dashboard/tests/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7.1-stretch 2 | 3 | WORKDIR /app 4 | 5 | RUN echo "deb http://ftp.debian.org/debian stretch-backports main libs contrib non-free" > /etc/apt/sources.list.d/backports.list 6 | RUN apt update && apt upgrade -y 7 | RUN apt-get -t stretch-backports -y install libsqlite3-0 8 | RUN pip install pytest 'moto==1.3.9' 9 | 10 | COPY requirements.txt requirements.txt 11 | 12 | RUN pip install -r requirements.txt 13 | 14 | COPY . . 15 | 16 | RUN mkdir /var/run/discreetly 17 | 18 | ENV AWS_ACCESS_KEY_ID=dummy 19 | ENV AWS_SECRET_ACCESS_KEY=dummy 20 | 21 | CMD ["pytest", "--disable-pytest-warnings"] 22 | -------------------------------------------------------------------------------- /dashboard/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/tests/__init__.py -------------------------------------------------------------------------------- /dashboard/tests/dataproviders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/tests/dataproviders/__init__.py -------------------------------------------------------------------------------- /dashboard/tests/dataproviders/test_airflow.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from dashboard.tests.mocks.database import MysqlService 4 | from dashboard.dataproviders.airflow import AirflowDBDataProvider 5 | from datetime import datetime 6 | 7 | logger = logging.Logger('Airflow.Test') 8 | 9 | config = { 10 | 'TECHNICAL_ETLS': set(['recover_partitions', 'jwplayer_import', 'operative_ftp_feed_2_s3', 'heartbeat']) 11 | } 12 | 13 | client = MysqlService() 14 | client.migrate('dashboard/tests/mocks/airflow.sql') 15 | 16 | airflow = AirflowDBDataProvider(config, logger, client) 17 | 18 | def test_dag_state(): 19 | assert airflow.get_dag_state('operative_data_import_v2.0', '2018-11-08 10:00:00.000') == 'success' 20 | assert airflow.get_dag_state('clickstream_view_special_v2.2', '2018-11-12 02:15:00.000') == 'failure' 21 | 22 | def test_get_history(): 23 | sample_dag = next(airflow.get_history(1)['clickstream_view_special']) 24 | assert sample_dag.dag_id == 'clickstream_view_special_v2.2' 25 | assert sample_dag.date == datetime(2018, 11, 12, 2, 15) 26 | assert sample_dag.state == 'failure' 27 | 28 | def test_should_retrieve_only_the_recent_version(): 29 | results = list(airflow.get_history(1)['multiple_versions']) 30 | assert len(results) == 1 31 | 32 | returned_object = results[0] 33 | assert returned_object.dag_id == 'multiple_versions_v2.3' 34 | assert returned_object.date == datetime(2018, 11, 12, 2, 15) 35 | assert returned_object.state == 'failure' 36 | 37 | def test_get_dag_tasks_success(): 38 | assert airflow.get_dag_tasks('clickstream_view_special_v2.2', '2018-11-12 02:15:00.000')[0].dag_id == 'clickstream_view_special_v2.2' 39 | 40 | def test_get_dag_tasks_failure_wrong_date(): 41 | assert airflow.get_dag_tasks('clickstream_view_special_v2.2', '2018-11-17 02:15:00.000') == [] 42 | 43 | def test_technical_dag_is_hidden(): 44 | status = airflow.get_dags_status() 45 | assert 'heartbeat' not in [dag['name'] for dag in status] 46 | task_instances = set([task.dag_name for task in airflow.get_newest_task_instances()]) 47 | assert 'heartbeat' not in task_instances 48 | 49 | def test_paused_dag_is_hidden(): 50 | status = airflow.get_dags_status() 51 | assert 'paused_dag' not in [dag['name'] for dag in status] 52 | task_instances = set([task.dag_name for task in airflow.get_newest_task_instances()]) 53 | assert 'paused_dag' not in task_instances 54 | 55 | def test_inactive_dag_is_hidden(): 56 | status = airflow.get_dags_status() 57 | assert 'inactive_dag' not in [dag['name'] for dag in status] 58 | task_instances = set([task.dag_name for task in airflow.get_newest_task_instances()]) 59 | assert 'inactive_dag' not in task_instances 60 | 61 | def test_get_last_success_for_manually_marked_task(): 62 | tasks = airflow.get_last_successful_tasks() 63 | succeeded_tasks_ids = [task.task_id for task in tasks] 64 | assert 'task_marked_manually' not in succeeded_tasks_ids 65 | -------------------------------------------------------------------------------- /dashboard/tests/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | mysql: 4 | image: 'mariadb' 5 | container_name: test_db 6 | environment: 7 | - MYSQL_ROOT_PASSWORD=test 8 | - MYSQL_DATABASE=pytest 9 | ports: 10 | - 3306:3306 11 | volumes: 12 | - type: tmpfs 13 | target: /var/lib/mysql 14 | healthcheck: 15 | test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] 16 | timeout: 20s 17 | retries: 10 18 | start_period: 10s 19 | tests: 20 | image: dashboard:tests 21 | environment: 22 | - TEST_DB_HOST=test_db 23 | depends_on: 24 | - mysql 25 | volumes: 26 | - type: tmpfs 27 | target: /var/run/discreetly -------------------------------------------------------------------------------- /dashboard/tests/mocks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikia/discreETLy/62dbea37dd54c47b06830e5f5662a4e219c04e06/dashboard/tests/mocks/__init__.py -------------------------------------------------------------------------------- /dashboard/tests/mocks/database.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | from dashboard.service.mysql import MySQLClient 3 | import logging 4 | import time 5 | import os 6 | 7 | class SqliteService: 8 | 9 | def __init__(self, location=':memory:'): 10 | self.conn: sqlite3.Connection = sqlite3.connect(location) 11 | self.conn.row_factory = sqlite3.Row 12 | 13 | @staticmethod 14 | def migrate(client: sqlite3.Connection, migration_script): 15 | cursor: sqlite3.Cursor = client.cursor() 16 | with open(migration_script, 'r') as script: 17 | cursor.executescript(script.read()) 18 | client.commit() 19 | cursor.close() 20 | 21 | def query(self, query: str): 22 | try: 23 | cursor = self.conn.cursor() 24 | cursor.execute(query.replace('\n', '')) 25 | self.conn.commit() 26 | return [dict(item) for item in cursor.fetchall()] 27 | finally: 28 | cursor.close() 29 | 30 | class MysqlService: 31 | 32 | def __init__(self): 33 | db_host = os.getenv('TEST_DB_HOST') 34 | configuration = { 35 | 'AIRFLOW_DB_HOST': db_host, 36 | 'AIRFLOW_USERNAME': 'root', 37 | 'AIRFLOW_PASSWORD': 'test', 38 | 'AIRFLOW_DATABASE': 'pytest' 39 | } 40 | 41 | logger = logging.getLogger('pytest') 42 | 43 | retries = 0 44 | connected = False 45 | last_error = None 46 | while not connected and retries < 10: 47 | try: 48 | self.db = MySQLClient(configuration, logger) 49 | connected = True 50 | except Exception as e: 51 | print("Waiting for database...") 52 | time.sleep(1) 53 | last_error = e 54 | retries = retries + 1 55 | 56 | if not connected: 57 | raise RuntimeError("Cannot connect to the test database") from last_error 58 | 59 | def migrate(self, migration_script): 60 | with open(migration_script, 'r') as script: 61 | self.db.query(script.read()) 62 | 63 | def query(self, query: str): 64 | return self.db.query(query) 65 | -------------------------------------------------------------------------------- /dashboard/utils/__init__.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | import yaml 4 | import time 5 | 6 | def clean_dag_id(dag_id): 7 | return re.sub('_v?[0-9]+.[0-9]+$', '', dag_id) 8 | 9 | 10 | def handle_resource(resource, value = None): 11 | def check_optional(func): 12 | def func_wrapper(self, *args, **kwargs): 13 | return func(self, *args, **kwargs) if self.__dict__.get(resource) else value 14 | return func_wrapper 15 | return check_optional 16 | 17 | def simple_state(state): 18 | if state in ('success', 'failed'): 19 | return state 20 | else: # simplifies statuses like "queued", "up_for_retry", etc 21 | return 'running' 22 | 23 | def load_data_provider(classname, params): 24 | try: 25 | parts = classname.split('.') 26 | module = __import__('.'.join(parts[:-1])) 27 | for comp in parts[1:]: 28 | module = getattr(module, comp) 29 | return module(**params) 30 | except Exception as e: 31 | raise Exception(f'Could not instantiate the provided TABLE_DESCRIPTION_SERVICE: {classname}', e) 32 | 33 | def get_yaml_file_content(path): 34 | config_file = path 35 | if os.path.exists(config_file): 36 | with open(config_file, 'r') as file: 37 | return yaml.load(file) 38 | else: 39 | return None 40 | 41 | # source: https://stackoverflow.com/a/1094933/7098262 42 | def sizeof_fmt(num, suffix='B'): 43 | for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: 44 | if abs(num) < 1024.0: 45 | return "%3.1f%s%s" % (num, unit, suffix) 46 | num /= 1024.0 47 | return "%.1f%s%s" % (num, 'Yi', suffix) 48 | 49 | 50 | class Timer: 51 | 52 | def __init__(self, logger, description): 53 | self.logger = logger 54 | self.description = description 55 | 56 | def __enter__(self): 57 | self.start = time.time() 58 | self.logger.debug(f"{self.description} started") 59 | return self 60 | 61 | def __exit__(self, *args): 62 | self.interval = time.time() - self.start 63 | self.logger.info(f"{self.description} took {self.interval:.2f} sec") -------------------------------------------------------------------------------- /dashboard/utils/vis.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | DEFAULT_COLORS = { 4 | 3: "#f86c6b", 5 | 2: "#ffc107", 6 | 1: "#4dbd74", 7 | 0: "black" 8 | } 9 | 10 | 11 | def tree_diagram( 12 | values, 13 | width=400, 14 | height=250, 15 | padding=5, 16 | circle_size=400, 17 | text_size=12, 18 | colors=None, 19 | provide_links = False): 20 | vis = { 21 | "$schema": "https://vega.github.io/schema/vega/v4.json", 22 | "width": width, 23 | "height": height, 24 | "padding": padding, 25 | "signals": [{ 26 | "name": "colors", 27 | "value": DEFAULT_COLORS 28 | }], 29 | "data": [{ 30 | "name": "tree", 31 | "values": values, 32 | "transform": [{ 33 | "type": "stratify", 34 | "key": "id", 35 | "parentKey": "parent" 36 | }, 37 | { 38 | "type": "tree", 39 | "method": "tidy", 40 | "size": [{ 41 | "signal": "height" 42 | }, { 43 | "signal": "width - 100" 44 | }], 45 | "as": ["y", "x", "depth", "children"] 46 | }, 47 | { 48 | "type": "formula", 49 | "as": "tlink", 50 | "expr": "\"tables#\" + datum.tid" 51 | }] 52 | }, 53 | { 54 | "name": "links", 55 | "source": "tree", 56 | "transform": [{ 57 | "type": "treelinks" 58 | }, 59 | { 60 | "type": "linkpath", 61 | "orient": "horizontal", 62 | "shape": "diagonal" 63 | } 64 | ] 65 | } 66 | ], 67 | "marks": [{ 68 | "type": "path", 69 | "from": { 70 | "data": "links" 71 | }, 72 | "encode": { 73 | "update": { 74 | "path": { 75 | "field": "path" 76 | }, 77 | "stroke": { 78 | "value": "#5f7a7b" 79 | }, 80 | "stroke-width": { 81 | "value": "60" 82 | } 83 | } 84 | } 85 | }, 86 | { 87 | "type": "symbol", 88 | "from": { 89 | "data": "tree" 90 | }, 91 | "encode": { 92 | "enter": { 93 | "size": { 94 | "value": circle_size 95 | }, 96 | "stroke": { 97 | "value": "#1a1a1a" 98 | } 99 | }, 100 | "update": { 101 | "x": { 102 | "field": "x" 103 | }, 104 | "y": { 105 | "field": "y" 106 | }, 107 | "fill": { 108 | "signal": "colors[datum.success]" 109 | # "scale": "color", "field": "success" 110 | } 111 | } 112 | } 113 | }, 114 | { 115 | "type": "text", 116 | "from": { 117 | "data": "tree" 118 | }, 119 | "encode": { 120 | "enter": { 121 | "text": { 122 | "field": "name" 123 | }, 124 | "fontSize": { 125 | "value": text_size 126 | }, 127 | "baseline": { 128 | "value": "bottom" 129 | } 130 | }, 131 | "update": { 132 | "x": { 133 | "field": "x" 134 | }, 135 | "y": { 136 | "field": "y" 137 | }, 138 | "dx": { 139 | "signal": "datum.children ? -12 : 12" 140 | }, 141 | "dy": { 142 | "signal": "datum.children ? 10 : -5" 143 | }, 144 | "align": { 145 | "signal": "datum.children ? 'right' : 'left'" 146 | }, 147 | "opacity": 1 148 | } 149 | } 150 | } 151 | ] 152 | } 153 | 154 | if provide_links: 155 | activate_links = {"href": { 156 | "signal":"datum.tlink" 157 | }} 158 | 159 | vis['marks'][1]['encode']['update'].update(activate_links) 160 | 161 | return vis 162 | -------------------------------------------------------------------------------- /examples/extra.html: -------------------------------------------------------------------------------- 1 | {% extends 'layouts/base.html' %} {% import 'macros/common.html' as common with context %} 2 | 3 | {% block title %}{{ config['EXTRA_TAB'] }}{% endblock %} 4 | 5 | {% block nav %} 6 | 7 |
  • Home
    • 8 |
  • {{ config['EXTRA_TAB'] }}
    • 9 | 10 | {% endblock %} 11 | 12 | {% block body %} 13 | 14 | 15 | 16 |
    17 |
    18 |
    19 |
    Streaming name
    20 |
    Streaming table
    21 |
    Status (✅ or ❌)
    22 |
    23 | {% for stream in extra %} 24 |
    25 |
    {{ stream.name }}
    26 |
    27 | {{ stream.table }} 28 |
    29 |
    30 | {% if stream.active_alerts %} 31 |

    ❌

    32 | {% else %} 33 |

    ✅

    34 | {% endif %} 35 |
    36 |
    37 | {% endfor %} 38 |
    39 |
    40 | 41 | {% endblock %} -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | werkzeug==0.16.* 2 | Flask==1.0.* 3 | Flask-SSLify 4 | gunicorn==19.9.* 5 | influxdb==5.2.* 6 | pyyaml>=4.2b1 7 | loginpass==0.3 8 | Authlib==0.12.1 9 | mysqlclient==1.3.12 10 | backoff 11 | requests==2.20.0 12 | boto3==1.9.67 13 | --------------------------------------------------------------------------------