├── doc ├── img │ ├── 06_chart.png │ ├── 07_chart.png │ ├── 08_sqllab.png │ ├── 01_database.png │ ├── 02_database.png │ ├── 03_dataset.png │ ├── 04_dataset.png │ ├── 05_dataset.png │ ├── 12_alpha_user.png │ ├── 09_virtual_table.png │ ├── 10_virtual_table.png │ ├── 13_visualize_data.png │ ├── 11_row_level_security.png │ ├── 15_time_range_custom.png │ └── 14_time_range_no_filter.png ├── 02_simple_bar_chart.md ├── 03_sqllab_virtual_table.md ├── 05_table_chart_time_filter.md ├── 04_row_level_security.md └── 01_setup_database_datasets.md ├── mongo-data ├── Dockerfile └── entrypoint.sh ├── superset ├── Dockerfile └── entrypoint.sh ├── sqlalchemy ├── Dockerfile └── script │ └── query.py ├── mongo-bi ├── Dockerfile ├── ssl │ ├── mongobi.crt │ ├── mongobi.key │ └── mongobi.pem └── entrypoint.sh ├── mysql-client └── Dockerfile ├── LICENSE ├── docker-compose.yaml └── README.md /doc/img/06_chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/06_chart.png -------------------------------------------------------------------------------- /doc/img/07_chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/07_chart.png -------------------------------------------------------------------------------- /doc/img/08_sqllab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/08_sqllab.png -------------------------------------------------------------------------------- /doc/img/01_database.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/01_database.png -------------------------------------------------------------------------------- /doc/img/02_database.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/02_database.png -------------------------------------------------------------------------------- /doc/img/03_dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/03_dataset.png -------------------------------------------------------------------------------- /doc/img/04_dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/04_dataset.png -------------------------------------------------------------------------------- /doc/img/05_dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/05_dataset.png -------------------------------------------------------------------------------- /doc/img/12_alpha_user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/12_alpha_user.png -------------------------------------------------------------------------------- /doc/img/09_virtual_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/09_virtual_table.png -------------------------------------------------------------------------------- /doc/img/10_virtual_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/10_virtual_table.png -------------------------------------------------------------------------------- /doc/img/13_visualize_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/13_visualize_data.png -------------------------------------------------------------------------------- /doc/img/11_row_level_security.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/11_row_level_security.png -------------------------------------------------------------------------------- /doc/img/15_time_range_custom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/15_time_range_custom.png -------------------------------------------------------------------------------- /doc/img/14_time_range_no_filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbouzekri/docker-superset-mongo-poc/HEAD/doc/img/14_time_range_no_filter.png -------------------------------------------------------------------------------- /mongo-data/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mongo:5.0.1 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | wget \ 5 | openssl \ 6 | libssl-dev \ 7 | zip \ 8 | && rm -rf /var/lib/apt/lists/* 9 | 10 | COPY entrypoint.sh / 11 | 12 | ENTRYPOINT ["/entrypoint.sh"] -------------------------------------------------------------------------------- /superset/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/superset:1.2.0 2 | 3 | USER root 4 | 5 | RUN apt-get update && apt-get install -y \ 6 | vim \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | RUN pip install mysqlclient \ 10 | && pip install sqlalchemy-mongobi 11 | 12 | COPY entrypoint.sh / 13 | RUN chmod a+x /entrypoint.sh 14 | 15 | USER superset 16 | 17 | ENV LIBMYSQL_ENABLE_CLEARTEXT_PLUGIN=1 18 | 19 | ENTRYPOINT ["/entrypoint.sh"] -------------------------------------------------------------------------------- /doc/02_simple_bar_chart.md: -------------------------------------------------------------------------------- 1 | # Superset - 02 Create a simple bar chart 2 | 3 | Next we can create our first chart using the `companies` dataset. This is : 4 | 5 | * A Bar Chart 6 | * With the metric `count(*)` 7 | * 2 filters 8 | * `count(*) > 100` 9 | * `acquisition.acquired_year IS NOT NULL` 10 | * 1 serie (to group by) `acquisition.acquired_year` 11 | 12 | ![](./img/06_chart.png) 13 | 14 | The bar chart is customized to `sort bars` 15 | 16 | ![](./img/07_chart.png) 17 | 18 | This is the first proof of concept of our POC. We are able to create charts using a collection in a mongodb database. 19 | -------------------------------------------------------------------------------- /sqlalchemy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | wget \ 5 | libgssapi-krb5-2 \ 6 | rsyslog \ 7 | openssl \ 8 | libssl-dev \ 9 | python3 \ 10 | python3-dev \ 11 | build-essential \ 12 | libmysqlclient-dev \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | RUN wget https://bootstrap.pypa.io/pip/3.5/get-pip.py \ 16 | && python3 get-pip.py \ 17 | && rm get-pip.py 18 | 19 | RUN pip install \ 20 | sqlalchemy \ 21 | mysqlclient \ 22 | sqlalchemy-mongobi 23 | 24 | ENV LIBMYSQL_ENABLE_CLEARTEXT_PLUGIN=1 25 | 26 | WORKDIR /sqlalchemy 27 | -------------------------------------------------------------------------------- /mongo-bi/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | wget \ 5 | libgssapi-krb5-2 \ 6 | rsyslog \ 7 | openssl \ 8 | libssl-dev \ 9 | && rm -rf /var/lib/apt/lists/* 10 | 11 | RUN wget https://info-mongodb-com.s3.amazonaws.com/mongodb-bi/v2/mongodb-bi-linux-x86_64-ubuntu2004-v2.14.3.tgz \ 12 | && tar -xvzf mongodb-bi-linux-x86_64-ubuntu2004-v2.14.3.tgz \ 13 | && install -m755 mongodb-bi-linux-x86_64-ubuntu2004-v2.14.3/bin/mongo* /usr/local/bin/ \ 14 | && rm -rf mongodb-bi-linux-x86_64-ubuntu2004-v2.14.3 mongodb-bi-linux-x86_64-ubuntu2004-v2.14.3.tgz 15 | 16 | COPY entrypoint.sh / 17 | 18 | ENTRYPOINT ["/entrypoint.sh"] -------------------------------------------------------------------------------- /mysql-client/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mysql:5.7-debian 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | wget \ 5 | libgssapi-krb5-2 \ 6 | rsyslog \ 7 | openssl \ 8 | libssl-dev \ 9 | && rm -rf /var/lib/apt/lists/* 10 | 11 | RUN wget https://github.com/mongodb/mongosql-auth-c/releases/download/v1.3.0/mongosql-auth-linux-x86_64-ubuntu-1604-v1.3.0.tgz \ 12 | && tar xfz mongosql-auth-linux-x86_64-ubuntu-1604-v1.3.0.tgz \ 13 | && mv mongosql-auth-linux-x86_64-ubuntu-1604/lib/mongosql_auth.so /usr/lib/mysql/plugin/ \ 14 | && rm -rf mongosql-auth-linux-x86_64-ubuntu-1604 \ 15 | && rm mongosql-auth-linux-x86_64-ubuntu-1604-v1.3.0.tgz 16 | 17 | RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/o/openssl1.0/libssl1.0.0_1.0.2n-1ubuntu6_amd64.deb \ 18 | && dpkg -i libssl1.0.0_1.0.2n-1ubuntu6_amd64.deb \ 19 | && rm libssl1.0.0_1.0.2n-1ubuntu6_amd64.deb 20 | -------------------------------------------------------------------------------- /mongo-data/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env /bin/bash 2 | 3 | if [[ -z "${MONGO_DB_URI}" ]] ; then 4 | echo "missing MONGO_DB_URI environment variable" 5 | exit 1 6 | fi 7 | 8 | if [[] $(mongo ${MONGO_DB_URI} --eval 'db.getMongo().getDBNames().indexOf("samples")' --quiet) -gt 0 ]] ; then 9 | echo "samples database exist. Exiting ..." 10 | exit 0 11 | fi 12 | 13 | wget https://raw.githubusercontent.com/ozlerhakan/mongodb-json-files/master/datasets/companies.json 14 | mongoimport --drop -c companies --uri ${MONGO_DB_URI} companies.json 15 | rm companies.json 16 | 17 | wget https://github.com/ozlerhakan/mongodb-json-files/raw/master/datasets/tweets.zip 18 | unzip tweets.zip 19 | mongorestore --drop --uri ${MONGO_DB_URI} --nsInclude=samples.tweets dump/twitter/tweets.bson 20 | rm -rf dump tweets.zip 21 | mongo ${MONGO_DB_URI} --eval 'db.tweets.updateMany({}, [{ "$set": { "created_at": { "$toDate": "$created_at" } }}]);' -------------------------------------------------------------------------------- /sqlalchemy/script/query.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy 2 | import os 3 | import json 4 | 5 | mongobi_host = os.getenv('MONGO_BI_HOST') 6 | mongobi_username = os.getenv('MONGO_BI_USERNAME') 7 | mongobi_password = os.getenv('MONGO_BI_PASSWORD') 8 | mongobi_port = os.getenv('MONGO_BI_PORT') 9 | mongobi_db = os.getenv('MONGO_BI_DATABASE') 10 | 11 | mongobi_uri = "mongobi://{}:{}@{}:{}/{}".format(mongobi_username, mongobi_password, mongobi_host, mongobi_port, mongobi_db) 12 | 13 | engine = sqlalchemy.create_engine( 14 | #"mongobi://root?source=admin:root@poc_ssm_mongo_bi:3307/samples", 15 | mongobi_uri, 16 | connect_args={ 17 | "ssl": { 18 | "mode": "PREFERRED" 19 | }, 20 | }, 21 | ) 22 | 23 | with engine.connect() as conn: 24 | result = conn.execute("select _id, name from companies limit 2") 25 | print('------------') 26 | print(json.dumps([dict(zip(row.keys(), row)) for row in result], indent=2)) 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Jonathan Bouzekri 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /superset/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env /bin/bash 2 | 3 | INIT_DONE_FILE=/init.done 4 | 5 | superset fab create-admin \ 6 | --username $ADMIN_USERNAME \ 7 | --firstname Superset \ 8 | --lastname Admin \ 9 | --email $ADMIN_EMAIL \ 10 | --password $ADMIN_PASSWORD 11 | 12 | superset db upgrade 13 | 14 | superset init 15 | 16 | gunicorn_reload='' 17 | if [[ "$ENABLE_DEV_MODE" == "1" ]]; then 18 | gunicorn_reload='--reload' 19 | fi 20 | 21 | if ! grep -q '# SSM_MONBI' /app/superset/sql_lab.py; then 22 | sed -i -z 's/ # Commit the connection so CTA queries will create the table.\n conn.commit()/ # SSM_MONBI\n # Commit the connection so CTA queries will create the table.\n # conn.commit()/g' /app/superset/sql_lab.py 23 | fi 24 | 25 | gunicorn \ 26 | --bind "0.0.0.0:${SUPERSET_PORT}" \ 27 | --access-logfile '-' \ 28 | --error-logfile '-' \ 29 | --workers 1 \ 30 | --worker-class gthread \ 31 | --threads 20 \ 32 | --timeout 60 \ 33 | --limit-request-line 0 \ 34 | --limit-request-field_size 0 \ 35 | $gunicorn_reload \ 36 | "${FLASK_APP}" -------------------------------------------------------------------------------- /doc/03_sqllab_virtual_table.md: -------------------------------------------------------------------------------- 1 | # Superset - 03 SQLlab and virtual table 2 | 3 | Now let's start to create a chart based on a query built in the sql lab query editor. And to add a little bit more of a complexity, it will be a query with a join condition. 4 | 5 | Open the `SQL Lab / SQL Editor` and type the following query then `RUN` : 6 | 7 | ```sql 8 | SELECT companies._id, companies.name, companies_offices.`offices.city`, companies_offices.`offices.country_code` 9 | FROM companies 10 | LEFT JOIN companies_offices ON companies._id = companies_offices._id; 11 | ``` 12 | 13 | ![](./img/08_sqllab.png) 14 | 15 | Look at the table extract where we have office in multiple countries. 16 | 17 | Then click on `EXPLORE` and save this query as a new virtual table / virtual dataset named `test` 18 | 19 | ![](./img/09_virtual_table.png) 20 | 21 | Create a new chart using this query : 22 | 23 | * A table in query mode `RAW RECORDS` 24 | * With all the columns selected 25 | * and one filter `offices.country_code = 'USA` 26 | 27 | ![](./img/10_virtual_table.png) 28 | 29 | The table is filtered to only show companies and their offices located in USA. This demonstrate that we can create virtual table in superset and manipulate them in the dashboard and chart editors. 30 | -------------------------------------------------------------------------------- /doc/05_table_chart_time_filter.md: -------------------------------------------------------------------------------- 1 | # Superset - 05 Table chart with time filter 2 | 3 | Explore the dataset `tweets` created during [01 - Setup the database and datasets](./01_setup_database_datasets.md). We use a table chart with the following configuration : 4 | 5 | * Time column `created_at` 6 | * Time range `No filter` 7 | * In `RAW RECORDS` query mode 8 | * with the `_id`, `user.name`, `favorited` and `created_at` columns displayed 9 | 10 | ![](./img/14_time_range_no_filter.png) 11 | 12 | We can see that most of the tweets are around the date `2015-04-26` and there are 966 records being returned. 13 | 14 | IN the time filter area, we change the date range type to `custom` with a `Specific Date/Time` of `2015-04-26 00:00:00` as start date and `2025-04-26 23:59:59` as end date. 15 | 16 | We can now see that there are 200 records returned. We can conclude that time filters are working correctly. 17 | 18 | ![](./img/15_time_range_custom.png) 19 | 20 | *Note : the tweets dataset has a field named text. As we are using the auto discover schema of the mongo bi connector tool. It detects this field without renaming it. As text is a keyword in many SQL implementation, it can result in an error when querying it. We advise to define a custom definition for this field using the manual schema declaration feature of the mongo bi connector tool.* -------------------------------------------------------------------------------- /mongo-bi/ssl/mongobi.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDrTCCApWgAwIBAgIUdPCaKZY0WcfF897Br6wHY5e6N8EwDQYJKoZIhvcNAQEL 3 | BQAwZjELMAkGA1UEBhMCRlIxFjAUBgNVBAgMDUlsZS1EZS1GcmFuY2UxDjAMBgNV 4 | BAcMBVBhcmlzMRQwEgYDVQQKDAtKYm8gQ29tcGFueTEZMBcGA1UEAwwQcG9jX3Nz 5 | bV9tb25nb19iaTAeFw0yMTA4MDExMTMwMjFaFw0zMTA3MzAxMTMwMjFaMGYxCzAJ 6 | BgNVBAYTAkZSMRYwFAYDVQQIDA1JbGUtRGUtRnJhbmNlMQ4wDAYDVQQHDAVQYXJp 7 | czEUMBIGA1UECgwLSmJvIENvbXBhbnkxGTAXBgNVBAMMEHBvY19zc21fbW9uZ29f 8 | YmkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmvfzl1o8yWX+uYbwK 9 | /VN4WYs/z1vo44IwB0jm8ioGLZjuJOJzAzU9EOFfpulmQ3Wqu+GWGKC09c+hRioQ 10 | uk1w0LkLpqrmNTy7ABTndwIFbzOsq57zT/m5LTSvTzRSKLDpr2x4bAaDQgd2Fdtj 11 | EceoKPTzH7/Y+0SCToku+siYEl4dAPqXnuecBU6oXTabaC80u8YGJpg2UcVoqtbe 12 | gExE6B5QKZICOOgxc0ocLadglqifTnAcHIFKobVy+yjQlgboJrbrEw25zKQh31+F 13 | y1eJjh8OPb1Q5CC5OApqaxnAtxMKOVWvAWbW6FBXBJ3TPAysViqxE1t8hWCDsPZB 14 | 1+9nAgMBAAGjUzBRMB0GA1UdDgQWBBQ0ycmdbTJ5O5LnOHxLpVGvR5DUDDAfBgNV 15 | HSMEGDAWgBQ0ycmdbTJ5O5LnOHxLpVGvR5DUDDAPBgNVHRMBAf8EBTADAQH/MA0G 16 | CSqGSIb3DQEBCwUAA4IBAQBJSGaP2NeDuLleiJN/OsK9qKCuZ52AZR5MlX9FgkA0 17 | NVKUQAWfE8wExtV6TCFGhuSp4qPGg5LdvNHrEnzbK/5P+Hqx8cox76zdLGJXuBit 18 | IX5z1akE4mwV6kDIWaOHQ5THK65fVjVZhQ3SWzC79MRYJMos8Jbf2Os929tjwJsC 19 | jwSOOwNvxAeRbuN/en8flAkZ7ghudssther7NL6uk1stBXOGll/aB0NFPz4LhODl 20 | yjMGEWe5g9QNjEohE+0o3uCzdmTJsCJUcw5Uu4Ta8q20wd0lSszjssgS97P+4v07 21 | Uyao/jl/KYGjd/vLGaXof1b7MkhFZySv8P0n1Bx39GeL 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /doc/04_row_level_security.md: -------------------------------------------------------------------------------- 1 | # Superset - 04 Row level security 2 | 3 | This example uses the virtual table `test` created in [03 - SQLlab and virtual table](./03_sqllab_virtual_table.md). 4 | 5 | Click on `Settings` in the top right corner then `Row level security` and create a new policy. 6 | 7 | * *Filter type* : `Regular` 8 | * *Tables* : `test` 9 | * *Roles* : `alpha` 10 | * *Group key* : `` 11 | * *Clause* : `` `offices.country_code` = 'USA' `` 12 | 13 | ![](./img/11_row_level_security.png) 14 | 15 | Next go to user administration and create a new User with role `alpha` : 16 | 17 | * *First Name* : `Alpha` 18 | * *Last Name* : `Alpha` 19 | * *User Name* : `alpha` 20 | * *Is Active?* : `` 21 | * *Email* : `alpha@test.com` 22 | * *Role* : `Alpha` 23 | * *Password* : `alpha` 24 | 25 | ![](./img/12_alpha_user.png) 26 | 27 | Open a private browser and authenticate with the new `alpha` user. Then go to the dataset `test` and explore it. 28 | 29 | In this screenshot, it uses a table chart in agreggate mode grouped by the `offices.country_code` field exposing the metric `count(*)` without any filter. As you can see, only results with the country_code `USA` are being returned so we can prove that row level security is working correctly. To resume, this is : 30 | 31 | * A Table Chart 32 | * In `AGREGGATE` query mode 33 | * With the metric `count(*)` 34 | * Group by `offices.country_code` 35 | 36 | ![](./img/13_visualize_data.png) 37 | -------------------------------------------------------------------------------- /doc/01_setup_database_datasets.md: -------------------------------------------------------------------------------- 1 | # Superset - 01 Setup the database and datasets 2 | 3 | Go to [http://localhost:8088/](http://localhost:8088/) and connect using admin credentials `admin` / `admin` 4 | 5 | Create a new Database with the following configuration : 6 | 7 | * *BASIC / DISPLAY NAME* : `MongoBI` 8 | * *BASIC / SQLALCHEMY URI* : `mongobi://root?source=admin:root@poc_ssm_mongo_bi:3307/samples` 9 | 10 | ![](./img/01_database.png) 11 | 12 | * *ADVANCED / OTHER / EXTRA* : 13 | 14 | ```json 15 | { 16 | "metadata_params": {}, 17 | "engine_params": { 18 | "connect_args": { 19 | "ssl": { 20 | "mode": "PREFERRED" 21 | } 22 | } 23 | }, 24 | "metadata_cache_timeout": {}, 25 | "schemas_allowed_for_csv_upload": [] 26 | } 27 | ``` 28 | 29 | ![](./img/02_database.png) 30 | 31 | Create a new dataset for the table `companies` in the `samples` database of the previously created `MongoBI` superset database. 32 | 33 | * *DATASOURCE* : `MongoBI` 34 | * *SCHEMA* : `samples` 35 | * *TABLE* : `companies` 36 | 37 | ![](./img/03_dataset.png) 38 | 39 | Create another new dataset for the table `companies_office` in the `samples` database of the previously created `MongoBI` superset database. 40 | 41 | * *DATASOURCE* : `MongoBI` 42 | * *SCHEMA* : `samples` 43 | * *TABLE* : `companies_offices` 44 | 45 | ![](./img/04_dataset.png) 46 | 47 | Create another new dataset for the table `tweets` in the `samples` database of the previously created `MongoBI` superset database. 48 | 49 | * *DATASOURCE* : `MongoBI` 50 | * *SCHEMA* : `samples` 51 | * *TABLE* : `tweets` 52 | 53 | ![](./img/05_dataset.png) 54 | 55 | **Next we are going to create a [simple bar chart]() -------------------------------------------------------------------------------- /mongo-bi/ssl/mongobi.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmvfzl1o8yWX+u 3 | YbwK/VN4WYs/z1vo44IwB0jm8ioGLZjuJOJzAzU9EOFfpulmQ3Wqu+GWGKC09c+h 4 | RioQuk1w0LkLpqrmNTy7ABTndwIFbzOsq57zT/m5LTSvTzRSKLDpr2x4bAaDQgd2 5 | FdtjEceoKPTzH7/Y+0SCToku+siYEl4dAPqXnuecBU6oXTabaC80u8YGJpg2UcVo 6 | qtbegExE6B5QKZICOOgxc0ocLadglqifTnAcHIFKobVy+yjQlgboJrbrEw25zKQh 7 | 31+Fy1eJjh8OPb1Q5CC5OApqaxnAtxMKOVWvAWbW6FBXBJ3TPAysViqxE1t8hWCD 8 | sPZB1+9nAgMBAAECggEAT9DD3RUuqJBU4RlhlPUcXpCmal4NBdbZu7nQ+NPUr+5l 9 | tSEs0JcphdituUbBlLUX/yk+W8XgL582y2E1w5oadbVDzsLyLY4wSudIGmBKNGW9 10 | m5rvVFLlG/H6tSw5xFY8ETBqwN4c4/nYbnsZ8DtTkjhAwJxjDi+A5MtJOZbMLJZg 11 | Xuvr9Tk0Y/NJd8LJ/36eEfBwAeyc8T7T7wRI8667lcfgiC5cZJIVOgPfDGhFdCTD 12 | AfUUv2vW855etgyh5vGjK6plP0pIhUjFCRLfpemaHmr+4OAGq4dRsWBE2NL7oEMs 13 | MiHDtOcuwTYq2SWlmt80mq9k3npF4giaZiCjXmVoAQKBgQDUGTFA9bdS8D+qD9b3 14 | Y0CnYayUnrURCeV5WIvkhM+TW2zcVyyXGreGJ7NZCZCB77ODDZi1i43jO7WDUYXL 15 | RdZkB6pCQp/0LTcJywXiVHDvds79t/6Esn+SX022xxpwQlG5bEFwtMXQffw6je21 16 | w3+2wUC7o2FfbVQqzng9a6pDZwKBgQDJQW2hQZHZx7lbrqUpGByVNgyOv3siR4q7 17 | 5PeGJt6yInx8xJEP7yoE85oFEDyPlHcDE9OnxnwCNyBWjikO0Fo3ws9whfOFh/is 18 | /K+eynkQnjcRxJWkv2jtoF1J5Md7ZtrS0BkXn6c3t1KHwmCpiEAEIrirg9UOGXhp 19 | +CUNKGV0AQKBgENxC0dTmopl2GiUBs+9IKbYmQ7c9EhJtE0Eid8NGBAMJy5dDdGy 20 | bh7batzp/KhEdvk6Do/TozdZGht6haegOk6uXUYad1AmPHUBuIpCPXL3fVQR+H1r 21 | YMNDSKvQ3ahKfkAOmzRiVvA3z0czMB323xpWkuXXSrQUSIogAv3MyWQLAoGBAI9w 22 | uKh0SXXE6hG0Wjb93nFiwnSDeUmnX/Qnb1wIHDUuazeqChInTfWVgjMVrXRQRrrE 23 | yNgNmZkinUPTsuPCEGspyBvU/QoxLUZuGuW8PalgerOJP3bajfi/kX33N052soQK 24 | mbfCrLYDTSlBmw958tZ/wMtN/O49rRsqJ45HOngBAoGBAJmfXCWxSSJXtRTQIY1z 25 | Qs556rb1aMQZWjYF9oSDSu5qiVo2BYFHCiPEseamTNFTd9vMHtkY8zb/BU4LoLsg 26 | a99p0KC+acdz8PQ1IGMRqVD1uObTmzB4SxpELrUeDHnLVNZJzAEA+vHcsYVWSJdc 27 | lUwZUK7+UG1DtiwO/Lzfqtn+ 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /mongo-bi/ssl/mongobi.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDrTCCApWgAwIBAgIUdPCaKZY0WcfF897Br6wHY5e6N8EwDQYJKoZIhvcNAQEL 3 | BQAwZjELMAkGA1UEBhMCRlIxFjAUBgNVBAgMDUlsZS1EZS1GcmFuY2UxDjAMBgNV 4 | BAcMBVBhcmlzMRQwEgYDVQQKDAtKYm8gQ29tcGFueTEZMBcGA1UEAwwQcG9jX3Nz 5 | bV9tb25nb19iaTAeFw0yMTA4MDExMTMwMjFaFw0zMTA3MzAxMTMwMjFaMGYxCzAJ 6 | BgNVBAYTAkZSMRYwFAYDVQQIDA1JbGUtRGUtRnJhbmNlMQ4wDAYDVQQHDAVQYXJp 7 | czEUMBIGA1UECgwLSmJvIENvbXBhbnkxGTAXBgNVBAMMEHBvY19zc21fbW9uZ29f 8 | YmkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmvfzl1o8yWX+uYbwK 9 | /VN4WYs/z1vo44IwB0jm8ioGLZjuJOJzAzU9EOFfpulmQ3Wqu+GWGKC09c+hRioQ 10 | uk1w0LkLpqrmNTy7ABTndwIFbzOsq57zT/m5LTSvTzRSKLDpr2x4bAaDQgd2Fdtj 11 | EceoKPTzH7/Y+0SCToku+siYEl4dAPqXnuecBU6oXTabaC80u8YGJpg2UcVoqtbe 12 | gExE6B5QKZICOOgxc0ocLadglqifTnAcHIFKobVy+yjQlgboJrbrEw25zKQh31+F 13 | y1eJjh8OPb1Q5CC5OApqaxnAtxMKOVWvAWbW6FBXBJ3TPAysViqxE1t8hWCDsPZB 14 | 1+9nAgMBAAGjUzBRMB0GA1UdDgQWBBQ0ycmdbTJ5O5LnOHxLpVGvR5DUDDAfBgNV 15 | HSMEGDAWgBQ0ycmdbTJ5O5LnOHxLpVGvR5DUDDAPBgNVHRMBAf8EBTADAQH/MA0G 16 | CSqGSIb3DQEBCwUAA4IBAQBJSGaP2NeDuLleiJN/OsK9qKCuZ52AZR5MlX9FgkA0 17 | NVKUQAWfE8wExtV6TCFGhuSp4qPGg5LdvNHrEnzbK/5P+Hqx8cox76zdLGJXuBit 18 | IX5z1akE4mwV6kDIWaOHQ5THK65fVjVZhQ3SWzC79MRYJMos8Jbf2Os929tjwJsC 19 | jwSOOwNvxAeRbuN/en8flAkZ7ghudssther7NL6uk1stBXOGll/aB0NFPz4LhODl 20 | yjMGEWe5g9QNjEohE+0o3uCzdmTJsCJUcw5Uu4Ta8q20wd0lSszjssgS97P+4v07 21 | Uyao/jl/KYGjd/vLGaXof1b7MkhFZySv8P0n1Bx39GeL 22 | -----END CERTIFICATE----- 23 | -----BEGIN PRIVATE KEY----- 24 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmvfzl1o8yWX+u 25 | YbwK/VN4WYs/z1vo44IwB0jm8ioGLZjuJOJzAzU9EOFfpulmQ3Wqu+GWGKC09c+h 26 | RioQuk1w0LkLpqrmNTy7ABTndwIFbzOsq57zT/m5LTSvTzRSKLDpr2x4bAaDQgd2 27 | FdtjEceoKPTzH7/Y+0SCToku+siYEl4dAPqXnuecBU6oXTabaC80u8YGJpg2UcVo 28 | qtbegExE6B5QKZICOOgxc0ocLadglqifTnAcHIFKobVy+yjQlgboJrbrEw25zKQh 29 | 31+Fy1eJjh8OPb1Q5CC5OApqaxnAtxMKOVWvAWbW6FBXBJ3TPAysViqxE1t8hWCD 30 | sPZB1+9nAgMBAAECggEAT9DD3RUuqJBU4RlhlPUcXpCmal4NBdbZu7nQ+NPUr+5l 31 | tSEs0JcphdituUbBlLUX/yk+W8XgL582y2E1w5oadbVDzsLyLY4wSudIGmBKNGW9 32 | m5rvVFLlG/H6tSw5xFY8ETBqwN4c4/nYbnsZ8DtTkjhAwJxjDi+A5MtJOZbMLJZg 33 | Xuvr9Tk0Y/NJd8LJ/36eEfBwAeyc8T7T7wRI8667lcfgiC5cZJIVOgPfDGhFdCTD 34 | AfUUv2vW855etgyh5vGjK6plP0pIhUjFCRLfpemaHmr+4OAGq4dRsWBE2NL7oEMs 35 | MiHDtOcuwTYq2SWlmt80mq9k3npF4giaZiCjXmVoAQKBgQDUGTFA9bdS8D+qD9b3 36 | Y0CnYayUnrURCeV5WIvkhM+TW2zcVyyXGreGJ7NZCZCB77ODDZi1i43jO7WDUYXL 37 | RdZkB6pCQp/0LTcJywXiVHDvds79t/6Esn+SX022xxpwQlG5bEFwtMXQffw6je21 38 | w3+2wUC7o2FfbVQqzng9a6pDZwKBgQDJQW2hQZHZx7lbrqUpGByVNgyOv3siR4q7 39 | 5PeGJt6yInx8xJEP7yoE85oFEDyPlHcDE9OnxnwCNyBWjikO0Fo3ws9whfOFh/is 40 | /K+eynkQnjcRxJWkv2jtoF1J5Md7ZtrS0BkXn6c3t1KHwmCpiEAEIrirg9UOGXhp 41 | +CUNKGV0AQKBgENxC0dTmopl2GiUBs+9IKbYmQ7c9EhJtE0Eid8NGBAMJy5dDdGy 42 | bh7batzp/KhEdvk6Do/TozdZGht6haegOk6uXUYad1AmPHUBuIpCPXL3fVQR+H1r 43 | YMNDSKvQ3ahKfkAOmzRiVvA3z0czMB323xpWkuXXSrQUSIogAv3MyWQLAoGBAI9w 44 | uKh0SXXE6hG0Wjb93nFiwnSDeUmnX/Qnb1wIHDUuazeqChInTfWVgjMVrXRQRrrE 45 | yNgNmZkinUPTsuPCEGspyBvU/QoxLUZuGuW8PalgerOJP3bajfi/kX33N052soQK 46 | mbfCrLYDTSlBmw958tZ/wMtN/O49rRsqJ45HOngBAoGBAJmfXCWxSSJXtRTQIY1z 47 | Qs556rb1aMQZWjYF9oSDSu5qiVo2BYFHCiPEseamTNFTd9vMHtkY8zb/BU4LoLsg 48 | a99p0KC+acdz8PQ1IGMRqVD1uObTmzB4SxpELrUeDHnLVNZJzAEA+vHcsYVWSJdc 49 | lUwZUK7+UG1DtiwO/Lzfqtn+ 50 | -----END PRIVATE KEY----- 51 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.1' 2 | 3 | services: 4 | poc_ssm_mongo: 5 | image: mongo:5.0.1 6 | container_name: poc_ssm_mongo 7 | restart: always 8 | environment: 9 | MONGO_INITDB_ROOT_USERNAME: root 10 | MONGO_INITDB_ROOT_PASSWORD: root 11 | 12 | poc_ssm_mongo_express: 13 | image: mongo-express:1.0.0-alpha.4 14 | container_name: poc_ssm_mongo_express 15 | restart: always 16 | ports: 17 | - 8081:8081 18 | environment: 19 | ME_CONFIG_MONGODB_SERVER: poc_ssm_mongo 20 | ME_CONFIG_MONGODB_ADMINUSERNAME: root 21 | ME_CONFIG_MONGODB_ADMINPASSWORD: root 22 | depends_on: 23 | - poc_ssm_mongo 24 | 25 | poc_ssm_mongo_bi: 26 | image: poc-ssm-mongo-bi 27 | container_name: poc_ssm_mongo_bi 28 | build: 29 | context: ./mongo-bi 30 | #entrypoint: ['/bin/sh', '-c', 'sleep 100000'] 31 | ports: 32 | - 3307:3307 33 | environment: 34 | CONFIG_MONGO_BI_MONGO_USERNAME: root 35 | CONFIG_MONGO_BI_MONGO_PASSWORD: root 36 | CONFIG_MONGO_BI_AUTH: 1 37 | CONFIG_MONGO_BI_MONGO_URI: mongodb://poc_ssm_mongo:27017 38 | CONFIG_MONGO_BI_VERY_VERBOSE: 1 39 | CONFIG_MONGO_BI_SCHEMA_REFRESH_INTERVAL_SECS: 60 40 | CONFIG_MONGO_BI_SSL_MODE: allowSSL 41 | CONFIG_MONGO_BI_SSL_PEM_KEY_FILE: /ssl/mongobi.pem 42 | volumes: 43 | - ./mongo-bi/ssl:/ssl 44 | depends_on: 45 | - poc_ssm_mongo 46 | - poc_ssm_mongo_express # added to wait longer and be sure mongo is started 47 | 48 | poc_ssm_mongo_data: 49 | image: poc-ssm-mongo-data 50 | container_name: poc_ssm_mongo_data 51 | build: 52 | context: ./mongo-data 53 | restart: on-failure 54 | environment: 55 | MONGO_DB_URI: mongodb://root:root@poc_ssm_mongo:27017/samples?authSource=admin 56 | depends_on: 57 | - poc_ssm_mongo 58 | - poc_ssm_mongo_express # added to wait longer and be sure mongo is started 59 | 60 | poc_ssm_mysql_client: 61 | image: poc-ssm-mysql-client 62 | container_name: poc_ssm_mysql_client 63 | build: 64 | context: ./mysql-client 65 | entrypoint: ['/bin/sh', '-c', 'sleep 100000'] 66 | depends_on: 67 | - poc_ssm_mongo_data 68 | 69 | poc_ssm_sqlalchemy: 70 | image: poc-ssm-sqlalchemy 71 | container_name: poc_ssm_sqlalchemy 72 | build: 73 | context: ./sqlalchemy 74 | environment: 75 | MONGO_BI_HOST: poc_ssm_mongo_bi 76 | MONGO_BI_USERNAME: root?source=admin 77 | MONGO_BI_PASSWORD: root 78 | MONGO_BI_PORT: 3307 79 | MONGO_BI_DATABASE: samples 80 | entrypoint: ['/bin/sh', '-c', 'sleep 100000'] 81 | volumes: 82 | - ./sqlalchemy/script:/sqlalchemy 83 | depends_on: 84 | - poc_ssm_mongo_data 85 | 86 | poc_ssm_superset: 87 | image: poc-ssm-superset 88 | container_name: poc_ssm_superset 89 | build: 90 | context: ./superset 91 | environment: 92 | ENABLE_DEV_MODE: 1 93 | ADMIN_USERNAME: admin 94 | ADMIN_EMAIL: admin@dev.local 95 | ADMIN_PASSWORD: admin 96 | ports: 97 | - 8088:8088 98 | depends_on: 99 | - poc_ssm_mongo_data -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Superset Mongo via Mongo BI connector 2 | 3 | This is a simple [docker-compose](./docker-compose.yaml) project to demonstrate how to query a [MongoDB](https://www.mongodb.com/try/download/community) database with [Apache Superset](https://superset.apache.org/) through the [MongoDB BI Connector](https://docs.mongodb.com/bi-connector/v2.14/). 4 | 5 | I would like to thanks [@smarzola](https://github.com/smarzola/) for his [sqlalchemy-mongobi](https://github.com/smarzola/sqlalchemy-mongobi) python package which adds a dialect for sqlalchemy disabling transactions not supported with the BI connector. 6 | 7 | ## Description 8 | 9 | This docker compose project is made up of these containers : 10 | 11 | * `poc_ssm_mongo` : a mongo 5 single instance 12 | * `poc_ssm_mongo_express` : a mongo express UI to browse the data in the instance at [http://localhost:8081/](http://localhost:8081/) 13 | * `poc_ssm_mongo_bi` : a mongo bi connector connected to the mongo instance 14 | * `poc_ssm_mongo_data` : a container loading a sample dataset in the mongo instance 15 | * `poc_ssm_mysql_client` : a container with a mysql cli client to demonstrate connection through the mongo bi connector 16 | * `poc_ssm_sqlalchemy` : a container with a simple sqlalchemy script to demonstrate connection through the mongo bi connector 17 | * `poc_ssm_superset` : a container with a superset instance to demonstrate connection through the mongo bi connector 18 | 19 | ## Usage 20 | 21 | Starts the project 22 | 23 | ``` 24 | docker-compose up 25 | ``` 26 | 27 | Wait for everything to be ready and for the `poc_ssm_mongo_data` to exit with a 0 status code. 28 | 29 | *Note : look at the logs of the superset instance before connecting to wait for all the init scripts in the entrypoint to finish before the gunicorn web server is started* 30 | 31 | Then you can try the different connection mode. 32 | 33 | ### 1. mysql cli client 34 | 35 | ``` 36 | $ docker-compose exec poc_ssm_mysql_client bash 37 | 38 | # mysql -h poc_ssm_mongo_bi -P 3307 -u 'root?source=admin' --default-auth=mongosql_auth -proot 39 | mysql: [Warning] Using a password on the command line interface can be insecure. 40 | Welcome to the MySQL monitor. Commands end with ; or \g. 41 | Your MySQL connection id is 457 42 | Server version: 5.7.12 mongosqld v2.14.3 mongosqld v2.14.3 43 | 44 | Copyright (c) 2000, 2021, Oracle and/or its affiliates. 45 | 46 | Oracle is a registered trademark of Oracle Corporation and/or its 47 | affiliates. Other names may be trademarks of their respective 48 | owners. 49 | 50 | Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. 51 | 52 | mysql> show databases; 53 | +--------------------+ 54 | | Database | 55 | +--------------------+ 56 | | INFORMATION_SCHEMA | 57 | | mysql | 58 | | samples | 59 | +--------------------+ 60 | 3 rows in set (0.00 sec) 61 | 62 | mysql> use samples; 63 | Reading table information for completion of table and column names 64 | You can turn off this feature to get a quicker startup with -A 65 | 66 | Database changed 67 | 68 | mysql> show tables; 69 | +---------------------------------------+ 70 | | Tables_in_samples | 71 | +---------------------------------------+ 72 | | companies | 73 | | companies_acquisitions | 74 | | companies_competitions | 75 | | companies_external_links | 76 | | companies_funding_rounds | 77 | | companies_funding_rounds_investments | 78 | | companies_image_available_sizes | 79 | | companies_investments | 80 | | companies_milestones | 81 | | companies_offices | 82 | | companies_partners | 83 | | companies_products | 84 | | companies_providerships | 85 | | companies_relationships | 86 | | companies_screenshots | 87 | | companies_screenshots_available_sizes | 88 | | companies_video_embeds | 89 | +---------------------------------------+ 90 | 17 rows in set (0.00 sec) 91 | 92 | mysql> select * from companies limit 1\G 93 | *************************** 1. row *************************** 94 | _id: 52cdef7c4bab8bd675297d8b 95 | name: AdventNet 96 | 1 row in set (0.00 sec) 97 | 98 | mysql> 99 | ``` 100 | 101 | ### 2. sqlalchemy script 102 | 103 | ``` 104 | $ docker-compose exec poc_ssm_sqlalchemy bash 105 | 106 | # python3 query.py 107 | /usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py:410: SAWarning: Exception attempting to detect unicode returns: ProgrammingError('(MySQLdb._exceptions.ProgrammingError) (1064, "parse sql \'SELECT CAST(\'test collated returns\' AS CHAR CHARACTER SET utf8mb4) COLLATE utf8mb4_bin AS anon_1\' error: unexpected CHARACTER at position 55 near CHARACTER")',) 108 | "detect unicode returns: %r" % de 109 | ------------ 110 | [ 111 | { 112 | "name": "AdventNet", 113 | "_id": "52cdef7c4bab8bd675297d8b" 114 | }, 115 | { 116 | "name": "Wetpaint", 117 | "_id": "52cdef7c4bab8bd675297d8a" 118 | } 119 | ] 120 | ``` 121 | 122 | ### 3. Superset 123 | 124 | * [01 - Setup the database and datasets](./doc/01_setup_database_datasets.md) 125 | * [02 - Create a simple bar chart](./doc/02_simple_bar_chart.md) 126 | * [03 - SQLlab and virtual table](./doc/03_sqllab_virtual_table.md) 127 | * [04 - Row level security](./doc/04_row_level_security.md) 128 | * [05 - Table chart with time filter](./doc/05_table_chart_time_filter.md) 129 | 130 | ## Important 131 | 132 | You can look at the different Dockerfile and entrypoint.sh scripts to see the details. However there are a few things I did in this POC and I want to point out. 133 | 134 | 1. Mongo BI connector : 135 | 136 | 1. As the database is in auth mode, the bi connector needs to be too (`--auth`) and needs to be secured with TLS `--sslMode=allowSSL` and `--sslPEMKeyFile=/path/to/an/autosigned/pem/cert/and/key` 137 | 138 | 2. schema refresh has been enabled and is automatic but you can [build your own schema](https://docs.mongodb.com/bi-connector/v2.14/schema-configuration/) 139 | 140 | 2. Superset : 141 | 142 | 1. in file `/app/superset/sql_lab.py` around line `432`, there is a `conn.commit()` instruction that needs to be commented in order to be able to use the sql lab. 143 | 144 | 2. as the mysqlclient sends the password as cleartext (over tls), you need to add the `LIBMYSQL_ENABLE_CLEARTEXT_PLUGIN=1` environment variable -------------------------------------------------------------------------------- /mongo-bi/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env /bin/bash 2 | 3 | service rsyslog start 4 | 5 | VERBOSE="${VERBOSE:=1}" 6 | 7 | CONFIG_MONGO_BI_ADDR="${CONFIG_MONGO_BI_ADDR:=0.0.0.0:3307}" 8 | CONFIG_MONGO_BI_LOG_PATH="${CONFIG_MONGO_BI_LOG_PATH:=/dev/stderr}" 9 | CONFIG_MONGO_BI_LOG_APPEND="${CONFIG_MONGO_BI_LOG_APPEND:=1}" 10 | CONFIG_MONGO_BI_LOG_ROTATE="${CONFIG_MONGO_BI_LOG_ROTATE:=reopen}" 11 | 12 | # Based on https://docs.mongodb.com/bi-connector/current/reference/mongosqld/#command-line-options 13 | declare -A CLI_OPTS=( 14 | # Core Options 15 | ['CONFIG_MONGO_BI_ADDR']='--addr' 16 | ['CONFIG_MONGO_BI_VERSION']='flag:--version' 17 | ['CONFIG_MONGO_BI_CONFIG']='--config' 18 | ['CONFIG_MONGO_BI_MONGO_URI']='--mongo-uri' 19 | ['CONFIG_MONGO_BI_MONGO_VERSION_COMPATIBILITY']='--mongo-versionCompatibility' 20 | ['CONFIG_MONGO_BI_MAX_VARCHAR_LENGTH']='--maxVarcharLength' 21 | ['CONFIG_MONGO_BI_MONGO_USERNAME']='--mongo-username' 22 | ['CONFIG_MONGO_BI_MONGO_PASSWORD']='--mongo-password' 23 | ['CONFIG_MONGO_BI_MONGO_AUTHENTIFICATION_SOURCE']='--mongo-authenticationSource' 24 | ['CONFIG_MONGO_BI_MONGO_AUTHENTIFICATION_MECHANISM']='--mongo-authenticationMechanism' 25 | # Schema Options 26 | ['CONFIG_MONGO_BI_SCHEMA']='--schema' 27 | ['CONFIG_MONGO_BI_SCHEMA_DIRECTORY']='--schemaDirectory' 28 | ['multiple:CONFIG_MONGO_BI_SAMPLE_NAMESPACES']='--sampleNamespaces' 29 | ['CONFIG_MONGO_BI_SAMPLE_NAMESPACES']='--sampleNamespaces' 30 | ['CONFIG_MONGO_BI_SCHEMA_MODE']='--schemaMode' 31 | ['CONFIG_MONGO_BI_SCHEMA_SOURCE']='--schemaSource' 32 | ['CONFIG_MONGO_BI_SCHEMA_NAME']='--schemaName' 33 | ['CONFIG_MONGO_BI_SAMPLE_SIZE']='--sampleSize' 34 | ['CONFIG_MONGO_BI_SCHEMA_REFRESH_INTERVAL_SECS']='--schemaRefreshIntervalSecs' 35 | ['CONFIG_MONGO_BI_UUID_SUBTYPE3_ENCODING']='--uuidSubtype3Encoding' 36 | ['CONFIG_MONGO_BI_PREJOIN']='flag:--prejoin' 37 | # Log Options 38 | ['CONFIG_MONGO_BI_LOG_APPEND']='flag:--logAppend' 39 | ['CONFIG_MONGO_BI_LOG_PATH']='--logPath' 40 | ['CONFIG_MONGO_BI_LOG_ROTATE']='--logRotate' 41 | ['CONFIG_MONGO_BI_USAGE_LOG_INTERVAL']='--usageLogInterval' 42 | ['CONFIG_MONGO_BI_VERBOSE']='flag:--verbose' 43 | ['CONFIG_MONGO_BI_VERY_VERBOSE']='flag:-vv' 44 | # MongoDB TLS/SSL Options 45 | ['CONFIG_MONGO_BI_MONGO_SSL']='flag:--mongo-ssl' 46 | ['CONFIG_MONGO_BI_MONGO_SSL_PEM_KEY_FILE']='--mongo-sslPEMKeyFile' 47 | ['CONFIG_MONGO_BI_MONGO_SSL_PEM_KEY_PASSWORD']='--mongo-sslPEMKeyPassword' 48 | ['CONFIG_MONGO_BI_MONGO_SSL_ALLOW_INVALID_HOSTNAMES']='flag:--mongo-sslAllowInvalidHostnames' 49 | ['CONFIG_MONGO_BI_MONGO_SSL_ALLOW_INVALID_CERTIFICATES']='flag:--mongo-sslAllowInvalidCertificates' 50 | ['CONFIG_MONGO_BI_MONGO_SSL_CA_FILE']='--mongo-sslCAFile' 51 | ['CONFIG_MONGO_BI_MONGO_SSL_CRL_FILE']='--mongo-sslCRLFile' 52 | ['CONFIG_MONGO_BI_MONGO_SSL_FIPS_MODE']='flag:--mongo-sslFIPSMode' 53 | ['CONFIG_MONGO_BI_MONGO_MINIMUM_TLS_VERSION']='--mongo-minimumTLSVersion' 54 | # Client TLS/SSL Options 55 | ['CONFIG_MONGO_BI_SSL_MODE']='--sslMode' 56 | ['CONFIG_MONGO_BI_SSL_PEM_KEY_FILE']='--sslPEMKeyFile' 57 | ['CONFIG_MONGO_BI_SSL_PEM_KEY_PASSWORD']='--sslPEMKeyPassword' 58 | ['CONFIG_MONGO_BI_SSL_ALLOW_INVALID_HOSTNAMES']='flag:--sslAllowInvalidHostnames' 59 | ['CONFIG_MONGO_BI_SSL_ALLOW_INVALID_CERTIFICATES']='flag:--sslAllowInvalidCertificates' 60 | ['CONFIG_MONGO_BI_SSL_CA_FILE']='--sslCAFile' 61 | ['CONFIG_MONGO_BI_SSL_CRL_FILE']='--sslCRLFile' 62 | ['CONFIG_MONGO_BI_AUTH']='flag:--auth' 63 | ['CONFIG_MONGO_BI_DEFAULT_AUTH_SOURCE']='--defaultAuthSource' 64 | ['CONFIG_MONGO_BI_DEFAULT_AUTH_MECHANISM']='--defaultAuthMechanism' 65 | ['CONFIG_MONGO_BI_MINIMUM_TLS_VERSION']='--minimumTLSVersion' 66 | # Service Options 67 | ['CONFIG_MONGO_BI_SERVICE_NAME']='--serviceName' 68 | ['CONFIG_MONGO_BI_SERVICE_DISPLAY_NAME']='--serviceDisplayName' 69 | ['CONFIG_MONGO_BI_SERVICE_DESCRIPTION']='--serviceDescription' 70 | # Kerberos Options 71 | ['CONFIG_MONGO_BI_GSS_API_HOSTNAME']='--gssapiHostname' 72 | ['CONFIG_MONGO_BI_GSS_API_SERVICE_NAME']='--gssapiServiceName' 73 | ['CONFIG_MONGO_BI_MONGO_GSS_API_SERVICE_NAME']='--mongo-gssapiServiceName' 74 | ['CONFIG_MONGO_BI_GSS_API_CONSTRAINED_DELEGATION']='--gssapiConstrainedDelegation' 75 | # Socket Options 76 | ['CONFIG_MONGO_BI_FILE_PERMISSIONS']='--filePermissions' 77 | ['CONFIG_MONGO_BI_NO_UNIX_SOCKET']='flag:--noUnixSocket' 78 | ['CONFIG_MONGO_BI_UNIX_SOCKET_PREFIX']='--unixSocketPrefix' 79 | # Set Parameter Option 80 | ['CONFIG_MONGO_BI_SET_PARAMETER']='--setParameter' 81 | ) 82 | 83 | 84 | params=() 85 | 86 | function add_param() { 87 | # add_param 88 | # add a param to the global 89 | arg_name="$1" 90 | value="$2" 91 | if [[ $arg_name == flag:* ]] ; then 92 | new_arg=$(echo $arg_name | cut -c6-) 93 | else 94 | new_arg="${arg_name} ${value}" 95 | fi 96 | params+=($new_arg) 97 | } 98 | 99 | for env_name in "${!CLI_OPTS[@]}"; do 100 | 101 | # Handle case where we can have multiple params with the same name but different values 102 | # Example 103 | # CONFIG_MONGO_BI_SAMPLE_NAMESPACES_1=toto 104 | # CONFIG_MONGO_BI_SAMPLE_NAMESPACES_1=tata 105 | # will generate 106 | # mongosql --sampleNamespaces toto --sampleNamespaces tata 107 | if [[ $env_name == multiple:* ]] ; then 108 | arg_name="${CLI_OPTS[$env_name]}" 109 | original_env_name=$env_name 110 | 111 | # Override env_name to the version without multiple: prefix 112 | env_name=$(echo $env_name | cut -c10-) 113 | 114 | x=1 115 | multiple_env_name=${env_name}_${x} 116 | while [[ ! -z "${!multiple_env_name}" ]] 117 | do 118 | add_param ${CLI_OPTS[$original_env_name]} ${!multiple_env_name} 119 | [ $VERBOSE -eq 1 ] && echo "$multiple_env_name set, add ${CLI_OPTS[$original_env_name]} to cli" 120 | 121 | x=$(( $x + 1 )) 122 | multiple_env_name=${env_name}_${x} 123 | done 124 | 125 | continue 126 | fi 127 | 128 | # Handle simple clase of env var (without) 129 | if [ ! -z "${!env_name}" ] ; then 130 | add_param ${CLI_OPTS[$env_name]} ${!env_name} 131 | [ $VERBOSE -eq 1 ] && echo "$env_name set, add ${!env_name} to cli" 132 | else 133 | [ $VERBOSE -eq 1 ] && echo "$env_name not set or empty" 134 | fi 135 | done 136 | 137 | [ $VERBOSE -eq 1 ] && echo "-----------------" 138 | [ $VERBOSE -eq 1 ] && echo "mongosqld ${params[@]}" 139 | [ $VERBOSE -eq 1 ] && echo "..." 140 | 141 | mongosqld "${params[@]}" --------------------------------------------------------------------------------