├── gradle.properties ├── coordinator ├── log.properties ├── catalog │ ├── jmx.properties │ ├── tpcds.properties │ ├── memory.properties │ ├── tpch.properties │ └── hive.properties ├── node.properties ├── password-authenticator.properties ├── access-control.properties ├── event-listener.properties ├── jvm.config └── config.properties ├── worker ├── log.properties ├── catalog │ ├── jmx.properties │ ├── memory.properties │ ├── tpch.properties │ ├── tpcds.properties │ └── hive.properties ├── node.properties ├── jvm.config └── config.properties ├── postgres └── scripts │ └── init.sql ├── certs ├── trino.cer ├── trino.p12 ├── keystore.jks ├── truststore.jks ├── trino-proxy │ ├── keystore.p12 │ ├── truststore.jks │ ├── server.csr │ ├── server.crt │ ├── server.rsa.key │ ├── server.key │ └── bundle.crt ├── ca.crt ├── trino.crt └── trino.pem ├── docs ├── minio-ui.png ├── superset-query.png └── trino-superset-access.png ├── trino-ext-authz ├── src │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── io.trino.spi.Plugin │ │ └── java │ │ └── io │ │ └── trino │ │ └── plugin │ │ └── base │ │ └── security │ │ ├── StringFunctions.java │ │ ├── PlainPasswordAuthenticatorFactory.java │ │ ├── ExternalAuthsPlugin.java │ │ └── AuditLoggingEventListenerFactory.java ├── README.md └── build.gradle.kts ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── hive ├── Dockerfile2 ├── run.sh ├── output │ └── activity │ │ ├── version=2021-03-02 │ │ ├── part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet │ │ ├── part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet │ │ ├── part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet │ │ ├── part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet │ │ ├── part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet │ │ ├── part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet │ │ ├── part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet │ │ ├── part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet │ │ ├── .part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet.crc │ │ ├── .part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet.crc │ │ ├── .part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet.crc │ │ ├── .part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet.crc │ │ ├── .part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet.crc │ │ ├── .part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet.crc │ │ ├── .part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet.crc │ │ └── .part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet.crc │ │ ├── version=2021-03-03 │ │ ├── part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet │ │ ├── part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet │ │ ├── part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet │ │ ├── part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet │ │ ├── part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet │ │ ├── part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet │ │ ├── part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet │ │ ├── part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet │ │ ├── part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet │ │ ├── part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet │ │ ├── part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet │ │ ├── part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet │ │ ├── part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet │ │ ├── part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet │ │ ├── part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet │ │ ├── part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet │ │ ├── .part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet.crc │ │ ├── .part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet.crc │ │ ├── .part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet.crc │ │ ├── .part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet.crc │ │ ├── .part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet.crc │ │ ├── .part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet.crc │ │ ├── .part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet.crc │ │ ├── .part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet.crc │ │ ├── .part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet.crc │ │ ├── .part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet.crc │ │ ├── .part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet.crc │ │ ├── .part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet.crc │ │ ├── .part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet.crc │ │ ├── .part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet.crc │ │ ├── .part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet.crc │ │ └── .part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet.crc │ │ ├── version=2021-03-05 │ │ ├── part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet │ │ ├── part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet │ │ ├── part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet │ │ ├── part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet │ │ ├── part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet │ │ ├── part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet │ │ ├── part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet │ │ ├── part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet │ │ ├── part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet │ │ ├── part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet │ │ ├── part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet │ │ ├── part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet │ │ ├── part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet │ │ ├── part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet │ │ ├── part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet │ │ ├── part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet │ │ ├── part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet │ │ ├── part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet │ │ ├── .part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet.crc │ │ ├── .part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet.crc │ │ ├── .part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet.crc │ │ ├── .part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet.crc │ │ ├── .part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet.crc │ │ ├── .part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet.crc │ │ ├── .part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet.crc │ │ ├── .part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet.crc │ │ ├── .part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet.crc │ │ ├── .part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet.crc │ │ ├── .part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet.crc │ │ ├── .part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet.crc │ │ ├── .part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet.crc │ │ ├── .part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet.crc │ │ ├── .part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet.crc │ │ ├── .part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet.crc │ │ ├── .part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet.crc │ │ └── .part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet.crc │ │ ├── version=2021-03-08 │ │ ├── part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet │ │ ├── part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet │ │ ├── .part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet.crc │ │ └── .part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet.crc │ │ ├── version=2021-03-09 │ │ ├── part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet │ │ ├── part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet │ │ ├── part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet │ │ ├── .part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet.crc │ │ ├── .part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet.crc │ │ └── .part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet.crc │ │ ├── _delta_manifest │ │ ├── version=2021-03-02 │ │ │ └── manifest │ │ ├── version=2021-03-03 │ │ │ └── manifest │ │ ├── version=2021-03-05 │ │ │ └── manifest │ │ ├── version=2021-03-08 │ │ │ └── manifest │ │ └── version=2021-03-09 │ │ │ └── manifest │ │ ├── _symlink_format_manifest │ │ ├── version=2021-03-08 │ │ │ └── manifest │ │ ├── version=2021-03-09 │ │ │ └── manifest │ │ ├── version=2021-03-02 │ │ │ └── manifest │ │ ├── version=2021-03-03 │ │ │ └── manifest │ │ └── version=2021-03-05 │ │ │ └── manifest │ │ └── _delta_log │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000004.json │ │ ├── 00000000000000000003.json │ │ ├── 00000000000000000001.json │ │ └── 00000000000000000002.json ├── Dockerfile ├── sampledata.csv └── setup.sh ├── settings.gradle.kts ├── .gitignore ├── security ├── passwords │ └── password.db └── rules │ └── rules.json ├── Dockerfile ├── .env.template ├── env.sh ├── hive-authz ├── src │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── skhatri │ │ ├── s3 │ │ ├── AwsCustomCredentialsProvider.java │ │ ├── TrinoCustomAwsCredentialsProvider.java │ │ └── CredentialsFactory.java │ │ └── hive │ │ ├── HiveExtAuthenticator.java │ │ └── HiveExtAuthorizer.java └── build.gradle.kts ├── superset ├── Dockerfile └── docker-entrypoint.sh ├── curl-query.sh ├── Makefile ├── ca.sh ├── envoy └── trino-envoy.yaml ├── gradlew.bat ├── docker-compose.yaml ├── gradlew └── README.md /gradle.properties: -------------------------------------------------------------------------------- 1 | trinoVersion=446 2 | -------------------------------------------------------------------------------- /coordinator/log.properties: -------------------------------------------------------------------------------- 1 | io.trino=INFO -------------------------------------------------------------------------------- /worker/log.properties: -------------------------------------------------------------------------------- 1 | io.trino=INFO 2 | 3 | -------------------------------------------------------------------------------- /worker/catalog/jmx.properties: -------------------------------------------------------------------------------- 1 | connector.name=jmx 2 | 3 | -------------------------------------------------------------------------------- /coordinator/catalog/jmx.properties: -------------------------------------------------------------------------------- 1 | connector.name=jmx 2 | 3 | -------------------------------------------------------------------------------- /coordinator/catalog/tpcds.properties: -------------------------------------------------------------------------------- 1 | connector.name=tpcds 2 | -------------------------------------------------------------------------------- /worker/catalog/memory.properties: -------------------------------------------------------------------------------- 1 | connector.name=memory 2 | 3 | -------------------------------------------------------------------------------- /worker/catalog/tpch.properties: -------------------------------------------------------------------------------- 1 | connector.name=tpch 2 | 3 | -------------------------------------------------------------------------------- /coordinator/catalog/memory.properties: -------------------------------------------------------------------------------- 1 | connector.name=memory 2 | 3 | -------------------------------------------------------------------------------- /coordinator/catalog/tpch.properties: -------------------------------------------------------------------------------- 1 | connector.name=tpch 2 | 3 | -------------------------------------------------------------------------------- /postgres/scripts/init.sql: -------------------------------------------------------------------------------- 1 | create database hive; 2 | create database superset; 3 | -------------------------------------------------------------------------------- /worker/catalog/tpcds.properties: -------------------------------------------------------------------------------- 1 | connector.name=tpcds 2 | tpcds.splits-per-node=3 3 | -------------------------------------------------------------------------------- /certs/trino.cer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/certs/trino.cer -------------------------------------------------------------------------------- /certs/trino.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/certs/trino.p12 -------------------------------------------------------------------------------- /certs/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/certs/keystore.jks -------------------------------------------------------------------------------- /docs/minio-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/docs/minio-ui.png -------------------------------------------------------------------------------- /certs/truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/certs/truststore.jks -------------------------------------------------------------------------------- /docs/superset-query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/docs/superset-query.png -------------------------------------------------------------------------------- /certs/trino-proxy/keystore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/certs/trino-proxy/keystore.p12 -------------------------------------------------------------------------------- /docs/trino-superset-access.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/docs/trino-superset-access.png -------------------------------------------------------------------------------- /trino-ext-authz/src/main/resources/META-INF/services/io.trino.spi.Plugin: -------------------------------------------------------------------------------- 1 | io.trino.plugin.base.security.ExternalAuthsPlugin -------------------------------------------------------------------------------- /certs/trino-proxy/truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/certs/trino-proxy/truststore.jks -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /worker/node.properties: -------------------------------------------------------------------------------- 1 | node.environment=development 2 | node.data-dir=/data/trino 3 | node.id=bc9e4f2e-cc06-403b-91d6-c36c2b9a618a 4 | -------------------------------------------------------------------------------- /coordinator/node.properties: -------------------------------------------------------------------------------- 1 | node.environment=development 2 | node.data-dir=/data/trino 3 | node.id=89505ebc-1b33-4f96-adfc-235d6af158b0 4 | -------------------------------------------------------------------------------- /coordinator/password-authenticator.properties: -------------------------------------------------------------------------------- 1 | #file 2 | password-authenticator.name=plain-auth 3 | file.password-file=/opt/trino/security/passwords/password.db 4 | -------------------------------------------------------------------------------- /hive/Dockerfile2: -------------------------------------------------------------------------------- 1 | FROM skhatri/apache-hive:3.1.3 2 | ARG HIVE_BIN_VERSION="3.1.3" 3 | COPY hive-authz/build/libs/hive-authz.jar /opt/app/apache-hive-${HIVE_BIN_VERSION}-bin/lib/hive-authz.jar 4 | 5 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name="trino-demo" 2 | 3 | listOf("trino-ext-authz", "hive-authz").forEach { folder -> 4 | include(folder) 5 | project(":${folder}").projectDir = file(folder) 6 | } 7 | -------------------------------------------------------------------------------- /coordinator/access-control.properties: -------------------------------------------------------------------------------- 1 | access-control.name=extauthz 2 | #extauthz 3 | #extauthz is modified from file implementation 4 | security.config-file=/opt/trino/security/rules/rules.json 5 | security.refresh-period=120s 6 | -------------------------------------------------------------------------------- /hive/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | schematool -dbType postgres -validate 4 | if [[ $? -ne 0 ]]; 5 | then 6 | schematool -dbType postgres -initSchema 7 | fi; 8 | 9 | hiveserver2 --service metastore & 10 | hiveserver2 -------------------------------------------------------------------------------- /coordinator/event-listener.properties: -------------------------------------------------------------------------------- 1 | event-listener.name=auditlog 2 | trino.auditlog.logback.file=/tmp/logback.xml 3 | trino.auditlog.logback.level=INFO 4 | 5 | trino.auditlog.log.created=true 6 | trino.auditlog.log.completed=true 7 | trino.auditlog.log.split=true 8 | 9 | -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-08/part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-08/part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-08/part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-08/part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-09/part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-09/part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-09/part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-09/part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-09/part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-09/part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet -------------------------------------------------------------------------------- /trino-ext-authz/README.md: -------------------------------------------------------------------------------- 1 | ServiceLoader factories = ServiceLoader.load(SystemAccessControlFactory.class); 2 | for (SystemAccessControlFactory factory : factories) { 3 | addSystemAccessControlFactory(factory); 4 | } 5 | -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/.part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/.part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/.part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/.part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/.part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/.part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/.part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/.part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/.part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/.part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/.part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/.part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/.part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/.part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-02/.part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-02/.part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-03/.part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-03/.part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-05/.part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-05/.part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-08/.part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-08/.part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-08/.part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-08/.part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-09/.part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-09/.part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-09/.part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-09/.part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /hive/output/activity/version=2021-03-09/.part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skhatri/trino-by-example/HEAD/hive/output/activity/version=2021-03-09/.part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /trino 2 | #Secrets 3 | .env 4 | hive-site.xml 5 | local.yaml 6 | 7 | *tar.gz 8 | hive/data 9 | .tmp 10 | superset/data 11 | *.iml 12 | *.iws 13 | *.ipr 14 | ca 15 | build 16 | out 17 | lib 18 | .gradle 19 | .idea 20 | bin 21 | opa 22 | minio/data 23 | out.json 24 | 25 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_manifest/version=2021-03-02/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-02/part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet 3 | -------------------------------------------------------------------------------- /hive/output/activity/_symlink_format_manifest/version=2021-03-08/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-08/part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-08/part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /security/passwords/password.db: -------------------------------------------------------------------------------- 1 | user1:$2y$10$549IGvUMRT6EHcmzpE8HteRNHTrWx5rqu6sqkmlynXsaO94X/ER7. 2 | user2:$2y$10$tRGmmPVwXnfl1bEBUw.1.OYSZGgELdDv0iiAPXbvF0HBJtjKZI.yW 3 | user3:$2y$10$Vcu6ADWukCxhqIRkYtmdMOkqQhhva71LztQOM8lxJ4YndkY/mIJ9G 4 | skhatri:$2y$10$/flTMu5tnyzWoYpyMh.e3eAcq9teDRF6kcIqG9vNHHhti2bHanCRC 5 | admin:$2y$10$lPwxNpcWIMVyUwQcbY23Mug4YcL1k3BsYOfqSj2w1cQ8QDYLRAdaC 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG TRINO_VERSION 2 | FROM trinodb/trino:${TRINO_VERSION} 3 | 4 | 5 | RUN mkdir -p /usr/lib/trino/plugin/ext 6 | COPY trino-ext-authz/build/libs/trino-ext-authz.jar /usr/lib/trino/plugin/ext/trino-ext-authz.jar 7 | COPY trino-ext-authz/build/ext/*.jar /usr/lib/trino/plugin/ext 8 | 9 | COPY hive-authz/build/libs/hive-authz.jar /usr/lib/trino/plugin/hive/hdfs/hive-authz.jar 10 | 11 | 12 | -------------------------------------------------------------------------------- /hive/output/activity/_symlink_format_manifest/version=2021-03-09/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-09/part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-09/part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet 3 | file:/opt/data/output/activity/version=2021-03-09/part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet 4 | -------------------------------------------------------------------------------- /worker/jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xmx756M 3 | -XX:+UseG1GC 4 | -XX:G1HeapRegionSize=32M 5 | -XX:+ExplicitGCInvokesConcurrent 6 | -XX:+HeapDumpOnOutOfMemoryError 7 | -XX:+UseGCOverheadLimit 8 | -XX:+ExitOnOutOfMemoryError 9 | -XX:ReservedCodeCacheSize=256M 10 | -XX:PerMethodRecompilationCutoff=10000 11 | -XX:PerBytecodeRecompilationCutoff=10000 12 | -Djdk.attach.allowAttachSelf=true 13 | -Djdk.nio.maxCachedBufferSize=2000000 14 | 15 | -------------------------------------------------------------------------------- /coordinator/jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xmx756M 3 | -XX:+UseG1GC 4 | -XX:G1HeapRegionSize=32M 5 | -XX:+ExplicitGCInvokesConcurrent 6 | -XX:+HeapDumpOnOutOfMemoryError 7 | -XX:+UseGCOverheadLimit 8 | -XX:+ExitOnOutOfMemoryError 9 | -XX:ReservedCodeCacheSize=256M 10 | -XX:PerMethodRecompilationCutoff=10000 11 | -XX:PerBytecodeRecompilationCutoff=10000 12 | -Djdk.attach.allowAttachSelf=true 13 | -Djdk.nio.maxCachedBufferSize=2000000 14 | 15 | -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | #set this if using gateway s3 as the backend 2 | AWS_ACCESS_KEY= 3 | AWS_SECRET_ACCESS_KEY= 4 | POSTGRES_PASSWORD=password 5 | POSTGRES_URI=jdbc:postgresql://postgres/hive?ssl=false 6 | 7 | #set this to AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if not using minio 8 | STORE_KEY=minio 9 | STORE_SECRET=minio123 10 | #set this empty if not using minio 11 | S3_ENDPOINT=http://minio:9000 12 | #gateway s3 13 | #server /data 14 | S3_BACKEND=gateway s3 15 | REPO="docker.io/" 16 | 17 | -------------------------------------------------------------------------------- /env.sh: -------------------------------------------------------------------------------- 1 | cp hive/conf/hive-site-template.xml hive/conf/hive-site.xml 2 | while read line; 3 | do 4 | if [[ ! $line =~ ^# ]]; then 5 | read a b <<< $(echo $line | awk -F"=" '{print $1" "$2}') 6 | if [[ $a != "" ]]; 7 | then 8 | echo replacing \${env:$a} with $b 9 | sed s#"\${env:$a}"#"$b"#g hive/conf/hive-site.xml > hive/conf/.tmp 10 | cp hive/conf/.tmp hive/conf/hive-site.xml 11 | fi; 12 | fi; 13 | done < .env 14 | rm hive/conf/.tmp 15 | -------------------------------------------------------------------------------- /hive-authz/src/main/java/com/github/skhatri/s3/AwsCustomCredentialsProvider.java: -------------------------------------------------------------------------------- 1 | package com.github.skhatri.s3; 2 | 3 | import com.amazonaws.auth.AWSCredentials; 4 | import com.amazonaws.auth.AWSCredentialsProvider; 5 | import com.amazonaws.auth.BasicAWSCredentials; 6 | 7 | public class AwsCustomCredentialsProvider implements AWSCredentialsProvider { 8 | 9 | @Override 10 | public AWSCredentials getCredentials() { 11 | return CredentialsFactory.create(); 12 | } 13 | 14 | @Override 15 | public void refresh() { 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /worker/catalog/hive.properties: -------------------------------------------------------------------------------- 1 | connector.name=hive 2 | hive.metastore.uri=thrift://hive:9083 3 | hive.parquet.use-column-names=true 4 | #hive.s3.aws-access-key=${ENV:STORE_KEY} 5 | #hive.s3.aws-secret-key=${ENV:STORE_SECRET} 6 | hive.s3.endpoint=${ENV:S3_ENDPOINT} 7 | hive.s3.sse.enabled=true 8 | hive.s3.path-style-access=true 9 | 10 | #com.amazonaws.auth.STSAssumeRoleWithWebIdentitySessionCredentialsProvider 11 | #com.amazonaws.auth.DefaultAWSCredentialsProviderChain 12 | #trino.s3.credentials-provider=com.github.skhatri.s3.TrinoCustomAwsCredentialsProvider 13 | 14 | hive.config.resources=/etc/trino/s3.xml 15 | 16 | -------------------------------------------------------------------------------- /coordinator/catalog/hive.properties: -------------------------------------------------------------------------------- 1 | connector.name=hive 2 | hive.metastore.uri=thrift://hive:9083 3 | hive.parquet.use-column-names=true 4 | #hive.s3.aws-access-key=${ENV:STORE_KEY} 5 | #hive.s3.aws-secret-key=${ENV:STORE_SECRET} 6 | hive.s3.endpoint=${ENV:S3_ENDPOINT} 7 | hive.s3.sse.enabled=true 8 | hive.s3.path-style-access=true 9 | 10 | #com.amazonaws.auth.STSAssumeRoleWithWebIdentitySessionCredentialsProvider 11 | #com.amazonaws.auth.DefaultAWSCredentialsProviderChain 12 | #trino.s3.credentials-provider=com.github.skhatri.s3.TrinoCustomAwsCredentialsProvider 13 | 14 | hive.config.resources=/etc/trino/s3.xml 15 | 16 | -------------------------------------------------------------------------------- /hive-authz/src/main/java/com/github/skhatri/hive/HiveExtAuthenticator.java: -------------------------------------------------------------------------------- 1 | package com.github.skhatri.hive; 2 | 3 | import org.apache.hive.service.auth.PasswdAuthenticationProvider; 4 | 5 | import javax.security.sasl.AuthenticationException; 6 | 7 | //hive.server2.custom.authentication.class 8 | public class HiveExtAuthenticator implements PasswdAuthenticationProvider { 9 | @Override 10 | public void Authenticate(String user, String password) throws AuthenticationException { 11 | if (!"password".equals(password)) { 12 | throw new AuthenticationException("invalid password"); 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /superset/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/superset:latest-dev 2 | 3 | USER root 4 | RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org --upgrade pip setuptools 5 | RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org psycopg2 pyhive elasticsearch-dbapi sqlalchemy-trino 6 | 7 | RUN chown -R superset /usr/local/share/ca-certificates/ && \ 8 | chown -R superset /etc/ssl/certs && \ 9 | chown -R superset /usr/share/ca-certificates && \ 10 | chown -R superset /etc/ca-certificates.conf 11 | COPY docker-entrypoint.sh /usr/bin/docker-entrypoint.sh 12 | ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] 13 | USER superset 14 | 15 | -------------------------------------------------------------------------------- /hive-authz/build.gradle.kts: -------------------------------------------------------------------------------- 1 | 2 | 3 | java { 4 | sourceCompatibility = JavaVersion.VERSION_1_8 5 | targetCompatibility = JavaVersion.VERSION_1_8 6 | } 7 | 8 | configurations { 9 | implementation { 10 | resolutionStrategy.failOnVersionConflict() 11 | } 12 | } 13 | 14 | dependencies { 15 | listOf("hive-service").forEach { name -> 16 | implementation("org.apache.hive:${name}:3.1.3") 17 | } 18 | listOf("aws-java-sdk-sts", "aws-java-sdk-s3", "aws-java-sdk-bom").forEach { mod -> 19 | compileOnly("com.amazonaws:${mod}:1.11.860") 20 | } 21 | listOf("trino-spi", "trino-plugin-toolkit", "trino-hive").forEach { name -> 22 | compileOnly("io.trino:${name}:455") 23 | } 24 | compileOnly("org.slf4j:slf4j-api:1.7.36") 25 | } 26 | 27 | -------------------------------------------------------------------------------- /trino-ext-authz/src/main/java/io/trino/plugin/base/security/StringFunctions.java: -------------------------------------------------------------------------------- 1 | package io.trino.plugin.base.security; 2 | 3 | import io.airlift.slice.Slice; 4 | import io.trino.spi.connector.ConnectorSession; 5 | import io.trino.spi.function.Description; 6 | import io.trino.spi.function.LiteralParameters; 7 | import io.trino.spi.function.ScalarFunction; 8 | import io.trino.spi.function.SqlType; 9 | 10 | import static io.airlift.slice.SliceUtf8.toUpperCase; 11 | 12 | public class StringFunctions { 13 | 14 | 15 | @Description("Converts the string to upper case") 16 | @ScalarFunction("to_upper") 17 | @LiteralParameters("x") 18 | @SqlType("varchar(x)") 19 | public static Slice upper(ConnectorSession session, @SqlType("varchar(x)") Slice slice) { 20 | return toUpperCase(slice); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /hive/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | ARG HIVE_BIN_VERSION="3.1.3" 3 | ARG HADOOP_BIN_VERSION="3.3.5" 4 | ENV HIVE_BIN_VERSION=${HIVE_BIN_VERSION} 5 | ENV HADOOP_BIN_VERSION=${HADOOP_BIN_VERSION} 6 | 7 | RUN apt-get update && apt-get install -y openjdk-8-jdk curl 8 | RUN mkdir /opt/app && mkdir /opt/app/work 9 | 10 | COPY hive/ /tmp/hive/ 11 | 12 | RUN /tmp/hive/setup.sh 13 | COPY hive-authz/build/libs/hive-authz.jar /opt/app/apache-hive-${HIVE_BIN_VERSION}-bin/lib/hive-authz.jar 14 | 15 | ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 16 | ENV HADOOP_HOME /opt/app/hadoop-${HADOOP_BIN_VERSION} 17 | ENV HIVE_HOME /opt/app/apache-hive-${HIVE_BIN_VERSION}-bin 18 | ENV PATH $HADOOP_HOME/bin:$HIVE_HOME/bin:$JAVA_HOME/bin:$PATH 19 | 20 | WORKDIR /opt/app/work 21 | COPY hive/run.sh /opt/app/work/hive-start.sh 22 | CMD ["./hive-start.sh"] 23 | 24 | -------------------------------------------------------------------------------- /trino-ext-authz/build.gradle.kts: -------------------------------------------------------------------------------- 1 | val trinoVersion:String by project 2 | 3 | java { 4 | sourceCompatibility = JavaVersion.VERSION_21 5 | targetCompatibility = JavaVersion.VERSION_21 6 | } 7 | 8 | configurations { 9 | implementation { 10 | resolutionStrategy.failOnVersionConflict() 11 | } 12 | } 13 | val plugin by configurations.runtimeClasspath 14 | dependencies { 15 | listOf("trino-spi", "trino-plugin-toolkit").forEach { name -> 16 | implementation("io.trino:${name}:$trinoVersion") 17 | } 18 | implementation("com.google.guava:guava:33.3.0-jre") 19 | compileOnly("org.slf4j:slf4j-api:1.7.36") 20 | 21 | } 22 | 23 | 24 | tasks.register("copyDeps") { 25 | from(plugin) 26 | include("**/*.jar") 27 | into("$buildDir/ext") 28 | } 29 | 30 | tasks.build { 31 | dependsOn("copyDeps") 32 | } 33 | -------------------------------------------------------------------------------- /hive-authz/src/main/java/com/github/skhatri/s3/TrinoCustomAwsCredentialsProvider.java: -------------------------------------------------------------------------------- 1 | package com.github.skhatri.s3; 2 | 3 | import com.amazonaws.auth.*; 4 | import org.apache.hadoop.conf.Configuration; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.net.URI; 9 | 10 | public class TrinoCustomAwsCredentialsProvider implements AWSCredentialsProvider { 11 | private static final Logger LOGGER = LoggerFactory.getLogger(TrinoCustomAwsCredentialsProvider.class); 12 | 13 | public TrinoCustomAwsCredentialsProvider(URI uri, Configuration hadoopConf) { 14 | LOGGER.info("credentials initialised for S3 URI {}", uri); 15 | } 16 | 17 | @Override 18 | public AWSCredentials getCredentials() { 19 | return CredentialsFactory.create(); 20 | } 21 | 22 | @Override 23 | public void refresh() { 24 | 25 | } 26 | 27 | 28 | } 29 | -------------------------------------------------------------------------------- /hive/output/activity/_symlink_format_manifest/version=2021-03-02/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-02/part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-02/part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet 3 | file:/opt/data/output/activity/version=2021-03-02/part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet 4 | file:/opt/data/output/activity/version=2021-03-02/part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet 5 | file:/opt/data/output/activity/version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet 6 | file:/opt/data/output/activity/version=2021-03-02/part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet 7 | file:/opt/data/output/activity/version=2021-03-02/part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet 8 | -------------------------------------------------------------------------------- /curl-query.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e -o pipefail 3 | 4 | query="select * from hive.fitness.activity limit 1" 5 | user="user2" 6 | next_uri=$(curl -s -k --cacert ./certs/trino-proxy/bundle.crt \ 7 | --header "Authorization: Basic $(echo -n admin:password|base64)" \ 8 | --header "X-Trino-User: ${user}" --data "${query}" \ 9 | https://trino-proxy:8453/v1/statement/|jq -r '.nextUri') 10 | while true; 11 | do 12 | curl -s -o out.json -k --cacert ./certs/trino-proxy/bundle.crt \ 13 | --header "Authorization: Basic $(echo -n admin:password|base64)" \ 14 | --header "X-Trino-User: ${user}" \ 15 | ${next_uri} 16 | cat out.json 17 | data_val=$(cat out.json|jq -r '.data') 18 | if [[ "${data_val}" != "null" ]]; 19 | then 20 | echo "-----DATA------" 21 | echo ${data_val} 22 | echo "---------------" 23 | break; 24 | fi; 25 | next_uri=$(cat out.json|jq -r '.nextUri') 26 | if [[ "${next_uri}" == "null" ]]; 27 | then 28 | break; 29 | fi; 30 | sleep 1; 31 | done; 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TARGET=coordinator 2 | TRINO_VERSION := $(shell grep trinoVersion gradle.properties | awk -F'=' '{print $$2}' | xargs) 3 | .PHONY: build coordinator trino 4 | 5 | 6 | trino: 7 | @./trino --debug --user=admin --password --truststore-path=./certs/truststore.jks --truststore-password=password --server https://localhost:8443 8 | 9 | trino-k8s: 10 | @./trino --debug --user=admin --password --truststore-path=./certs/truststore.jks --truststore-password=password --server https://coordinator:32538 11 | 12 | start: up 13 | up: 14 | @docker-compose --env-file .env up -d 15 | 16 | stop: down 17 | down: 18 | @docker-compose down 19 | 20 | ps: status 21 | status: 22 | @docker-compose ps 23 | 24 | hive-cli: 25 | @docker exec -it hive hive 26 | 27 | psql: 28 | @docker exec -it postgres psql -U postgres 29 | 30 | build: 31 | @./gradlew clean build -q 32 | @docker-compose build --build-arg TRINO_VERSION=$(TRINO_VERSION) --no-cache $(TARGET) 33 | 34 | build-hive: 35 | @./gradlew clean build -q 36 | @docker-compose build --no-cache hive 37 | -------------------------------------------------------------------------------- /hive/sampledata.csv: -------------------------------------------------------------------------------- 1 | activity_date,average_weight,max_weight,min_weight,calories,heart_points,heart_minutes,low_latitude,low_longitude,high_latitude,high_longitude,step_count,distance,average_speed,max_speed,min_speed,move_minutes_count,cycling_duration,inactive_duration,walking_duration,running_duration 2 | 2019-11-03,,,,2909.2953466643,72.0,63,33.9449006,-122.4768472,37.8091422,-118.4007828,19363,13745.627644442604,1.3299761534129335,18.032257080078125,0.045891135931015015,222,,64861505,7164427,6170764 3 | 2019-11-04,,,,2074.4301283498266,24.0,24,33.9449006,-122.4768472,33.9449006,-122.4768472,8517,4256.207019860856,1.0685129861952911,22.17345428466797,0.11649135500192642,96,,75858068,5643595, 4 | 2019-11-05,,,,2252.8376169513185,23.0,23,33.9449016,-122.4768472,33.9449016,-122.4768472,11620,5605.175660118461,0.9157865808977221,19.012468338012695,0.2459133416414261,129,,69543818,9275899,105833 5 | 2019-11-06,,,,1928.8807108441429,17.0,17,33.9449016,-122.4768472,33.9449016,-122.4768472,7008,3270.494076527655,0.5904966302886753,6.193288326263428,0.06855156272649765,77,,76200792,4631700, -------------------------------------------------------------------------------- /certs/trino-proxy/server.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN NEW CERTIFICATE REQUEST----- 2 | MIIC4DCCAcgCAQAwazELMAkGA1UEBhMCV08xDjAMBgNVBAgTBVdvcmxkMQ4wDAYD 3 | VQQHEwVXb3JsZDEZMBcGA1UEChMQU29mdHdhcmUgQ29tcGFueTELMAkGA1UECxMC 4 | SVQxFDASBgNVBAMTC3RyaW5vLXByb3h5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A 5 | MIIBCgKCAQEAxXqrUcx5wUv3FyF/tRjQaGOBSV/mZgs4TBsRjBuR/8LyNj/U46du 6 | MKu20kz6FbKVqKtMigoQLSCLQkgHRVi7bd1+Xk0a5HvqIUtL+YhzRyWjQi7wJlL3 7 | LvgE6794w7wi0VIyZEH2Di++cqr2SaUGi51yO5uQz/YTJM/TC5ZOdTOuXTbVPTZS 8 | VxpIKJIv6wHltGexgLl4SMsTqt8Ww++sXK5+jR+SDYS+QU4zopqgCLaArQU2eJ2E 9 | JyB6anTSvRF0SWM/YYv/3WlsubW6C+GjDQbQs2egUdCbx7yhAWLoo4/xomS0O+8S 10 | 010VtognTo7wmToxdPu9CrpKWjYkIuWkeQIDAQABoDAwLgYJKoZIhvcNAQkOMSEw 11 | HzAdBgNVHQ4EFgQUDzHac9eSknUCZ7Be7/Cz6lyoQ2wwDQYJKoZIhvcNAQEMBQAD 12 | ggEBADXH9yT+vuA/AlO78Sw/kzjZ2jzHTP59A6RCWV9i5o+DI5kG/5uWQ2ZpZ1MD 13 | fQgRcWPiES4+9wolnvchnwq6p1fPUwuFQ3fEfGjTrHbblbH1Qw/0HXB2qov+mk1l 14 | xai1jpuMCcF8acTL+/6K9ptYYExUfJAKziwRTet14RK1JCodHbgy79h0zizqlJDA 15 | fW0S3IwUqjFhCfnE8MpOCYloQAff3FgznhbtmnOStRR4rB1N6mqgsSQudIyKW4o6 16 | JMZh1PZp3jc5KACLk6f+gi+4T8UY5rN7uiyBPQ7qDEUuBobocAZzahoegdobA7qK 17 | 9K9Z2D5cz2MNWmBhMKqp1Bqd8e8= 18 | -----END NEW CERTIFICATE REQUEST----- 19 | -------------------------------------------------------------------------------- /trino-ext-authz/src/main/java/io/trino/plugin/base/security/PlainPasswordAuthenticatorFactory.java: -------------------------------------------------------------------------------- 1 | package io.trino.plugin.base.security; 2 | 3 | import io.trino.spi.security.AccessDeniedException; 4 | import io.trino.spi.security.BasicPrincipal; 5 | import io.trino.spi.security.PasswordAuthenticator; 6 | import io.trino.spi.security.PasswordAuthenticatorFactory; 7 | 8 | import java.security.Principal; 9 | import java.util.Map; 10 | 11 | public class PlainPasswordAuthenticatorFactory implements PasswordAuthenticatorFactory { 12 | @Override 13 | public String getName() { 14 | return "plain-auth"; 15 | } 16 | 17 | @Override 18 | public PasswordAuthenticator create(Map config) { 19 | return new PlainPasswordAuthenticator(); 20 | } 21 | 22 | } 23 | 24 | class PlainPasswordAuthenticator implements PasswordAuthenticator { 25 | @Override 26 | public Principal createAuthenticatedPrincipal(String user, String password) { 27 | if ("admin".equals(user) && "admin".equals(password)) { 28 | throw new AccessDeniedException("demo only: admin password can't be admin"); 29 | } 30 | return new BasicPrincipal(user); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /certs/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDSjCCAjICCQDECQcZup4gnjANBgkqhkiG9w0BAQsFADBnMQswCQYDVQQGEwJB 3 | VTEMMAoGA1UECAwDTlNXMQ8wDQYDVQQHDAZTeWRuZXkxGTAXBgNVBAoMEFNvZnR3 4 | YXJlIENvbXBhbnkxCzAJBgNVBAsMAklUMREwDwYDVQQDDAh0eXBlYm9vdDAeFw0y 5 | MTA3MzEwOTUzNDlaFw0yMTA4MzAwOTUzNDlaMGcxCzAJBgNVBAYTAkFVMQwwCgYD 6 | VQQIDANOU1cxDzANBgNVBAcMBlN5ZG5leTEZMBcGA1UECgwQU29mdHdhcmUgQ29t 7 | cGFueTELMAkGA1UECwwCSVQxETAPBgNVBAMMCHR5cGVib290MIIBIjANBgkqhkiG 8 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwK4epBwDXq8FAHCMSVNhQemQ/+uq9aPSYivH 9 | B4fGKIgialg6lbeei8vXjyqq/N0YqY9Gcgq5wTkO/cgpukPJlL+Ykzmu6MdfvFze 10 | nyZWAwwvNmoTnEa27NqgGRha3i5JhSISqFu9lCLjnxwfib6Sb/+FkAsw//7DLVru 11 | uwTpvxWJ94eymO07WnRr3+TCMrmpU6WQ8Zd+lT5FX1TRv83TJXCEs0RC8YxMBQNj 12 | SrH5Z/PSLtlzHbiosPqf0BuN5fF446LkvcsQof4p/j1La6v0nwa9YeQKZni9uHYZ 13 | OVkg5PwqP2VtPT1Jt/OTVDDiq6X0Q7muRrcaBm2w272hFrAVIQIDAQABMA0GCSqG 14 | SIb3DQEBCwUAA4IBAQCHMnNXn505k/TmjgD0DAp0OqpvwU8x3ugdKcnxNcZb2fhZ 15 | bHIi8uIUMowd0k+G+/2HNpEV6scjZxKKjm+YsHtWKlLUjUTJ+9PhFJHwNltzPK5s 16 | CT0EFGuuJYY7/C2Kt86FWDffAxOOB8prTsTk8LmXXwjGcQQGeOwUhI1tzyO9F9tC 17 | p6U1amkUrsCKqSXR6ocYMpDJhenoAgr+9ITyaGxIMCcn3WErzpk70o1Wf5AlFjgJ 18 | 7e62krg/ek01dF/0eUH0i/tO4TSBDrGA7SbU4ndeSE6gyOFW/R0lDUAEamCEBUTd 19 | 7IwBS13z40O253KXoqjzQdiULzF716wbtBimZ5Ym 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /worker/config.properties: -------------------------------------------------------------------------------- 1 | #single node install config 2 | coordinator=false 3 | node-scheduler.include-coordinator=false 4 | node.internal-address=worker-1 5 | discovery.uri=http://coordinator:8080 6 | 7 | http-server.http.enabled=true 8 | http-server.http.port=8080 9 | 10 | http-server.https.enabled=true 11 | http-server.https.port=8443 12 | http-server.https.keystore.path=/opt/trino/certs/keystore.jks 13 | http-server.https.keystore.key=password 14 | 15 | internal-communication.shared-secret=vtudcHKq/rf6KUiJ4N3aQiMW5nc6/Tve3t1R/yTbVPVXDVZVkWZIoMk3mg3IDyTk6sj+K9H5nLR38AhfwK1W/7me6UnrexM8s9KAoylSPL0zCXNiVgONrKriG9Q9qkVX6cWNXU/Yux8RJM5+7ZQYINJYajZKJwFv7yTOW3WUo305m7Y1E5y/OJ2aHMd8EqAYdHPO8MvrR+rc5cb7urn2Q8bCLEuTm8W/6f6qmb75u+FNtZGqIL1Ro/yV3sSdyYmka47OnwZ+XVxYV+O3M/6sT5plzHKgd3pXt0CdBQGMiFfxWDRJ6CCClJ8JK60FKUctb3w9rhS8wCF0Md8PM05qyFuR0rTl3TS+4/6rn0REWJYXINXMyywUq/TkIFU4EoNaKVFS4uW2utmc1xV/K4PJDG7kJojyXDyRsGI4saraflZGsPnqg3kO3jjFJI/nZxtEt7dNhWFxiqCEStkfv5+TmajvMgFhn4ZoeeNjTh1yyEKexNYRL/nqFHBHSzFjt+Ab5/j5PHAaBW4/HXPbpFfv1JjgKjQit7R7d7QbgDOKepGTemw7hPZcwwHvfqAT5rMmDGZwrQmBjXrJZ12GDEmuBsqjlCaAQbjMtBHbbS81KYmrIZlyXPNCP9S3lWOkNmwLgtY8sVs3TOpP0Ze/MRqtLvaePAwtEhLeUgLIBGHBCKQ= 16 | internal-communication.https.required=false 17 | internal-communication.https.keystore.path=/opt/trino/certs/keystore.jks 18 | internal-communication.https.keystore.key=password 19 | 20 | 21 | query.max-memory=1GB 22 | query.max-memory-per-node=512MB 23 | 24 | -------------------------------------------------------------------------------- /hive/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ -z ${HIVE_BIN_VERSION} ]] || [[ -z $HADOOP_BIN_VERSION ]]; then 4 | echo hadoop and hive bin version required 5 | exit 1; 6 | fi; 7 | 8 | if [[ ! -f /tmp/hive/apache-hive-${HIVE_BIN_VERSION}-bin.tar.gz ]]; 9 | then 10 | curl -o /tmp/hive/apache-hive-${HIVE_BIN_VERSION}-bin.tar.gz https://downloads.apache.org/hive/hive-${HIVE_BIN_VERSION}/apache-hive-${HIVE_BIN_VERSION}-bin.tar.gz 11 | fi; 12 | if [[ ! -f /tmp/hive/hadoop-${HADOOP_BIN_VERSION}.tar.gz ]]; 13 | then 14 | curl -o /tmp/hive/hadoop-${HADOOP_BIN_VERSION}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_BIN_VERSION}/hadoop-${HADOOP_BIN_VERSION}.tar.gz 15 | fi; 16 | 17 | tar zxf /tmp/hive/apache-hive-${HIVE_BIN_VERSION}-bin.tar.gz -C /opt/app && \ 18 | tar zxf /tmp/hive/hadoop-${HADOOP_BIN_VERSION}.tar.gz -C /opt/app && \ 19 | rm /opt/app/apache-hive-${HIVE_BIN_VERSION}-bin/lib/guava-19.0.jar && \ 20 | cp /opt/app/hadoop-${HADOOP_BIN_VERSION}/share/hadoop/hdfs/lib/guava-27.0-jre.jar /opt/app/apache-hive-${HIVE_BIN_VERSION}-bin/lib/ && \ 21 | cp /opt/app/hadoop-${HADOOP_BIN_VERSION}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_BIN_VERSION}.jar /opt/app/apache-hive-${HIVE_BIN_VERSION}-bin/lib/ && \ 22 | cp /opt/app/hadoop-${HADOOP_BIN_VERSION}/share/hadoop/tools/lib/aws-java-sdk-bundle-1.12.316.jar /opt/app/apache-hive-${HIVE_BIN_VERSION}-bin/lib/ && \ 23 | rm /tmp/hive/*.gz 24 | 25 | -------------------------------------------------------------------------------- /trino-ext-authz/src/main/java/io/trino/plugin/base/security/ExternalAuthsPlugin.java: -------------------------------------------------------------------------------- 1 | package io.trino.plugin.base.security; 2 | 3 | import com.google.common.collect.ImmutableSet; 4 | import io.trino.spi.Plugin; 5 | import io.trino.spi.eventlistener.EventListenerFactory; 6 | import io.trino.spi.security.PasswordAuthenticatorFactory; 7 | import io.trino.spi.security.SystemAccessControlFactory; 8 | 9 | import java.util.Set; 10 | 11 | public class ExternalAuthsPlugin implements Plugin { 12 | 13 | @Override 14 | public Iterable getSystemAccessControlFactories() { 15 | return ImmutableSet.builder() 16 | .add(new ExternalAuthzSystemAccessControl.Factory()) 17 | .build(); 18 | } 19 | 20 | @Override 21 | public Set> getFunctions() { 22 | return ImmutableSet.>builder() 23 | .add(StringFunctions.class) 24 | .build(); 25 | } 26 | 27 | @Override 28 | public Iterable getEventListenerFactories() { 29 | return ImmutableSet.builder() 30 | .add(new AuditLoggingEventListenerFactory()) 31 | .build(); 32 | } 33 | 34 | @Override 35 | public Iterable getPasswordAuthenticatorFactories() { 36 | return ImmutableSet.builder() 37 | .add(new PlainPasswordAuthenticatorFactory()) 38 | .build(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /certs/trino-proxy/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/DCCAuSgAwIBAgIJAOtKfxabn2WZMA0GCSqGSIb3DQEBCwUAMGcxCzAJBgNV 3 | BAYTAkFVMQwwCgYDVQQIDANOU1cxDzANBgNVBAcMBlN5ZG5leTEZMBcGA1UECgwQ 4 | U29mdHdhcmUgQ29tcGFueTELMAkGA1UECwwCSVQxETAPBgNVBAMMCHR5cGVib290 5 | MB4XDTI0MDkwOTEyNDQ0MFoXDTI0MTAwOTEyNDQ0MFowazELMAkGA1UEBhMCV08x 6 | DjAMBgNVBAgTBVdvcmxkMQ4wDAYDVQQHEwVXb3JsZDEZMBcGA1UEChMQU29mdHdh 7 | cmUgQ29tcGFueTELMAkGA1UECxMCSVQxFDASBgNVBAMTC3RyaW5vLXByb3h5MIIB 8 | IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxXqrUcx5wUv3FyF/tRjQaGOB 9 | SV/mZgs4TBsRjBuR/8LyNj/U46duMKu20kz6FbKVqKtMigoQLSCLQkgHRVi7bd1+ 10 | Xk0a5HvqIUtL+YhzRyWjQi7wJlL3LvgE6794w7wi0VIyZEH2Di++cqr2SaUGi51y 11 | O5uQz/YTJM/TC5ZOdTOuXTbVPTZSVxpIKJIv6wHltGexgLl4SMsTqt8Ww++sXK5+ 12 | jR+SDYS+QU4zopqgCLaArQU2eJ2EJyB6anTSvRF0SWM/YYv/3WlsubW6C+GjDQbQ 13 | s2egUdCbx7yhAWLoo4/xomS0O+8S010VtognTo7wmToxdPu9CrpKWjYkIuWkeQID 14 | AQABo4GmMIGjMB0GA1UdDgQWBBQPMdpz15KSdQJnsF7v8LPqXKhDbDCBgQYDVR0j 15 | BHoweKFrpGkwZzELMAkGA1UEBhMCQVUxDDAKBgNVBAgMA05TVzEPMA0GA1UEBwwG 16 | U3lkbmV5MRkwFwYDVQQKDBBTb2Z0d2FyZSBDb21wYW55MQswCQYDVQQLDAJJVDER 17 | MA8GA1UEAwwIdHlwZWJvb3SCCQDECQcZup4gnjANBgkqhkiG9w0BAQsFAAOCAQEA 18 | Rvjx0qUYqN0qLyjZQimuRBTKT9lM9Ed6IJtUv35XGk+J6RCbE2EmcWtekzv2YSQO 19 | OhNBFlAfcYr7TUISV/V2Qcz0P3S1EzBchwpYR+zrQ0D7MKHs1ZjNLcpBQY2qJMlS 20 | 3AJTGzK6emkqWNsMPi8TtglC13qJtB0m7AP0GREQMvB51ERFBGDTwuCrUl3i7RTl 21 | BljM5zg472MVJ+GVpCCgsnQfN4jg6710PnwBifuFRAK7qhOIsPJlmugFi+WNHXap 22 | qoHNmzgSzgxE3457ccVdOHlHFum9FLf+CBxoo2VZQRYFOGGKT/JcGvrUEmIJATqy 23 | LifJT0pdU/M23iQ6TzPZ/w== 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /coordinator/config.properties: -------------------------------------------------------------------------------- 1 | #single node install config 2 | coordinator=true 3 | node-scheduler.include-coordinator=true 4 | node.internal-address=coordinator 5 | discovery-server.enabled=true 6 | discovery.uri=http://coordinator:8080 7 | 8 | web-ui.enabled=true 9 | 10 | http-server.http.enabled=true 11 | http-server.http.port=8080 12 | 13 | http-server.https.enabled=true 14 | http-server.https.port=8443 15 | http-server.https.keystore.path=/opt/trino/certs/keystore.jks 16 | http-server.https.keystore.key=password 17 | 18 | http-server.authentication.type=password 19 | http-server.auth.users-file=/opt/trino/security/passwords/password.db 20 | #http-server.authentication.allow-insecure-over-http=true 21 | 22 | internal-communication.shared-secret=vtudcHKq/rf6KUiJ4N3aQiMW5nc6/Tve3t1R/yTbVPVXDVZVkWZIoMk3mg3IDyTk6sj+K9H5nLR38AhfwK1W/7me6UnrexM8s9KAoylSPL0zCXNiVgONrKriG9Q9qkVX6cWNXU/Yux8RJM5+7ZQYINJYajZKJwFv7yTOW3WUo305m7Y1E5y/OJ2aHMd8EqAYdHPO8MvrR+rc5cb7urn2Q8bCLEuTm8W/6f6qmb75u+FNtZGqIL1Ro/yV3sSdyYmka47OnwZ+XVxYV+O3M/6sT5plzHKgd3pXt0CdBQGMiFfxWDRJ6CCClJ8JK60FKUctb3w9rhS8wCF0Md8PM05qyFuR0rTl3TS+4/6rn0REWJYXINXMyywUq/TkIFU4EoNaKVFS4uW2utmc1xV/K4PJDG7kJojyXDyRsGI4saraflZGsPnqg3kO3jjFJI/nZxtEt7dNhWFxiqCEStkfv5+TmajvMgFhn4ZoeeNjTh1yyEKexNYRL/nqFHBHSzFjt+Ab5/j5PHAaBW4/HXPbpFfv1JjgKjQit7R7d7QbgDOKepGTemw7hPZcwwHvfqAT5rMmDGZwrQmBjXrJZ12GDEmuBsqjlCaAQbjMtBHbbS81KYmrIZlyXPNCP9S3lWOkNmwLgtY8sVs3TOpP0Ze/MRqtLvaePAwtEhLeUgLIBGHBCKQ= 23 | internal-communication.https.required=false 24 | internal-communication.https.keystore.path=/opt/trino/certs/keystore.jks 25 | internal-communication.https.keystore.key=password 26 | 27 | 28 | query.max-memory=1GB 29 | query.max-memory-per-node=512MB 30 | 31 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_log/00000000000000000000.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1615860834102,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"version\"]"},"isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputBytes":"4679","numOutputRows":"15"}}} 2 | {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} 3 | {"metaData":{"id":"7636a7a1-243e-4411-9c79-bd017dcd7370","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"account\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"txn_id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"merchant\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"category\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"last_updated\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"deleted\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"version\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"txn_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"amount\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["version"],"configuration":{},"createdTime":1615860831812}} 4 | {"add":{"path":"version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet","partitionValues":{"version":"2021-03-02"},"size":2351,"modificationTime":1615860832974,"dataChange":true}} 5 | {"add":{"path":"version=2021-03-02/part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet","partitionValues":{"version":"2021-03-02"},"size":2328,"modificationTime":1615860832974,"dataChange":true}} 6 | -------------------------------------------------------------------------------- /hive-authz/src/main/java/com/github/skhatri/s3/CredentialsFactory.java: -------------------------------------------------------------------------------- 1 | package com.github.skhatri.s3; 2 | 3 | import com.amazonaws.auth.AWSCredentials; 4 | import com.amazonaws.auth.BasicAWSCredentials; 5 | import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | class CredentialsFactory { 10 | private static final Logger LOGGER = LoggerFactory.getLogger(CredentialsFactory.class); 11 | 12 | private CredentialsFactory() { 13 | } 14 | 15 | static final AWSCredentials create() { 16 | String accessKey = System.getenv("STORE_KEY"); 17 | String secretKey = System.getenv("STORE_SECRET"); 18 | String identityFile = System.getenv("STORE_TOKEN_FILE"); 19 | String roleArn = System.getenv("STORE_ROLE_ARN"); 20 | 21 | AWSCredentials credentials = null; 22 | if(accessKey != null && secretKey != null) { 23 | LOGGER.info("using basic AWS credentials {}**\n", accessKey.substring(0, accessKey.length() > 5 ? 5 : accessKey.length())); 24 | credentials = new BasicAWSCredentials(accessKey, secretKey); 25 | } else { 26 | WebIdentityTokenCredentialsProvider.Builder builder = WebIdentityTokenCredentialsProvider.builder(); 27 | if (roleArn != null) { 28 | LOGGER.info("adding ROLE ARN {}", roleArn); 29 | builder = builder.roleArn(roleArn); 30 | } 31 | if (identityFile != null) { 32 | LOGGER.info("adding identity file {} to create S3 credentials", identityFile); 33 | builder = builder.webIdentityTokenFile(identityFile); 34 | } 35 | credentials = builder.build().getCredentials(); 36 | } 37 | return credentials; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /certs/trino-proxy/server.rsa.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDFeqtRzHnBS/cX 3 | IX+1GNBoY4FJX+ZmCzhMGxGMG5H/wvI2P9Tjp24wq7bSTPoVspWoq0yKChAtIItC 4 | SAdFWLtt3X5eTRrke+ohS0v5iHNHJaNCLvAmUvcu+ATrv3jDvCLRUjJkQfYOL75y 5 | qvZJpQaLnXI7m5DP9hMkz9MLlk51M65dNtU9NlJXGkgoki/rAeW0Z7GAuXhIyxOq 6 | 3xbD76xcrn6NH5INhL5BTjOimqAItoCtBTZ4nYQnIHpqdNK9EXRJYz9hi//daWy5 7 | tboL4aMNBtCzZ6BR0JvHvKEBYuijj/GiZLQ77xLTXRW2iCdOjvCZOjF0+70Kukpa 8 | NiQi5aR5AgMBAAECggEAW+8lvqkArbpdx8jymS56J37be3tH2Y2zocqgOqfDKlxp 9 | QlQpGeqiY8otDG6FpoBDKAjHUyQoL7oM2siJigErEJSi9efTczc04lcZgYvCjwa4 10 | oGEVr+HD7XvNXqr+37DATM6FYyNKF4Smlt98VQCcsXaU7acYnZgjhWmzCJvq3qhk 11 | sDWaYBT4JLvN2q+IQBsYo3hd8dBpAU37/od3Az6mr88oSyrzNPDlRJgPkc4y8EJZ 12 | mBHdq8bON6b6FcbnACFmjRHnH9I7xzYbz1Fh0xYiHzYcSsBnrnkG68SSuXgqAv/J 13 | khxboA4at8R7CbKMN68wUlIr0vDz8WP0gbL0KnAs2QKBgQDP58e6quV69P2MBRS8 14 | wDvyrwyjoKXOOBbcnqXqgJYYsTXn2a3IZv5sMSz92FTIP+Hboj8RZ3z59DLV7pT6 15 | VjFvIGKbAou8XkSV8sW5JjFxn9UxfGC4saHBTE1siuSiH80qPBW9jgmLWDoLLFSj 16 | nwNFAp7G5mQ1lCa2BSDlPf/Z7wKBgQDzKXIJHx35nqjwPZ+xXKcqftPik+t/5WIG 17 | lCyBvdLBeqfCocNkdaxYbWxOMP5dYJLpyrlXv+sB3PbKYv8Ad5YCL5gCLmyf2Wlz 18 | ESKBgXyqAZs/hTBVmoFFm5vJ98UDabfD+iLhkJk8DL0nlkD0/d6JZFe8VxKUYckD 19 | ID4xgffwFwKBgQDNb/PNgYdG64UK8g5ewPm7wbuUiG6uioAUuvOp+CvY9GsW0eNG 20 | DnfDgwg5QaXtRiqw8+6sDP3So7oD6T/u6nLoozCsiWR42xGctgso/BOJem2Y+5U7 21 | i9MRhYtQASMRDKmX3U2CRDMsLR8V4VU1ZTA+5ertEcXdp6pk3EZ4H76MowKBgQCs 22 | C+0VqyCoV6wsn014TpgwpyAANqzwBzq+cOS0v7x5DPHZ8l6vm9rfjjyNbnjrVMNe 23 | DAPrpG0BGni7ZiQLDpHi5g3lrTdKOqgztDH388hi+VQBM5mpV/KTb7BgzKBm88wE 24 | K922P3tMU68abWN0dJOaHpc9OVZ6Cl4RiiS3HsBw7QKBgQClGaGkYaL4z4g2kMMi 25 | NlIoYwDELnOqevTnG7uzGlBwbk5M0iHA9PPIcCGFx4gWce/D4UB5QlvQIxdmm9GN 26 | qLs2GQvUs4syprGlS+cFrDsXIutH7yk2OWtmd6xWHkvzaVrjl6GcXkxIu0nEbCi9 27 | tJ6jZwnFr54VNexqOyENrz2gcw== 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /hive/output/activity/_symlink_format_manifest/version=2021-03-03/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-03/part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-03/part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet 3 | file:/opt/data/output/activity/version=2021-03-03/part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet 4 | file:/opt/data/output/activity/version=2021-03-03/part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet 5 | file:/opt/data/output/activity/version=2021-03-03/part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet 6 | file:/opt/data/output/activity/version=2021-03-03/part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet 7 | file:/opt/data/output/activity/version=2021-03-03/part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet 8 | file:/opt/data/output/activity/version=2021-03-03/part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet 9 | file:/opt/data/output/activity/version=2021-03-03/part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet 10 | file:/opt/data/output/activity/version=2021-03-03/part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet 11 | file:/opt/data/output/activity/version=2021-03-03/part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet 12 | file:/opt/data/output/activity/version=2021-03-03/part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet 13 | file:/opt/data/output/activity/version=2021-03-03/part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet 14 | file:/opt/data/output/activity/version=2021-03-03/part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet 15 | file:/opt/data/output/activity/version=2021-03-03/part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet 16 | -------------------------------------------------------------------------------- /hive/output/activity/_symlink_format_manifest/version=2021-03-05/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-05/part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-05/part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet 3 | file:/opt/data/output/activity/version=2021-03-05/part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet 4 | file:/opt/data/output/activity/version=2021-03-05/part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet 5 | file:/opt/data/output/activity/version=2021-03-05/part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet 6 | file:/opt/data/output/activity/version=2021-03-05/part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet 7 | file:/opt/data/output/activity/version=2021-03-05/part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet 8 | file:/opt/data/output/activity/version=2021-03-05/part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet 9 | file:/opt/data/output/activity/version=2021-03-05/part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet 10 | file:/opt/data/output/activity/version=2021-03-05/part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet 11 | file:/opt/data/output/activity/version=2021-03-05/part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet 12 | file:/opt/data/output/activity/version=2021-03-05/part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet 13 | file:/opt/data/output/activity/version=2021-03-05/part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet 14 | file:/opt/data/output/activity/version=2021-03-05/part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet 15 | file:/opt/data/output/activity/version=2021-03-05/part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet 16 | -------------------------------------------------------------------------------- /certs/trino-proxy/server.key: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: mykey 3 | localKeyID: 54 69 6D 65 20 31 37 32 35 38 38 35 38 38 31 33 39 30 4 | Key Attributes: 5 | -----BEGIN PRIVATE KEY----- 6 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDFeqtRzHnBS/cX 7 | IX+1GNBoY4FJX+ZmCzhMGxGMG5H/wvI2P9Tjp24wq7bSTPoVspWoq0yKChAtIItC 8 | SAdFWLtt3X5eTRrke+ohS0v5iHNHJaNCLvAmUvcu+ATrv3jDvCLRUjJkQfYOL75y 9 | qvZJpQaLnXI7m5DP9hMkz9MLlk51M65dNtU9NlJXGkgoki/rAeW0Z7GAuXhIyxOq 10 | 3xbD76xcrn6NH5INhL5BTjOimqAItoCtBTZ4nYQnIHpqdNK9EXRJYz9hi//daWy5 11 | tboL4aMNBtCzZ6BR0JvHvKEBYuijj/GiZLQ77xLTXRW2iCdOjvCZOjF0+70Kukpa 12 | NiQi5aR5AgMBAAECggEAW+8lvqkArbpdx8jymS56J37be3tH2Y2zocqgOqfDKlxp 13 | QlQpGeqiY8otDG6FpoBDKAjHUyQoL7oM2siJigErEJSi9efTczc04lcZgYvCjwa4 14 | oGEVr+HD7XvNXqr+37DATM6FYyNKF4Smlt98VQCcsXaU7acYnZgjhWmzCJvq3qhk 15 | sDWaYBT4JLvN2q+IQBsYo3hd8dBpAU37/od3Az6mr88oSyrzNPDlRJgPkc4y8EJZ 16 | mBHdq8bON6b6FcbnACFmjRHnH9I7xzYbz1Fh0xYiHzYcSsBnrnkG68SSuXgqAv/J 17 | khxboA4at8R7CbKMN68wUlIr0vDz8WP0gbL0KnAs2QKBgQDP58e6quV69P2MBRS8 18 | wDvyrwyjoKXOOBbcnqXqgJYYsTXn2a3IZv5sMSz92FTIP+Hboj8RZ3z59DLV7pT6 19 | VjFvIGKbAou8XkSV8sW5JjFxn9UxfGC4saHBTE1siuSiH80qPBW9jgmLWDoLLFSj 20 | nwNFAp7G5mQ1lCa2BSDlPf/Z7wKBgQDzKXIJHx35nqjwPZ+xXKcqftPik+t/5WIG 21 | lCyBvdLBeqfCocNkdaxYbWxOMP5dYJLpyrlXv+sB3PbKYv8Ad5YCL5gCLmyf2Wlz 22 | ESKBgXyqAZs/hTBVmoFFm5vJ98UDabfD+iLhkJk8DL0nlkD0/d6JZFe8VxKUYckD 23 | ID4xgffwFwKBgQDNb/PNgYdG64UK8g5ewPm7wbuUiG6uioAUuvOp+CvY9GsW0eNG 24 | DnfDgwg5QaXtRiqw8+6sDP3So7oD6T/u6nLoozCsiWR42xGctgso/BOJem2Y+5U7 25 | i9MRhYtQASMRDKmX3U2CRDMsLR8V4VU1ZTA+5ertEcXdp6pk3EZ4H76MowKBgQCs 26 | C+0VqyCoV6wsn014TpgwpyAANqzwBzq+cOS0v7x5DPHZ8l6vm9rfjjyNbnjrVMNe 27 | DAPrpG0BGni7ZiQLDpHi5g3lrTdKOqgztDH388hi+VQBM5mpV/KTb7BgzKBm88wE 28 | K922P3tMU68abWN0dJOaHpc9OVZ6Cl4RiiS3HsBw7QKBgQClGaGkYaL4z4g2kMMi 29 | NlIoYwDELnOqevTnG7uzGlBwbk5M0iHA9PPIcCGFx4gWce/D4UB5QlvQIxdmm9GN 30 | qLs2GQvUs4syprGlS+cFrDsXIutH7yk2OWtmd6xWHkvzaVrjl6GcXkxIu0nEbCi9 31 | tJ6jZwnFr54VNexqOyENrz2gcw== 32 | -----END PRIVATE KEY----- 33 | -------------------------------------------------------------------------------- /ca.sh: -------------------------------------------------------------------------------- 1 | #create CA cert and self sign 2 | if [[ ! -d ca ]]; 3 | then 4 | mkdir ca 5 | openssl genrsa -out ca/ca.key 2048 6 | openssl req -new -x509 -key ca/ca.key -out ca/ca.crt -subj "/C=AU/ST=NSW/L=Sydney/O=Software Company/OU=IT/CN=typeboot" 7 | fi; 8 | 9 | 10 | host_name=${1:-localhost} 11 | alt_name=$2 12 | 13 | if [[ $host_name == "ca" ]]; 14 | then 15 | echo "invalid server name" 16 | exit 1; 17 | fi; 18 | 19 | cert=certs/$host_name 20 | if [[ ! -d $cert ]]; 21 | then 22 | mkdir "$cert" 23 | fi; 24 | ext="san=dns:${host_name}" 25 | if [[ ! -z ${alt_name} ]]; 26 | then 27 | ext="${ext},dns:${alt_name}" 28 | fi; 29 | 30 | ### Server Setup 31 | KEY_FILE="$cert/keystore.p12" 32 | STORE_PASS="cassandra" 33 | keytool -genkeypair -keyalg RSA -keysize 2048 -storetype PKCS12 -keystore ${KEY_FILE} -validity 365 \ 34 | --dname "CN=$host_name, OU=IT, O=Software Company, L=World, ST=World, C=WO" \ 35 | -ext "${ext}" \ 36 | -storepass ${STORE_PASS} 37 | 38 | keytool -certreq -keystore "${KEY_FILE}" -file "${cert}/server.csr" -storepass ${STORE_PASS} -keypass ${STORE_PASS} 39 | 40 | 41 | #CA sign cert request 42 | openssl x509 -req -in $cert/server.csr -CA ca/ca.crt -CAkey ca/ca.key -CAcreateserial -out $cert/server.crt 43 | 44 | openssl x509 -in $cert/server.crt -noout -text 45 | cat $cert/server.crt ca/ca.crt > $cert/bundle.crt 46 | openssl x509 -in $cert/bundle.crt -noout -text 47 | 48 | ### Server Import Cert 49 | echo yes | keytool -importcert -file $cert/bundle.crt -keystore ${KEY_FILE} -keypass ${STORE_PASS} -storepass ${STORE_PASS} -trustcacerts 50 | ### Create Truststore 51 | echo yes | keytool -importcert -alias cassandra -file $cert/bundle.crt -keystore $cert/truststore.jks -storepass ${STORE_PASS} -keypass ${STORE_PASS} -trustcacerts 52 | 53 | echo extracting private key 54 | openssl pkcs12 -info -in ${KEY_FILE} -nodes -nocerts -out $cert/server.key -password "pass:${STORE_PASS}" 55 | openssl rsa -in $cert/server.key -out $cert/server.rsa.key 56 | echo files generated at ${cert} 57 | ls $cert 58 | 59 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_manifest/version=2021-03-03/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-03/part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-02/part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet 3 | file:/opt/data/output/activity/version=2021-03-03/part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet 4 | file:/opt/data/output/activity/version=2021-03-03/part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet 5 | file:/opt/data/output/activity/version=2021-03-03/part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet 6 | file:/opt/data/output/activity/version=2021-03-03/part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet 7 | file:/opt/data/output/activity/version=2021-03-03/part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet 8 | file:/opt/data/output/activity/version=2021-03-03/part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet 9 | file:/opt/data/output/activity/version=2021-03-03/part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet 10 | file:/opt/data/output/activity/version=2021-03-03/part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet 11 | file:/opt/data/output/activity/version=2021-03-03/part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet 12 | file:/opt/data/output/activity/version=2021-03-03/part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet 13 | file:/opt/data/output/activity/version=2021-03-03/part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet 14 | file:/opt/data/output/activity/version=2021-03-03/part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet 15 | file:/opt/data/output/activity/version=2021-03-03/part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet 16 | file:/opt/data/output/activity/version=2021-03-03/part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet 17 | file:/opt/data/output/activity/version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet 18 | -------------------------------------------------------------------------------- /certs/trino.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFkTCCA/mgAwIBAgIJAMU+8n5x4qiDMA0GCSqGSIb3DQEBDAUAMGYxCzAJBgNV 3 | BAYTAkFVMQwwCgYDVQQIEwNOU1cxDzANBgNVBAcTBlN5ZG5leTEPMA0GA1UEChMG 4 | ZGF0YWNvMREwDwYDVQQLEwhkYXRhbGFrZTEUMBIGA1UEAxMLY29vcmRpbmF0b3Iw 5 | HhcNMjQwOTA2MTIyMDM5WhcNMjQxMjA1MTIyMDM5WjBmMQswCQYDVQQGEwJBVTEM 6 | MAoGA1UECBMDTlNXMQ8wDQYDVQQHEwZTeWRuZXkxDzANBgNVBAoTBmRhdGFjbzER 7 | MA8GA1UECxMIZGF0YWxha2UxFDASBgNVBAMTC2Nvb3JkaW5hdG9yMIIBojANBgkq 8 | hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAr6B3VNktNl76/1Mkmf1s0CSgzDedBXj5 9 | 9zQDKRSRSvY9zZm7AQGNGM34mXh6QDWp7OtjVUfu6XVuL5uJoCBFBU1nnZs+hjBn 10 | DaSpeEFA1IotcgBpFuhNgcEOByNWxxz+qFF3iM+jhGZxC0K2TfU05lUTUw2mY3Yv 11 | GnZI8pa889K+f0qB9wu+rxNtJm+eHfWxoK20HMpcANutyxSgSPcJiMtlDjQPz+MC 12 | DWGOwwsH/qnG7pU+TJDFTP4OTuCmTOi8KcHufMhnMwbR2UUS5aEJ1/XwD+a/+QNl 13 | AjkhuqMqt61wuqavbzu/AYe7y/3CHE8YaaDfDdVBR9TFP9oP/yPdk2CE1VE9up/C 14 | Z5yj4F5+iW6/Dqm8DXUNhWybxZNX5n4sbiyb0STTgiaIzkPiILAWIHgc3Shf/bn1 15 | jADnEA62ClRz/C9M+DptJBAZ9+iiAVGG0Bxrem3R4lDDC2g0+sbW7MNmE9MGyyYJ 16 | lJnUR1486n67EQXbkJmbWrZS4aOZtMUbAgMBAAGjggFAMIIBPDAdBgNVHQ4EFgQU 17 | EJRt5JrfRBeDZ8kMORYvOOP5l9YwggEZBgNVHREEggEQMIIBDIILY29vcmRpbmF0 18 | b3KCEmNvb3JkaW5hdG9yLnByZXN0b4IWY29vcmRpbmF0b3IucHJlc3RvLnN2Y4Ik 19 | Y29vcmRpbmF0b3IucHJlc3RvLnN2Yy5jbHVzdGVyLmxvY2FsghRjb29yZGluYXRv 20 | ci1oZWFkbGVzc4IbY29vcmRpbmF0b3ItaGVhZGxlc3MucHJlc3Rvgh9jb29yZGlu 21 | YXRvci1oZWFkbGVzcy5wcmVzdG8uc3Zjgi1jb29yZGluYXRvci1oZWFkbGVzcy5w 22 | cmVzdG8uc3ZjLmNsdXN0ZXIubG9jYWyCCWxvY2FsaG9zdIILdHJpbm8tcHJveHmH 23 | BH8AAAGHBMCoQAWHBMCoQAYwDQYJKoZIhvcNAQEMBQADggGBAEAIYPxqeeG6R2dk 24 | MS84HDoSmsnzgISD6hwvECsyFk5jtvE97aDSWXyOwVMc1r9jOfofCuZ1fhU5nRkj 25 | RJDfwiAQ1B/UJHZf/UfgFzZlaUMetJpCMNElaYUfN13kDh7uGsD9Zg6LjkTS9LX4 26 | ggDwinCpDKQmWWQZEN6a1ayMH0l4WUqUwmr4dZ0ndFxIh0ZO1+iy8z2x89+EWdvQ 27 | 2E3DChy8gIMnirIF0itJz+75FB34MHdnnNCeMTurfXV0HCjiXlIUloLncMyyOtaG 28 | vlv49dy+HSCXvu1LHiAp686WfyLMJgfGaTVSkMHCX2m1inZneskUVOH776jB1Z9T 29 | lUq4a+4M8MAzkIdhYIJKQQ5n3kXaWdT1quZvsB85lG75+a+rwxtoNB9ALRRHSTJs 30 | 9IYmGPCaqHyToRj1ZBVWyzrEIEAofO+yNHoo4N+VR95SJ+w8VKlDpw+Iypb3GJeT 31 | GNwNKGTAn+w7Zr3kVLC3lRufKxUf1seSxw/S6KjcQUQDcLmmwA== 32 | -----END CERTIFICATE----- 33 | -------------------------------------------------------------------------------- /hive-authz/src/main/java/com/github/skhatri/hive/HiveExtAuthorizer.java: -------------------------------------------------------------------------------- 1 | package com.github.skhatri.hive; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.hive.metastore.api.Database; 5 | import org.apache.hadoop.hive.ql.metadata.AuthorizationException; 6 | import org.apache.hadoop.hive.ql.metadata.HiveException; 7 | import org.apache.hadoop.hive.ql.metadata.Partition; 8 | import org.apache.hadoop.hive.ql.metadata.Table; 9 | import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; 10 | import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; 11 | import org.apache.hadoop.hive.ql.security.authorization.Privilege; 12 | 13 | import java.util.List; 14 | 15 | public class HiveExtAuthorizer implements HiveAuthorizationProvider { 16 | @Override 17 | public void init(Configuration conf) throws HiveException { 18 | 19 | } 20 | 21 | @Override 22 | public HiveAuthenticationProvider getAuthenticator() { 23 | return null; 24 | } 25 | 26 | @Override 27 | public void setAuthenticator(HiveAuthenticationProvider authenticator) { 28 | 29 | } 30 | 31 | @Override 32 | public void authorize(Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) throws HiveException, AuthorizationException { 33 | 34 | } 35 | 36 | @Override 37 | public void authorize(Database db, Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) throws HiveException, AuthorizationException { 38 | 39 | } 40 | 41 | @Override 42 | public void authorize(Table table, Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) throws HiveException, AuthorizationException { 43 | 44 | } 45 | 46 | @Override 47 | public void authorize(Partition part, Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) throws HiveException, AuthorizationException { 48 | 49 | } 50 | 51 | @Override 52 | public void authorize(Table table, Partition part, List columns, Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) throws HiveException, AuthorizationException { 53 | 54 | } 55 | 56 | @Override 57 | public void setConf(Configuration configuration) { 58 | 59 | } 60 | 61 | @Override 62 | public Configuration getConf() { 63 | return null; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /superset/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | set -eo pipefail 19 | 20 | if [[ -d /opt/superset/certs ]]; 21 | then 22 | echo updating certificate chain 23 | mkdir -p /usr/share/ca-certificates/superset 24 | cp /opt/superset/certs/*.crt /usr/share/ca-certificates/superset/ 25 | for f in $(ls /usr/share/ca-certificates/superset); 26 | do 27 | echo superset/$f | tee -a /etc/ca-certificates.conf 28 | done; 29 | update-ca-certificates 30 | fi; 31 | 32 | superset db upgrade && superset init 33 | 34 | superset fab create-admin --username admin --firstname Admin --lastname Admin --email dev@github.com --password admin && \ 35 | superset fab create-user --role Alpha --username user1 --firstname User --lastname One --email user1@github.com --password password && \ 36 | superset fab create-user --role Alpha --username user2 --firstname User --lastname Two --email user2@github.com --password password && \ 37 | superset fab create-user --role Alpha --username user3 --firstname User --lastname Three --email user3@github.com --password password && \ 38 | superset fab create-user --role Alpha --username skhatri --firstname S --lastname Khatri --email skhatri@github.com --password password 39 | 40 | if [ "${#}" -ne 0 ]; then 41 | exec "${@}" 42 | else 43 | gunicorn \ 44 | --bind "0.0.0.0:${SUPERSET_PORT}" \ 45 | --access-logfile '-' \ 46 | --error-logfile '-' \ 47 | --workers 1 \ 48 | --worker-class gthread \ 49 | --threads 20 \ 50 | --timeout ${GUNICORN_TIMEOUT:-60} \ 51 | --limit-request-line 0 \ 52 | --limit-request-field_size 0 \ 53 | "${FLASK_APP}" 54 | fi 55 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_log/00000000000000000004.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1615860857781,"operation":"MERGE","operationParameters":{"predicate":"(existing_data.`txn_id` = new_data.`txn_id`)","matchedPredicates":"[{\"predicate\":\"(new_data.`deleted` = true)\",\"actionType\":\"delete\"},{\"actionType\":\"update\"}]","notMatchedPredicates":"[{\"actionType\":\"insert\"}]"},"readVersion":3,"isBlindAppend":false,"operationMetrics":{"numTargetRowsCopied":"0","numTargetRowsDeleted":"0","numTargetFilesAdded":"6","executionTimeMs":"0","numTargetRowsInserted":"3","scanTimeMs":"993","numTargetRowsUpdated":"3","numOutputRows":"6","numSourceRows":"6","numTargetFilesRemoved":"3","rewriteTimeMs":"2091"}}} 2 | {"remove":{"path":"version=2021-03-05/part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet","deletionTimestamp":1615860857781,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"version":"2021-03-05"},"size":2206}} 3 | {"remove":{"path":"version=2021-03-05/part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet","deletionTimestamp":1615860857781,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"version":"2021-03-05"},"size":2125}} 4 | {"remove":{"path":"version=2021-03-05/part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet","deletionTimestamp":1615860857781,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"version":"2021-03-05"},"size":2206}} 5 | {"add":{"path":"version=2021-03-05/part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2116,"modificationTime":1615860855991,"dataChange":true}} 6 | {"add":{"path":"version=2021-03-05/part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2197,"modificationTime":1615860855991,"dataChange":true}} 7 | {"add":{"path":"version=2021-03-09/part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet","partitionValues":{"version":"2021-03-09"},"size":2087,"modificationTime":1615860856028,"dataChange":true}} 8 | {"add":{"path":"version=2021-03-09/part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet","partitionValues":{"version":"2021-03-09"},"size":2197,"modificationTime":1615860856028,"dataChange":true}} 9 | {"add":{"path":"version=2021-03-09/part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet","partitionValues":{"version":"2021-03-09"},"size":2233,"modificationTime":1615860856064,"dataChange":true}} 10 | {"add":{"path":"version=2021-03-05/part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2179,"modificationTime":1615860856064,"dataChange":true}} 11 | -------------------------------------------------------------------------------- /certs/trino-proxy/bundle.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/DCCAuSgAwIBAgIJAOtKfxabn2WZMA0GCSqGSIb3DQEBCwUAMGcxCzAJBgNV 3 | BAYTAkFVMQwwCgYDVQQIDANOU1cxDzANBgNVBAcMBlN5ZG5leTEZMBcGA1UECgwQ 4 | U29mdHdhcmUgQ29tcGFueTELMAkGA1UECwwCSVQxETAPBgNVBAMMCHR5cGVib290 5 | MB4XDTI0MDkwOTEyNDQ0MFoXDTI0MTAwOTEyNDQ0MFowazELMAkGA1UEBhMCV08x 6 | DjAMBgNVBAgTBVdvcmxkMQ4wDAYDVQQHEwVXb3JsZDEZMBcGA1UEChMQU29mdHdh 7 | cmUgQ29tcGFueTELMAkGA1UECxMCSVQxFDASBgNVBAMTC3RyaW5vLXByb3h5MIIB 8 | IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxXqrUcx5wUv3FyF/tRjQaGOB 9 | SV/mZgs4TBsRjBuR/8LyNj/U46duMKu20kz6FbKVqKtMigoQLSCLQkgHRVi7bd1+ 10 | Xk0a5HvqIUtL+YhzRyWjQi7wJlL3LvgE6794w7wi0VIyZEH2Di++cqr2SaUGi51y 11 | O5uQz/YTJM/TC5ZOdTOuXTbVPTZSVxpIKJIv6wHltGexgLl4SMsTqt8Ww++sXK5+ 12 | jR+SDYS+QU4zopqgCLaArQU2eJ2EJyB6anTSvRF0SWM/YYv/3WlsubW6C+GjDQbQ 13 | s2egUdCbx7yhAWLoo4/xomS0O+8S010VtognTo7wmToxdPu9CrpKWjYkIuWkeQID 14 | AQABo4GmMIGjMB0GA1UdDgQWBBQPMdpz15KSdQJnsF7v8LPqXKhDbDCBgQYDVR0j 15 | BHoweKFrpGkwZzELMAkGA1UEBhMCQVUxDDAKBgNVBAgMA05TVzEPMA0GA1UEBwwG 16 | U3lkbmV5MRkwFwYDVQQKDBBTb2Z0d2FyZSBDb21wYW55MQswCQYDVQQLDAJJVDER 17 | MA8GA1UEAwwIdHlwZWJvb3SCCQDECQcZup4gnjANBgkqhkiG9w0BAQsFAAOCAQEA 18 | Rvjx0qUYqN0qLyjZQimuRBTKT9lM9Ed6IJtUv35XGk+J6RCbE2EmcWtekzv2YSQO 19 | OhNBFlAfcYr7TUISV/V2Qcz0P3S1EzBchwpYR+zrQ0D7MKHs1ZjNLcpBQY2qJMlS 20 | 3AJTGzK6emkqWNsMPi8TtglC13qJtB0m7AP0GREQMvB51ERFBGDTwuCrUl3i7RTl 21 | BljM5zg472MVJ+GVpCCgsnQfN4jg6710PnwBifuFRAK7qhOIsPJlmugFi+WNHXap 22 | qoHNmzgSzgxE3457ccVdOHlHFum9FLf+CBxoo2VZQRYFOGGKT/JcGvrUEmIJATqy 23 | LifJT0pdU/M23iQ6TzPZ/w== 24 | -----END CERTIFICATE----- 25 | -----BEGIN CERTIFICATE----- 26 | MIIDSjCCAjICCQDECQcZup4gnjANBgkqhkiG9w0BAQsFADBnMQswCQYDVQQGEwJB 27 | VTEMMAoGA1UECAwDTlNXMQ8wDQYDVQQHDAZTeWRuZXkxGTAXBgNVBAoMEFNvZnR3 28 | YXJlIENvbXBhbnkxCzAJBgNVBAsMAklUMREwDwYDVQQDDAh0eXBlYm9vdDAeFw0y 29 | MTA3MzEwOTUzNDlaFw0yMTA4MzAwOTUzNDlaMGcxCzAJBgNVBAYTAkFVMQwwCgYD 30 | VQQIDANOU1cxDzANBgNVBAcMBlN5ZG5leTEZMBcGA1UECgwQU29mdHdhcmUgQ29t 31 | cGFueTELMAkGA1UECwwCSVQxETAPBgNVBAMMCHR5cGVib290MIIBIjANBgkqhkiG 32 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwK4epBwDXq8FAHCMSVNhQemQ/+uq9aPSYivH 33 | B4fGKIgialg6lbeei8vXjyqq/N0YqY9Gcgq5wTkO/cgpukPJlL+Ykzmu6MdfvFze 34 | nyZWAwwvNmoTnEa27NqgGRha3i5JhSISqFu9lCLjnxwfib6Sb/+FkAsw//7DLVru 35 | uwTpvxWJ94eymO07WnRr3+TCMrmpU6WQ8Zd+lT5FX1TRv83TJXCEs0RC8YxMBQNj 36 | SrH5Z/PSLtlzHbiosPqf0BuN5fF446LkvcsQof4p/j1La6v0nwa9YeQKZni9uHYZ 37 | OVkg5PwqP2VtPT1Jt/OTVDDiq6X0Q7muRrcaBm2w272hFrAVIQIDAQABMA0GCSqG 38 | SIb3DQEBCwUAA4IBAQCHMnNXn505k/TmjgD0DAp0OqpvwU8x3ugdKcnxNcZb2fhZ 39 | bHIi8uIUMowd0k+G+/2HNpEV6scjZxKKjm+YsHtWKlLUjUTJ+9PhFJHwNltzPK5s 40 | CT0EFGuuJYY7/C2Kt86FWDffAxOOB8prTsTk8LmXXwjGcQQGeOwUhI1tzyO9F9tC 41 | p6U1amkUrsCKqSXR6ocYMpDJhenoAgr+9ITyaGxIMCcn3WErzpk70o1Wf5AlFjgJ 42 | 7e62krg/ek01dF/0eUH0i/tO4TSBDrGA7SbU4ndeSE6gyOFW/R0lDUAEamCEBUTd 43 | 7IwBS13z40O253KXoqjzQdiULzF716wbtBimZ5Ym 44 | -----END CERTIFICATE----- 45 | -------------------------------------------------------------------------------- /envoy/trino-envoy.yaml: -------------------------------------------------------------------------------- 1 | admin: 2 | access_log_path: "/tmp/admin_access.log" 3 | address: 4 | socket_address: { address: 0.0.0.0, port_value: 8001 } 5 | 6 | static_resources: 7 | listeners: 8 | - address: 9 | socket_address: 10 | address: 0.0.0.0 11 | port_value: 8453 12 | filter_chains: 13 | - filters: 14 | - name: envoy.filters.network.http_connection_manager 15 | typed_config: 16 | "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager 17 | codec_type: AUTO 18 | stat_prefix: trino_proxy 19 | route_config: 20 | name: local_route 21 | virtual_hosts: 22 | - name: local_service 23 | domains: 24 | - "*" 25 | routes: 26 | - match: 27 | prefix: "/" 28 | route: 29 | cluster: trino_coordinator 30 | timeout: 31 | seconds: 3600 32 | http_filters: 33 | - name: envoy.filters.http.router 34 | typed_config: 35 | "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router 36 | 37 | transport_socket: 38 | name: envoy.transport_sockets.tls 39 | typed_config: 40 | "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext 41 | common_tls_context: 42 | tls_certificates: 43 | - certificate_chain: 44 | filename: /opt/certs/trino-proxy/bundle.crt 45 | private_key: 46 | filename: /opt/certs/trino-proxy/server.rsa.key 47 | 48 | clusters: 49 | - name: trino_coordinator 50 | connect_timeout: 30s 51 | type: logical_dns 52 | lb_policy: round_robin 53 | dns_lookup_family: V4_ONLY 54 | http_protocol_options: {} 55 | load_assignment: 56 | cluster_name: trino_coordinator 57 | endpoints: 58 | - lb_endpoints: 59 | - endpoint: 60 | address: 61 | socket_address: 62 | #host.docker.internal 63 | address: coordinator 64 | port_value: 8443 65 | transport_socket: 66 | name: envoy.transport_sockets.tls 67 | typed_config: 68 | "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext 69 | sni: coordinator 70 | allow_renegotiation: true 71 | common_tls_context: 72 | validation_context: 73 | match_subject_alt_names: 74 | - exact: "coordinator" 75 | trusted_ca: 76 | filename: /opt/certs/trino.crt 77 | 78 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_log/00000000000000000003.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1615860853506,"operation":"MERGE","operationParameters":{"predicate":"(existing_data.`txn_id` = new_data.`txn_id`)","matchedPredicates":"[{\"predicate\":\"(new_data.`deleted` = true)\",\"actionType\":\"delete\"},{\"actionType\":\"update\"}]","notMatchedPredicates":"[{\"actionType\":\"insert\"}]"},"readVersion":2,"isBlindAppend":false,"operationMetrics":{"numTargetRowsCopied":"5","numTargetRowsDeleted":"0","numTargetFilesAdded":"9","executionTimeMs":"0","numTargetRowsInserted":"2","scanTimeMs":"1187","numTargetRowsUpdated":"3","numOutputRows":"10","numSourceRows":"5","numTargetFilesRemoved":"2","rewriteTimeMs":"2460"}}} 2 | {"remove":{"path":"version=2021-03-02/part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet","deletionTimestamp":1615860853505,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"version":"2021-03-02"},"size":2328}} 3 | {"remove":{"path":"version=2021-03-03/part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet","deletionTimestamp":1615860853506,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"version":"2021-03-03"},"size":2143}} 4 | {"add":{"path":"version=2021-03-02/part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet","partitionValues":{"version":"2021-03-02"},"size":2150,"modificationTime":1615860851380,"dataChange":true}} 5 | {"add":{"path":"version=2021-03-03/part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2143,"modificationTime":1615860851379,"dataChange":true}} 6 | {"add":{"path":"version=2021-03-02/part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet","partitionValues":{"version":"2021-03-02"},"size":2206,"modificationTime":1615860851423,"dataChange":true}} 7 | {"add":{"path":"version=2021-03-02/part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet","partitionValues":{"version":"2021-03-02"},"size":2114,"modificationTime":1615860851423,"dataChange":true}} 8 | {"add":{"path":"version=2021-03-02/part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet","partitionValues":{"version":"2021-03-02"},"size":2003,"modificationTime":1615860851460,"dataChange":true}} 9 | {"add":{"path":"version=2021-03-08/part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet","partitionValues":{"version":"2021-03-08"},"size":2260,"modificationTime":1615860851460,"dataChange":true}} 10 | {"add":{"path":"version=2021-03-02/part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet","partitionValues":{"version":"2021-03-02"},"size":2204,"modificationTime":1615860851498,"dataChange":true}} 11 | {"add":{"path":"version=2021-03-08/part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet","partitionValues":{"version":"2021-03-08"},"size":2296,"modificationTime":1615860851498,"dataChange":true}} 12 | {"add":{"path":"version=2021-03-02/part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet","partitionValues":{"version":"2021-03-02"},"size":2130,"modificationTime":1615860851545,"dataChange":true}} 13 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 1>&2 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 48 | echo. 1>&2 49 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 50 | echo location of your Java installation. 1>&2 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 1>&2 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 62 | echo. 1>&2 63 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 64 | echo location of your Java installation. 1>&2 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_log/00000000000000000001.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1615860843389,"operation":"MERGE","operationParameters":{"predicate":"(existing_data.`txn_id` = new_data.`txn_id`)","matchedPredicates":"[{\"predicate\":\"(new_data.`deleted` = true)\",\"actionType\":\"delete\"},{\"actionType\":\"update\"}]","notMatchedPredicates":"[{\"actionType\":\"insert\"}]"},"readVersion":0,"isBlindAppend":false,"operationMetrics":{"numTargetRowsCopied":"0","numTargetRowsDeleted":"0","numTargetFilesAdded":"15","executionTimeMs":"0","numTargetRowsInserted":"15","scanTimeMs":"2197","numTargetRowsUpdated":"0","numOutputRows":"15","numSourceRows":"15","numTargetFilesRemoved":"0","rewriteTimeMs":"3411"}}} 2 | {"add":{"path":"version=2021-03-03/part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2134,"modificationTime":1615860840720,"dataChange":true}} 3 | {"add":{"path":"version=2021-03-03/part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2116,"modificationTime":1615860840720,"dataChange":true}} 4 | {"add":{"path":"version=2021-03-03/part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2143,"modificationTime":1615860840789,"dataChange":true}} 5 | {"add":{"path":"version=2021-03-03/part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2224,"modificationTime":1615860840789,"dataChange":true}} 6 | {"add":{"path":"version=2021-03-03/part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2206,"modificationTime":1615860840853,"dataChange":true}} 7 | {"add":{"path":"version=2021-03-03/part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2224,"modificationTime":1615860840853,"dataChange":true}} 8 | {"add":{"path":"version=2021-03-03/part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2188,"modificationTime":1615860840913,"dataChange":true}} 9 | {"add":{"path":"version=2021-03-03/part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2179,"modificationTime":1615860840913,"dataChange":true}} 10 | {"add":{"path":"version=2021-03-03/part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2152,"modificationTime":1615860840977,"dataChange":true}} 11 | {"add":{"path":"version=2021-03-03/part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2197,"modificationTime":1615860840977,"dataChange":true}} 12 | {"add":{"path":"version=2021-03-03/part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2170,"modificationTime":1615860841037,"dataChange":true}} 13 | {"add":{"path":"version=2021-03-03/part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2125,"modificationTime":1615860841037,"dataChange":true}} 14 | {"add":{"path":"version=2021-03-03/part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2197,"modificationTime":1615860841100,"dataChange":true}} 15 | {"add":{"path":"version=2021-03-03/part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2161,"modificationTime":1615860841100,"dataChange":true}} 16 | {"add":{"path":"version=2021-03-03/part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet","partitionValues":{"version":"2021-03-03"},"size":2179,"modificationTime":1615860841157,"dataChange":true}} 17 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_log/00000000000000000002.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1615860848157,"operation":"MERGE","operationParameters":{"predicate":"(existing_data.`txn_id` = new_data.`txn_id`)","matchedPredicates":"[{\"predicate\":\"(new_data.`deleted` = true)\",\"actionType\":\"delete\"},{\"actionType\":\"update\"}]","notMatchedPredicates":"[{\"actionType\":\"insert\"}]"},"readVersion":1,"isBlindAppend":false,"operationMetrics":{"numTargetRowsCopied":"0","numTargetRowsDeleted":"0","numTargetFilesAdded":"15","executionTimeMs":"0","numTargetRowsInserted":"15","scanTimeMs":"1020","numTargetRowsUpdated":"0","numOutputRows":"15","numSourceRows":"15","numTargetFilesRemoved":"0","rewriteTimeMs":"2369"}}} 2 | {"add":{"path":"version=2021-03-05/part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2125,"modificationTime":1615860846076,"dataChange":true}} 3 | {"add":{"path":"version=2021-03-05/part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2206,"modificationTime":1615860846076,"dataChange":true}} 4 | {"add":{"path":"version=2021-03-05/part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2197,"modificationTime":1615860846118,"dataChange":true}} 5 | {"add":{"path":"version=2021-03-05/part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2161,"modificationTime":1615860846118,"dataChange":true}} 6 | {"add":{"path":"version=2021-03-05/part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2233,"modificationTime":1615860846170,"dataChange":true}} 7 | {"add":{"path":"version=2021-03-05/part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2188,"modificationTime":1615860846170,"dataChange":true}} 8 | {"add":{"path":"version=2021-03-05/part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2197,"modificationTime":1615860846218,"dataChange":true}} 9 | {"add":{"path":"version=2021-03-05/part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2134,"modificationTime":1615860846218,"dataChange":true}} 10 | {"add":{"path":"version=2021-03-05/part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2188,"modificationTime":1615860846259,"dataChange":true}} 11 | {"add":{"path":"version=2021-03-05/part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2143,"modificationTime":1615860846259,"dataChange":true}} 12 | {"add":{"path":"version=2021-03-05/part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2215,"modificationTime":1615860846300,"dataChange":true}} 13 | {"add":{"path":"version=2021-03-05/part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2197,"modificationTime":1615860846300,"dataChange":true}} 14 | {"add":{"path":"version=2021-03-05/part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2206,"modificationTime":1615860846340,"dataChange":true}} 15 | {"add":{"path":"version=2021-03-05/part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2206,"modificationTime":1615860846340,"dataChange":true}} 16 | {"add":{"path":"version=2021-03-05/part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet","partitionValues":{"version":"2021-03-05"},"size":2197,"modificationTime":1615860846380,"dataChange":true}} 17 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_manifest/version=2021-03-05/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-03/part-00029-6916b5b7-e668-4a52-bc94-8a904d218992.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-02/part-00001-963470e9-eac1-487a-9bda-90a8e65ebcb0.c000.snappy.parquet 3 | file:/opt/data/output/activity/version=2021-03-03/part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet 4 | file:/opt/data/output/activity/version=2021-03-05/part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet 5 | file:/opt/data/output/activity/version=2021-03-03/part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet 6 | file:/opt/data/output/activity/version=2021-03-05/part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet 7 | file:/opt/data/output/activity/version=2021-03-03/part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet 8 | file:/opt/data/output/activity/version=2021-03-03/part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet 9 | file:/opt/data/output/activity/version=2021-03-03/part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet 10 | file:/opt/data/output/activity/version=2021-03-05/part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet 11 | file:/opt/data/output/activity/version=2021-03-05/part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet 12 | file:/opt/data/output/activity/version=2021-03-03/part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet 13 | file:/opt/data/output/activity/version=2021-03-03/part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet 14 | file:/opt/data/output/activity/version=2021-03-03/part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet 15 | file:/opt/data/output/activity/version=2021-03-03/part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet 16 | file:/opt/data/output/activity/version=2021-03-03/part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet 17 | file:/opt/data/output/activity/version=2021-03-03/part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet 18 | file:/opt/data/output/activity/version=2021-03-03/part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet 19 | file:/opt/data/output/activity/version=2021-03-03/part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet 20 | file:/opt/data/output/activity/version=2021-03-05/part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet 21 | file:/opt/data/output/activity/version=2021-03-05/part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet 22 | file:/opt/data/output/activity/version=2021-03-05/part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet 23 | file:/opt/data/output/activity/version=2021-03-05/part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet 24 | file:/opt/data/output/activity/version=2021-03-03/part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet 25 | file:/opt/data/output/activity/version=2021-03-05/part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet 26 | file:/opt/data/output/activity/version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet 27 | file:/opt/data/output/activity/version=2021-03-05/part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet 28 | file:/opt/data/output/activity/version=2021-03-05/part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet 29 | file:/opt/data/output/activity/version=2021-03-05/part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet 30 | file:/opt/data/output/activity/version=2021-03-05/part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet 31 | file:/opt/data/output/activity/version=2021-03-05/part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet 32 | file:/opt/data/output/activity/version=2021-03-05/part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet 33 | -------------------------------------------------------------------------------- /security/rules/rules.json: -------------------------------------------------------------------------------- 1 | { 2 | "catalogs": [ 3 | { 4 | "user": "(admin|dev)", 5 | "catalog": "(mysql|system|hive|memory|tpch|tpcds|jmx)", 6 | "allow": "all" 7 | }, 8 | { 9 | "catalog": "(mysql|system|hive|memory|tpch|tpcds|jmx)", 10 | "allow": "all" 11 | }, 12 | { 13 | "group": "finance", 14 | "catalog": "finance", 15 | "allow": true 16 | }, 17 | { 18 | "user": "user1", 19 | "catalog": "hive", 20 | "allow": "read-only" 21 | }, 22 | { 23 | "catalog": "system", 24 | "allow": "none" 25 | } 26 | ], 27 | "schemas": [ 28 | { 29 | "schema": "information_schema", 30 | "owner": true 31 | }, 32 | { 33 | "user": "(admin|skhatri|dev)", 34 | "schema": ".*", 35 | "owner": true 36 | }, 37 | { 38 | "user": "skhatri", 39 | "schema": "(finance|fitness|information_schema)", 40 | "owner": true 41 | }, 42 | { 43 | "user": "user1", 44 | "schema": "finance", 45 | "owner": true 46 | }, 47 | { 48 | "user": "user2", 49 | "schema": "fitness", 50 | "owner": true 51 | }, 52 | { 53 | "user": "user3", 54 | "schema": "finance", 55 | "owner": true 56 | }, 57 | { 58 | "catalog": "default", 59 | "schema": "default", 60 | "owner": true 61 | } 62 | ], 63 | "tables": [ 64 | { 65 | "user": "(admin|dev)", 66 | "privileges": [ 67 | "SELECT", 68 | "INSERT", 69 | "DELETE", 70 | "OWNERSHIP" 71 | ] 72 | }, 73 | { 74 | "schema": "information_schema", 75 | "table": ".*", 76 | "privileges": [ 77 | "SELECT" 78 | ] 79 | }, 80 | { 81 | "user": "user1", 82 | "catalog": "hive", 83 | "schema": "finance", 84 | "table": ".*", 85 | "privileges": [ 86 | "SELECT" 87 | ] 88 | }, 89 | { 90 | "user": "user2", 91 | "catalog": "hive", 92 | "schema": "fitness", 93 | "table": ".*", 94 | "privileges": [ 95 | "SELECT" 96 | ] 97 | }, 98 | { 99 | "user": "user3", 100 | "catalog": "hive", 101 | "schema": "finance", 102 | "table": "activity", 103 | "privileges": [ 104 | "SELECT" 105 | ] 106 | }, 107 | { 108 | "user": "skhatri", 109 | "catalog": "hive", 110 | "schema": "(finance|fitness)", 111 | "table": ".*", 112 | "privileges": [ 113 | "SELECT" 114 | ] 115 | }, 116 | { 117 | "user": "banned_user", 118 | "privileges": [] 119 | }, 120 | { 121 | "catalog": "default", 122 | "schema": "hr", 123 | "table": "employee", 124 | "privileges": [ 125 | "SELECT" 126 | ], 127 | "filter": "user = current_user", 128 | "filter_environment": { 129 | "user": "admin" 130 | } 131 | }, 132 | { 133 | "catalog": "default", 134 | "schema": "default", 135 | "table": ".*", 136 | "privileges": [ 137 | "SELECT" 138 | ], 139 | "columns": [ 140 | { 141 | "name": "address", 142 | "allow": false 143 | }, 144 | { 145 | "name": "SSN", 146 | "mask": "'XXX-XX-' + substring(credit_card, -4)", 147 | "mask_environment": { 148 | "user": "admin" 149 | } 150 | } 151 | ] 152 | } 153 | ], 154 | "impersonation": [ 155 | { 156 | "original_user": "(admin|dev)", 157 | "new_user": ".*", 158 | "allow": true 159 | }, 160 | { 161 | "original_user": "skhatri", 162 | "new_user": "admin", 163 | "allow": false 164 | }, 165 | { 166 | "original_user": "skhatri", 167 | "new_user": ".*", 168 | "allow": true 169 | } 170 | ] 171 | } 172 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_manifest/version=2021-03-08/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-03/part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-02/part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet 3 | file:/opt/data/output/activity/version=2021-03-05/part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet 4 | file:/opt/data/output/activity/version=2021-03-02/part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet 5 | file:/opt/data/output/activity/version=2021-03-03/part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet 6 | file:/opt/data/output/activity/version=2021-03-05/part-00006-139f6dc0-18c3-4ad2-a219-c7b2d408b75a.c000.snappy.parquet 7 | file:/opt/data/output/activity/version=2021-03-08/part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet 8 | file:/opt/data/output/activity/version=2021-03-03/part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet 9 | file:/opt/data/output/activity/version=2021-03-03/part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet 10 | file:/opt/data/output/activity/version=2021-03-03/part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet 11 | file:/opt/data/output/activity/version=2021-03-02/part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet 12 | file:/opt/data/output/activity/version=2021-03-05/part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet 13 | file:/opt/data/output/activity/version=2021-03-05/part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet 14 | file:/opt/data/output/activity/version=2021-03-03/part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet 15 | file:/opt/data/output/activity/version=2021-03-03/part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet 16 | file:/opt/data/output/activity/version=2021-03-03/part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet 17 | file:/opt/data/output/activity/version=2021-03-03/part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet 18 | file:/opt/data/output/activity/version=2021-03-03/part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet 19 | file:/opt/data/output/activity/version=2021-03-03/part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet 20 | file:/opt/data/output/activity/version=2021-03-03/part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet 21 | file:/opt/data/output/activity/version=2021-03-02/part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet 22 | file:/opt/data/output/activity/version=2021-03-03/part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet 23 | file:/opt/data/output/activity/version=2021-03-05/part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet 24 | file:/opt/data/output/activity/version=2021-03-05/part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet 25 | file:/opt/data/output/activity/version=2021-03-05/part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet 26 | file:/opt/data/output/activity/version=2021-03-05/part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet 27 | file:/opt/data/output/activity/version=2021-03-03/part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet 28 | file:/opt/data/output/activity/version=2021-03-05/part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet 29 | file:/opt/data/output/activity/version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet 30 | file:/opt/data/output/activity/version=2021-03-03/part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet 31 | file:/opt/data/output/activity/version=2021-03-05/part-00003-2d854a60-2e29-4233-a499-3cd6de3ae4c1.c000.snappy.parquet 32 | file:/opt/data/output/activity/version=2021-03-02/part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet 33 | file:/opt/data/output/activity/version=2021-03-05/part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet 34 | file:/opt/data/output/activity/version=2021-03-05/part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet 35 | file:/opt/data/output/activity/version=2021-03-02/part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet 36 | file:/opt/data/output/activity/version=2021-03-05/part-00186-9ce386d1-1144-473e-a7c1-efa1c200c0dd.c000.snappy.parquet 37 | file:/opt/data/output/activity/version=2021-03-05/part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet 38 | file:/opt/data/output/activity/version=2021-03-05/part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet 39 | file:/opt/data/output/activity/version=2021-03-08/part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet 40 | -------------------------------------------------------------------------------- /hive/output/activity/_delta_manifest/version=2021-03-09/manifest: -------------------------------------------------------------------------------- 1 | file:/opt/data/output/activity/version=2021-03-03/part-00113-1d0971f0-d6a9-437c-b2da-74bf2f21a407.c000.snappy.parquet 2 | file:/opt/data/output/activity/version=2021-03-02/part-00030-f603a922-3832-4244-9f74-9470404c6413.c000.snappy.parquet 3 | file:/opt/data/output/activity/version=2021-03-05/part-00133-b2520158-0913-4304-85ab-42c958c068c7.c000.snappy.parquet 4 | file:/opt/data/output/activity/version=2021-03-09/part-00161-b0c083ee-3249-4e6d-954e-5c8f8bdc3ffd.c000.snappy.parquet 5 | file:/opt/data/output/activity/version=2021-03-02/part-00098-1ad78135-e8e3-4414-b6ed-3bfb88147fae.c000.snappy.parquet 6 | file:/opt/data/output/activity/version=2021-03-09/part-00136-074b92af-c0a2-4fa8-a365-e933814c3283.c000.snappy.parquet 7 | file:/opt/data/output/activity/version=2021-03-03/part-00066-1934faf6-e940-44c4-a9d9-314ccd20b64a.c000.snappy.parquet 8 | file:/opt/data/output/activity/version=2021-03-08/part-00124-e340283e-9c17-4bc8-89ed-5ee734fddda6.c000.snappy.parquet 9 | file:/opt/data/output/activity/version=2021-03-03/part-00141-321f3e3e-47ff-4f74-b0ea-7d235d28cdf3.c000.snappy.parquet 10 | file:/opt/data/output/activity/version=2021-03-03/part-00020-a249f9b8-ae26-499e-9e3a-e7f0bfd915f5.c000.snappy.parquet 11 | file:/opt/data/output/activity/version=2021-03-03/part-00185-78d9df69-5845-41dc-8881-de9b417d4c1b.c000.snappy.parquet 12 | file:/opt/data/output/activity/version=2021-03-02/part-00014-23c5ff71-e325-4251-932c-cf717a6d85e2.c000.snappy.parquet 13 | file:/opt/data/output/activity/version=2021-03-05/part-00132-f396e0b6-7dcb-45dd-954c-1a954f8b557a.c000.snappy.parquet 14 | file:/opt/data/output/activity/version=2021-03-05/part-00051-499d7cb7-d332-443f-91cc-51048ae795de.c000.snappy.parquet 15 | file:/opt/data/output/activity/version=2021-03-03/part-00053-ebb629b0-4d95-4344-a4b8-82595642384f.c000.snappy.parquet 16 | file:/opt/data/output/activity/version=2021-03-05/part-00003-ef712e17-8550-4ce6-82ab-b239493248c7.c000.snappy.parquet 17 | file:/opt/data/output/activity/version=2021-03-03/part-00045-fa75f80a-5886-40e9-aada-4e32d60a0578.c000.snappy.parquet 18 | file:/opt/data/output/activity/version=2021-03-03/part-00086-d413213e-5049-4607-b13b-13c8156eb862.c000.snappy.parquet 19 | file:/opt/data/output/activity/version=2021-03-03/part-00014-88f1872b-3b29-46d3-ba1d-4a145d42a6ae.c000.snappy.parquet 20 | file:/opt/data/output/activity/version=2021-03-03/part-00030-c4694389-4033-4f7c-b7cc-fd0fada3c425.c000.snappy.parquet 21 | file:/opt/data/output/activity/version=2021-03-03/part-00126-6adf0139-c00b-4040-99a6-724796c662d9.c000.snappy.parquet 22 | file:/opt/data/output/activity/version=2021-03-03/part-00107-10c0a379-c482-4dd6-8f61-b7b1da1e8547.c000.snappy.parquet 23 | file:/opt/data/output/activity/version=2021-03-02/part-00056-bedf4965-e4b2-49d4-a5a6-78c2ae41ccec.c000.snappy.parquet 24 | file:/opt/data/output/activity/version=2021-03-03/part-00089-4ccc9709-ab8e-48c8-b054-eabbd6cc85ef.c000.snappy.parquet 25 | file:/opt/data/output/activity/version=2021-03-05/part-00150-544fd9c3-a737-4ef0-9bf9-0d5387d46fb2.c000.snappy.parquet 26 | file:/opt/data/output/activity/version=2021-03-09/part-00140-9875e382-5db0-4bc8-b9aa-0469bdbd602d.c000.snappy.parquet 27 | file:/opt/data/output/activity/version=2021-03-05/part-00112-7b20db5d-2127-443c-ac44-70cd849b2a71.c000.snappy.parquet 28 | file:/opt/data/output/activity/version=2021-03-05/part-00186-0e6a433c-7935-4570-ad13-8652f9aae49e.c000.snappy.parquet 29 | file:/opt/data/output/activity/version=2021-03-05/part-00192-7c6cea4d-b77e-4d11-b7b3-9bf462817f55.c000.snappy.parquet 30 | file:/opt/data/output/activity/version=2021-03-05/part-00047-f9f6f027-db6a-43cb-b855-24c04a27c18b.c000.snappy.parquet 31 | file:/opt/data/output/activity/version=2021-03-03/part-00175-d87f56bf-2b82-4961-8dd3-9fd4785daf0d.c000.snappy.parquet 32 | file:/opt/data/output/activity/version=2021-03-05/part-00071-c4b23a18-ceea-45be-a8d0-47a75db52d1b.c000.snappy.parquet 33 | file:/opt/data/output/activity/version=2021-03-02/part-00000-6ac5eefe-c8a8-4ab1-a17a-0e9a693a140d.c000.snappy.parquet 34 | file:/opt/data/output/activity/version=2021-03-03/part-00029-7adeb8b7-1389-4f2d-88cb-b04ba1257e29.c000.snappy.parquet 35 | file:/opt/data/output/activity/version=2021-03-02/part-00031-3b0afc55-2bb2-44a4-ae8a-70f7ffbc05af.c000.snappy.parquet 36 | file:/opt/data/output/activity/version=2021-03-05/part-00044-2f7b0b13-b3a4-4b54-aeb7-92f059a2ded7.c000.snappy.parquet 37 | file:/opt/data/output/activity/version=2021-03-05/part-00006-df8a68fa-9d89-4693-9c02-854b1a2ce4d5.c000.snappy.parquet 38 | file:/opt/data/output/activity/version=2021-03-05/part-00113-5b933a76-2485-4213-ba39-f91d441402bd.c000.snappy.parquet 39 | file:/opt/data/output/activity/version=2021-03-02/part-00161-3d35960c-0161-4603-a0ae-df6b5003b236.c000.snappy.parquet 40 | file:/opt/data/output/activity/version=2021-03-05/part-00038-b8fce000-b8a2-49fe-813f-660fd10d218e.c000.snappy.parquet 41 | file:/opt/data/output/activity/version=2021-03-05/part-00120-f42eb863-5397-4cf4-a393-b24390d9cf34.c000.snappy.parquet 42 | file:/opt/data/output/activity/version=2021-03-08/part-00079-c73b3b8a-f75c-4251-90eb-c6f5334dc61f.c000.snappy.parquet 43 | -------------------------------------------------------------------------------- /trino-ext-authz/src/main/java/io/trino/plugin/base/security/AuditLoggingEventListenerFactory.java: -------------------------------------------------------------------------------- 1 | package io.trino.plugin.base.security; 2 | 3 | import io.trino.spi.eventlistener.EventListener; 4 | import io.trino.spi.eventlistener.EventListenerFactory; 5 | import io.trino.spi.eventlistener.QueryCompletedEvent; 6 | import io.trino.spi.eventlistener.QueryCreatedEvent; 7 | import io.trino.spi.eventlistener.QueryFailureInfo; 8 | import io.trino.spi.eventlistener.SplitCompletedEvent; 9 | 10 | import java.time.Duration; 11 | import java.time.ZoneId; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.TreeMap; 16 | import java.util.stream.Collectors; 17 | 18 | public class AuditLoggingEventListenerFactory implements EventListenerFactory { 19 | @Override 20 | public String getName() { 21 | return "auditlog"; 22 | } 23 | 24 | @Override 25 | public EventListener create(Map config) { 26 | System.out.println("event listener input " + config); 27 | return new LoggingEventListener(); 28 | } 29 | } 30 | 31 | class AttributeLogger { 32 | private Map attributes = new TreeMap<>(); 33 | 34 | private AttributeLogger() { 35 | } 36 | 37 | public AttributeLogger withAttribute(String name, Object value) { 38 | this.attributes.put(name, value); 39 | return this; 40 | } 41 | 42 | public void log() { 43 | List snippets = new ArrayList<>(); 44 | for (Map.Entry entry : attributes.entrySet()) { 45 | snippets.add("\"".concat(entry.getKey()).concat("\":\"").concat(entry.getValue().toString()).concat("\"")); 46 | } 47 | System.out.println(snippets.stream().collect(Collectors.joining(",", "{", "}"))); 48 | } 49 | 50 | public static AttributeLogger newInstance() { 51 | return new AttributeLogger(); 52 | } 53 | } 54 | 55 | class LoggingEventListener implements EventListener { 56 | @Override 57 | public void queryCreated(QueryCreatedEvent queryCreatedEvent) { 58 | AttributeLogger.newInstance() 59 | .withAttribute("phase", "created") 60 | .withAttribute("query", queryCreatedEvent.getMetadata().getQuery()) 61 | .withAttribute("query_id", queryCreatedEvent.getMetadata().getQueryId()) 62 | .withAttribute("catalog", queryCreatedEvent.getContext().getCatalog().orElse("")) 63 | .withAttribute("schema", queryCreatedEvent.getContext().getSchema().orElse("")) 64 | .withAttribute("principal", queryCreatedEvent.getContext().getPrincipal().orElse("")) 65 | .withAttribute("user", queryCreatedEvent.getContext().getUser()) 66 | .withAttribute("time", queryCreatedEvent.getCreateTime().atZone(ZoneId.systemDefault())) 67 | .log(); 68 | } 69 | 70 | 71 | @Override 72 | public void queryCompleted(QueryCompletedEvent queryCompletedEvent) { 73 | AttributeLogger builder = AttributeLogger.newInstance() 74 | .withAttribute("phase", "completed") 75 | .withAttribute("query", queryCompletedEvent.getMetadata().getQuery()) 76 | .withAttribute("query_id", queryCompletedEvent.getMetadata().getQueryId()) 77 | .withAttribute("catalog", queryCompletedEvent.getContext().getCatalog().orElse("")) 78 | .withAttribute("schema", queryCompletedEvent.getContext().getSchema().orElse("")) 79 | .withAttribute("principal", queryCompletedEvent.getContext().getPrincipal().orElse("")) 80 | .withAttribute("user", queryCompletedEvent.getContext().getUser()) 81 | .withAttribute("time", queryCompletedEvent.getCreateTime().atZone(ZoneId.systemDefault())) 82 | .withAttribute("execution_time", queryCompletedEvent.getStatistics().getExecutionTime().orElse(Duration.ZERO).toMillis()) 83 | .withAttribute("cpu_time", queryCompletedEvent.getStatistics().getCpuTime().toMillis()) 84 | .withAttribute("analysis_time", queryCompletedEvent.getStatistics().getAnalysisTime().orElse(Duration.ZERO).toMillis()) 85 | .withAttribute("rows", queryCompletedEvent.getStatistics().getOutputRows()); 86 | 87 | if (queryCompletedEvent.getFailureInfo().isPresent()) { 88 | QueryFailureInfo failure = queryCompletedEvent.getFailureInfo().get(); 89 | builder.withAttribute("error_code", failure.getErrorCode().getName()); 90 | builder.withAttribute("message", failure.getFailureMessage().orElse("")); 91 | builder.withAttribute("failure_type", failure.getFailureType().orElse("")); 92 | builder.withAttribute("failure_host", failure.getFailureHost().orElse("")); 93 | builder.withAttribute("status", "failure"); 94 | } else { 95 | builder.withAttribute("status", "success"); 96 | } 97 | 98 | builder.log(); 99 | } 100 | 101 | @Override 102 | public void splitCompleted(SplitCompletedEvent splitCompletedEvent) { 103 | 104 | } 105 | } -------------------------------------------------------------------------------- /certs/trino.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: trino 3 | localKeyID: 54 69 6D 65 20 31 37 32 35 36 32 35 32 35 33 38 36 31 4 | Key Attributes: 5 | -----BEGIN ENCRYPTED PRIVATE KEY----- 6 | MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQCRtXn8wSW4I/oit1 7 | MrJ6HgICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEF8TZdV/EXncguAx 8 | yvywzgkEggcQ6PFFp6gs4gmV1UpfyLZ1CJ5DtL2c7vZEbrBp1q/ADxcaYXukXUeH 9 | /XzicDdTiRGJ82YSMh/AHjL6nCcM2YpLsV78kT+FURJ9JwtHYwT10EzDTXteoHvc 10 | GhXu4eAr9DesxH/8HMeqOcoO5yVYgfe5DHHaAS4svvdvcTEzeDuMH3UoC6J8Mo66 11 | Zt+84hq+31b6ypld3EJUMORnHdr+NAsk1uscUaLTZxVgZtxNXpB/oPCOisRNijVC 12 | hmV1McmsrHsJvaNEHpIpaUMAeS0nhekl4ddwHgl98ufsJ+hEr/b+Qim5goxkwi6h 13 | 1/a9jIfNOAVmNXXxk8mo9jbArF8hmnVwMNxKImLC7u+PXQGHM8Z4/VdFRuwqYs4g 14 | Kk4z88to01IJ1PsL+/Cdp5e0ykKm5RTcTRVIaWk94dUKTGAlJ/TCBhFUlmCpXCiK 15 | g2GcO6vrvTKaiVOfmGtrE/YUnxy+UpQbbD2rVGs4nfxEPERnY8HIYCoP/QCeJESx 16 | d3HeQYD6a8BSHuZQkNlfSOmKDdCmK1tJME8r1UxFKAKwFiE8O+n6afvYDiI9wWFz 17 | bxwqHPdRKmP1dFMcOQ6nlVFLfWoUzV/4HHcXkQzd0Ksn2nyvMYqGfyfQfP3eOA1b 18 | BkjLwaIBDa+IlEnlYtA6Qo2tYd6q52UriHjR+9iEu4Tqkjpe5nExYMEhrK7SApEh 19 | P3H8wBsCqOZ5fNOLw+TfMVg0yOxtR4ZA9ZBgy0hrK1Xa4HIuZZRexDohOt+ozKs2 20 | DDX87ZW+EKBHydk871bw1ZizNL60E83aDWpLlJ0HLLt6K7VnihAHOjHg30IqgM+E 21 | 4d/c25XYrBvAjbU+nwk8Rki8wnMXBiNSYJFC3Nn/KPyVIRi4OhdddA1QqBxpudnA 22 | hZUCUW7Bl5XTd4KkbaxoteAkaGuDU3+hcDeDC6etIAONHaT+GpjIHDl6HP3lssjH 23 | JVafRe859J+iw7iWbt7Eh8LwOK+DLQ/6lwghaG/hIChK6pXtyfMVKpMFBP8eHmSD 24 | JEI55k0gTAcgrDSPNWT15QfOw1qDqjaZM1GGTzYHjn59DS3MFdP7r8wGmgE3BDDv 25 | TdeTExjfnWAt1d4laZx7iXMTsxMR0vdjD7G1YevqbKhOxV67wHNMV+sWZK4K/NcU 26 | QNBa/KKFDYEWkM42M2T1CqPOmg9RM/Jz9V+61DvBoIcDbuEa/5wjS4kS/cgZuOE1 27 | Is7So5dEvJEXho4D1853NnSsKwe9ET961xcQvMVk+8AwGVbkw5IqTfoibVpWfUHL 28 | QWdP1T4Cb+KbUBRPbxhSDZnfSnxRSY3pWW5mWtIEN130bbUoB7NEcs7O7ZyXWRax 29 | evTsQ/6/uDQV20e1knGtg5NaNDedGRAlSLN5oaordhrqmhGpHtklKkDfQsmp/NGI 30 | 99+Ue06yTT/93fUifjlx1gpbdwMe/dO1GIb8syNttL4T5u4UbZOybdsb+XcAUkYL 31 | 3BtakZhvw+3Mq7SQuO1UqOJx2Xad6AGGxLTlYh6HrTl+r3Zr8OLx5La/HxPbpxt+ 32 | xzjBXM+jsbdF3vaUrRZ7SC7zXaoF0MSaACpzq3RTItSMEeqTmHVWyWEpoRrIysfz 33 | 1Io3+jCW2anFYCivHfLmEXPEJo2QWzfjk1Kmjw9wLeqi9Oamp/iUIfFsR/b/Yrqv 34 | fZDDu8mA7bqnhyYe23SglLVTtwewfmkfK14Nt6SphtqL46WENZB1yDjiPXGHRsIm 35 | GA+llX2rwbsDRtXpGFPM8yCzkvdyNcshGIlF24/BJuoicz9okgW9YSfvPIKfN/Fp 36 | SZG1eSfNzaLG0d9GyaiJQFZhsaD4wkl5gI/0jBvRTiCnsJBSBIBsQqjswOzbBaul 37 | pGA0wya1NVkhfJDkWjADp935MOaiws7pIqvHR99l7XVV+3l7GOU39jSgOejXjp69 38 | qWBUFjTRUzwvdVoqPsrYFJbF/8889ey04Z6hXkfBvsnVpiV86RJ/vFcGHjpZCkBi 39 | p4tblqN1jQ2ksWOWRgba85gHvzYsLhslq+YsDcO8WEYBmKNNCohbL+BztKN4oMQm 40 | i77850fuE1+FoIjkDiEoXF8RdzI6upzCSEgT/OphpvE5Q7c+hULlxuXiIjJ+GP3R 41 | s3mz9lvwwt1UwyncFdXOj0I12o94tyJzMdR+FNy44HqdyO+pMaA1RaGKPn8GArFb 42 | FpQbR+SnSry8wNZx1Gv5iIt7jWtDXStEjEweG7iq1S7X4wB0B/U16j40jlYlQRfA 43 | Df6VgI66PE3x1rM+AKBlCPKPbI4tPMdTtjhQB0MKBQorWeDmALIS61DdpTeiMZVK 44 | QyuwXn19Ye0fc+mDZsJzibLNR/k3GmRqAziImq3Zvaw8uzJtsqVlHqjH5cIqYZUm 45 | J+QQbQCHJkZ6GpkJyCc79u1mMwmRLrrobw/WXPG+DybE0t5aVtPtEJg= 46 | -----END ENCRYPTED PRIVATE KEY----- 47 | Bag Attributes 48 | friendlyName: trino 49 | localKeyID: 54 69 6D 65 20 31 37 32 35 36 32 35 32 35 33 38 36 31 50 | subject=C=AU, ST=NSW, L=Sydney, O=dataco, OU=datalake, CN=coordinator 51 | issuer=C=AU, ST=NSW, L=Sydney, O=dataco, OU=datalake, CN=coordinator 52 | -----BEGIN CERTIFICATE----- 53 | MIIFkTCCA/mgAwIBAgIJAMU+8n5x4qiDMA0GCSqGSIb3DQEBDAUAMGYxCzAJBgNV 54 | BAYTAkFVMQwwCgYDVQQIEwNOU1cxDzANBgNVBAcTBlN5ZG5leTEPMA0GA1UEChMG 55 | ZGF0YWNvMREwDwYDVQQLEwhkYXRhbGFrZTEUMBIGA1UEAxMLY29vcmRpbmF0b3Iw 56 | HhcNMjQwOTA2MTIyMDM5WhcNMjQxMjA1MTIyMDM5WjBmMQswCQYDVQQGEwJBVTEM 57 | MAoGA1UECBMDTlNXMQ8wDQYDVQQHEwZTeWRuZXkxDzANBgNVBAoTBmRhdGFjbzER 58 | MA8GA1UECxMIZGF0YWxha2UxFDASBgNVBAMTC2Nvb3JkaW5hdG9yMIIBojANBgkq 59 | hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAr6B3VNktNl76/1Mkmf1s0CSgzDedBXj5 60 | 9zQDKRSRSvY9zZm7AQGNGM34mXh6QDWp7OtjVUfu6XVuL5uJoCBFBU1nnZs+hjBn 61 | DaSpeEFA1IotcgBpFuhNgcEOByNWxxz+qFF3iM+jhGZxC0K2TfU05lUTUw2mY3Yv 62 | GnZI8pa889K+f0qB9wu+rxNtJm+eHfWxoK20HMpcANutyxSgSPcJiMtlDjQPz+MC 63 | DWGOwwsH/qnG7pU+TJDFTP4OTuCmTOi8KcHufMhnMwbR2UUS5aEJ1/XwD+a/+QNl 64 | AjkhuqMqt61wuqavbzu/AYe7y/3CHE8YaaDfDdVBR9TFP9oP/yPdk2CE1VE9up/C 65 | Z5yj4F5+iW6/Dqm8DXUNhWybxZNX5n4sbiyb0STTgiaIzkPiILAWIHgc3Shf/bn1 66 | jADnEA62ClRz/C9M+DptJBAZ9+iiAVGG0Bxrem3R4lDDC2g0+sbW7MNmE9MGyyYJ 67 | lJnUR1486n67EQXbkJmbWrZS4aOZtMUbAgMBAAGjggFAMIIBPDAdBgNVHQ4EFgQU 68 | EJRt5JrfRBeDZ8kMORYvOOP5l9YwggEZBgNVHREEggEQMIIBDIILY29vcmRpbmF0 69 | b3KCEmNvb3JkaW5hdG9yLnByZXN0b4IWY29vcmRpbmF0b3IucHJlc3RvLnN2Y4Ik 70 | Y29vcmRpbmF0b3IucHJlc3RvLnN2Yy5jbHVzdGVyLmxvY2FsghRjb29yZGluYXRv 71 | ci1oZWFkbGVzc4IbY29vcmRpbmF0b3ItaGVhZGxlc3MucHJlc3Rvgh9jb29yZGlu 72 | YXRvci1oZWFkbGVzcy5wcmVzdG8uc3Zjgi1jb29yZGluYXRvci1oZWFkbGVzcy5w 73 | cmVzdG8uc3ZjLmNsdXN0ZXIubG9jYWyCCWxvY2FsaG9zdIILdHJpbm8tcHJveHmH 74 | BH8AAAGHBMCoQAWHBMCoQAYwDQYJKoZIhvcNAQEMBQADggGBAEAIYPxqeeG6R2dk 75 | MS84HDoSmsnzgISD6hwvECsyFk5jtvE97aDSWXyOwVMc1r9jOfofCuZ1fhU5nRkj 76 | RJDfwiAQ1B/UJHZf/UfgFzZlaUMetJpCMNElaYUfN13kDh7uGsD9Zg6LjkTS9LX4 77 | ggDwinCpDKQmWWQZEN6a1ayMH0l4WUqUwmr4dZ0ndFxIh0ZO1+iy8z2x89+EWdvQ 78 | 2E3DChy8gIMnirIF0itJz+75FB34MHdnnNCeMTurfXV0HCjiXlIUloLncMyyOtaG 79 | vlv49dy+HSCXvu1LHiAp686WfyLMJgfGaTVSkMHCX2m1inZneskUVOH776jB1Z9T 80 | lUq4a+4M8MAzkIdhYIJKQQ5n3kXaWdT1quZvsB85lG75+a+rwxtoNB9ALRRHSTJs 81 | 9IYmGPCaqHyToRj1ZBVWyzrEIEAofO+yNHoo4N+VR95SJ+w8VKlDpw+Iypb3GJeT 82 | GNwNKGTAn+w7Zr3kVLC3lRufKxUf1seSxw/S6KjcQUQDcLmmwA== 83 | -----END CERTIFICATE----- 84 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '4.2' 2 | volumes: 3 | hivedb: {} 4 | coordinatordb: {} 5 | workerdb: {} 6 | supersetdb: {} 7 | services: 8 | postgres: 9 | image: ${REPO}postgres:9.4 10 | container_name: postgres 11 | environment: 12 | - POSTGRES_USER=postgres 13 | - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} 14 | - POSTGRES_DB=postgres 15 | volumes: 16 | - hivedb:/var/lib/postgresql/data 17 | - ./postgres/scripts:/docker-entrypoint-initdb.d/ 18 | ports: 19 | - "5432:5432" 20 | networks: 21 | ntrino: 22 | aliases: 23 | - postgres 24 | healthcheck: 25 | test: ["CMD-SHELL", "pg_isready -U postgres"] 26 | interval: "20s" 27 | timeout: "20s" 28 | retries: 3 29 | 30 | hive: 31 | #use this if you do not want to build your own 32 | image: skhatri/apache-hive:3.1.3 33 | #building your own - download apache-hive and hadoop into hive folder. check Dockerfile 34 | #build: 35 | # context: . 36 | # dockerfile: hive/Dockerfile 37 | container_name: hive 38 | environment: 39 | - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} 40 | - STORE_KEY=${STORE_KEY} 41 | - STORE_SECRET=${STORE_SECRET} 42 | volumes: 43 | - ./hive/conf/hive-site.xml:/opt/app/apache-hive-3.1.3-bin/conf/hive-site.xml 44 | - ./hive/data:/tmp/hive 45 | #copy output data directory from https://github.com/skhatri/spark-delta-by-example project 46 | - ./hive/output:/opt/data/output 47 | ports: 48 | - "10000:10000" 49 | - "9083:9083" 50 | networks: 51 | ntrino: 52 | aliases: 53 | - hive 54 | depends_on: 55 | postgres: 56 | condition: service_healthy 57 | 58 | coordinator: 59 | build: . 60 | container_name: coordinator 61 | environment: 62 | - STORE_KEY=${STORE_KEY} 63 | - STORE_SECRET=${STORE_SECRET} 64 | - S3_ENDPOINT=${S3_ENDPOINT} 65 | ports: 66 | - "8080:8080" 67 | - "8443:8443" 68 | expose: 69 | - "8080" 70 | - "8443" 71 | user: trino 72 | volumes: 73 | - ./tmp/data/coordinator:/data/trino:rw 74 | - ./coordinator:/etc/trino:rw 75 | - ./security:/opt/trino/security:rw 76 | - ./certs:/opt/trino/certs:rw 77 | - ./hive/output:/opt/data/output:rw 78 | networks: 79 | ntrino: 80 | aliases: 81 | - coordinator 82 | depends_on: 83 | postgres: 84 | condition: service_healthy 85 | healthcheck: 86 | test: ["CMD-SHELL", "curl -sS http://localhost:8080/|| exit 1"] 87 | interval: "20s" 88 | timeout: "20s" 89 | retries: 3 90 | 91 | 92 | worker-1: 93 | build: . 94 | container_name: worker-1 95 | environment: 96 | - STORE_KEY=${STORE_KEY} 97 | - STORE_SECRET=${STORE_SECRET} 98 | - S3_ENDPOINT=${S3_ENDPOINT} 99 | ports: 100 | - "8090:8080" 101 | - "9443:8443" 102 | user: trino 103 | volumes: 104 | - workerdb:/data/trino 105 | - ./worker:/etc/trino 106 | - ./certs:/opt/trino/certs 107 | - ./hive/output:/opt/data/output 108 | networks: 109 | ntrino: 110 | aliases: 111 | - worker-1 112 | depends_on: 113 | coordinator: 114 | condition: service_healthy 115 | 116 | superset: 117 | #if issues building it, use the image below 118 | build: superset 119 | #image: ${REPO}cloudnativek8s/superset:latest-dev 120 | container_name: superset 121 | environment: 122 | - DATA_DIR=/opt/superset/data 123 | - MAPBOX_API_KEY=${MAPBOX_API_KEY} 124 | - SUPERSET_SECRET_KEY=HAjeudha2uahde*$Hau&@1 125 | volumes: 126 | - ./certs:/opt/superset/certs 127 | - supersetdb:/opt/superset/data 128 | ports: 129 | - "9000:8088" 130 | networks: 131 | ntrino: 132 | aliases: 133 | - superset 134 | depends_on: 135 | coordinator: 136 | condition: service_healthy 137 | postgres: 138 | condition: service_healthy 139 | 140 | 141 | trino-proxy: 142 | image: "${REPO}envoyproxy/envoy-dev:latest" 143 | command: /usr/local/bin/envoy -c /etc/trino-envoy.yaml --service-cluster trino_coordinator -l debug 144 | container_name: "trino-proxy" 145 | volumes: 146 | - ./envoy/trino-envoy.yaml:/etc/trino-envoy.yaml 147 | - ./certs/:/opt/certs 148 | networks: 149 | ntrino: 150 | aliases: 151 | - trino-proxy 152 | expose: 153 | - "8453" 154 | - "8001" 155 | ports: 156 | - "8453:8453" 157 | - "8001:8001" 158 | 159 | #AWS_ACCESS_KEY_ID=minio AWS_SECRET_ACCESS_KEY=minio123 aws s3 --endpoint-url http://localhost:9005 ls s3:// 160 | minio: 161 | image: ${REPO}minio/minio:RELEASE.2022-03-03T21-21-16Z 162 | #command: "server /data --console-address :9001" 163 | #s3 backend can be set to "gateway s3" or "server /data" 164 | #if this command fails, replace S3_BACKEND with one of the desired values 165 | command: "${S3_BACKEND} --address :9000 --console-address :9001" 166 | container_name: "minio" 167 | environment: 168 | - MINIO_ROOT_USER=${STORE_KEY} 169 | - MINIO_ROOT_PASSWORD=${STORE_SECRET} 170 | - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY} 171 | - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} 172 | volumes: 173 | - ./minio/data:/data 174 | networks: 175 | ntrino: 176 | aliases: 177 | - minio 178 | expose: 179 | - "9000" 180 | - "9001" 181 | ports: 182 | - "9005:9000" 183 | - "9001:9001" 184 | 185 | networks: 186 | ntrino: {} 187 | 188 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | # This is normally unused 84 | # shellcheck disable=SC2034 85 | APP_BASE_NAME=${0##*/} 86 | # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) 87 | APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit 88 | 89 | # Use the maximum available, or set MAX_FD != -1 to use that value. 90 | MAX_FD=maximum 91 | 92 | warn () { 93 | echo "$*" 94 | } >&2 95 | 96 | die () { 97 | echo 98 | echo "$*" 99 | echo 100 | exit 1 101 | } >&2 102 | 103 | # OS specific support (must be 'true' or 'false'). 104 | cygwin=false 105 | msys=false 106 | darwin=false 107 | nonstop=false 108 | case "$( uname )" in #( 109 | CYGWIN* ) cygwin=true ;; #( 110 | Darwin* ) darwin=true ;; #( 111 | MSYS* | MINGW* ) msys=true ;; #( 112 | NONSTOP* ) nonstop=true ;; 113 | esac 114 | 115 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 116 | 117 | 118 | # Determine the Java command to use to start the JVM. 119 | if [ -n "$JAVA_HOME" ] ; then 120 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 121 | # IBM's JDK on AIX uses strange locations for the executables 122 | JAVACMD=$JAVA_HOME/jre/sh/java 123 | else 124 | JAVACMD=$JAVA_HOME/bin/java 125 | fi 126 | if [ ! -x "$JAVACMD" ] ; then 127 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 128 | 129 | Please set the JAVA_HOME variable in your environment to match the 130 | location of your Java installation." 131 | fi 132 | else 133 | JAVACMD=java 134 | if ! command -v java >/dev/null 2>&1 135 | then 136 | die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | fi 142 | 143 | # Increase the maximum file descriptors if we can. 144 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 145 | case $MAX_FD in #( 146 | max*) 147 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 148 | # shellcheck disable=SC2039,SC3045 149 | MAX_FD=$( ulimit -H -n ) || 150 | warn "Could not query maximum file descriptor limit" 151 | esac 152 | case $MAX_FD in #( 153 | '' | soft) :;; #( 154 | *) 155 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 156 | # shellcheck disable=SC2039,SC3045 157 | ulimit -n "$MAX_FD" || 158 | warn "Could not set maximum file descriptor limit to $MAX_FD" 159 | esac 160 | fi 161 | 162 | # Collect all arguments for the java command, stacking in reverse order: 163 | # * args from the command line 164 | # * the main class name 165 | # * -classpath 166 | # * -D...appname settings 167 | # * --module-path (only if needed) 168 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 169 | 170 | # For Cygwin or MSYS, switch paths to Windows format before running java 171 | if "$cygwin" || "$msys" ; then 172 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 173 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 174 | 175 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 176 | 177 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 178 | for arg do 179 | if 180 | case $arg in #( 181 | -*) false ;; # don't mess with options #( 182 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 183 | [ -e "$t" ] ;; #( 184 | *) false ;; 185 | esac 186 | then 187 | arg=$( cygpath --path --ignore --mixed "$arg" ) 188 | fi 189 | # Roll the args list around exactly as many times as the number of 190 | # args, so each arg winds up back in the position where it started, but 191 | # possibly modified. 192 | # 193 | # NB: a `for` loop captures its iteration list before it begins, so 194 | # changing the positional parameters here affects neither the number of 195 | # iterations, nor the values presented in `arg`. 196 | shift # remove old arg 197 | set -- "$@" "$arg" # push replacement arg 198 | done 199 | fi 200 | 201 | 202 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 203 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 204 | 205 | # Collect all arguments for the java command: 206 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, 207 | # and any embedded shellness will be escaped. 208 | # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be 209 | # treated as '${Hostname}' itself on the command line. 210 | 211 | set -- \ 212 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 213 | -classpath "$CLASSPATH" \ 214 | org.gradle.wrapper.GradleWrapperMain \ 215 | "$@" 216 | 217 | # Stop when "xargs" is not available. 218 | if ! command -v xargs >/dev/null 2>&1 219 | then 220 | die "xargs is not available" 221 | fi 222 | 223 | # Use "xargs" to parse quoted args. 224 | # 225 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 226 | # 227 | # In Bash we could simply go: 228 | # 229 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 230 | # set -- "${ARGS[@]}" "$@" 231 | # 232 | # but POSIX shell has neither arrays nor command substitution, so instead we 233 | # post-process each arg (as a line of input to sed) to backslash-escape any 234 | # character that might be a shell metacharacter, then use eval to reverse 235 | # that process (while maintaining the separation between arguments), and wrap 236 | # the whole thing up as a single "set" statement. 237 | # 238 | # This will of course break if any of these variables contains a newline or 239 | # an unmatched quote. 240 | # 241 | 242 | eval "set -- $( 243 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 244 | xargs -n1 | 245 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 246 | tr '\n' ' ' 247 | )" '"$@"' 248 | 249 | exec "$JAVACMD" "$@" 250 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | This is a recipe to create a Presto/Trino cluster with hive. 3 | 4 | ## Index 5 | - Build Images 6 | - Download Client 7 | - Certificates 8 | - Configure Environment/Secrets 9 | - Test Trino Connectivity 10 | - Setup S3 environment 11 | - S3 Credentials 12 | - Create Table in Hive with S3 13 | - Queries using Hive 14 | - Queries using Trino 15 | - Access Control 16 | - Querying with Superset 17 | - Running with Envoy 18 | 19 | ### Trino 20 | Trino is a massively parallel processing engine that can be used to perform query federation in a modern open source data lakehouse setup. 21 | We are using the version of trino listed in gradle.properties. It is referenced by the gradle build to create trino authorization and audit plugin jar. 22 | It is then used by the Makefile to build trino image as well. 23 | 24 | ### Build Images 25 | If you do want to build your own images, update docker-compose.yaml to enable image: for hive and trino 26 | 27 | ```make build``` will build trino image. 28 | 29 | While we have referenced to an already built hive image in docker-compose.yaml, you can build one and use your own image. 30 | Check hive/Dockerfile2 if you want to simply overlay the hive-authz jar on top of a apache-hive. 31 | 32 | ```make build-hive``` will build a hive image from scratch. 33 | 34 | Note hive/Dockerfile relies on hive and hadoop binary to be available locally to save bandwidth for local builds. 35 | So if you download it once, the image build process will be faster as it will try to reuse it. 36 | 37 | ### Download client 38 | ```shell 39 | TRINO_VERSION=$(grep trinoVersion gradle.properties | awk -F'=' '{print $2}' | xargs) 40 | curl -s -o trino https://repo1.maven.org/maven2/io/trino/trino-cli/${TRINO_VERSION}/trino-cli-${TRINO_VERSION}-executable.jar 41 | chmod +x trino 42 | ``` 43 | 44 | ### Certificates 45 | 46 | Generate keystore and truststore 47 | ```shell 48 | keytool -genkeypair -alias trino -keyalg RSA -keystore certs/keystore.jks \ 49 | -dname "CN=coordinator, OU=datalake, O=dataco, L=Sydney, ST=NSW, C=AU" \ 50 | -ext san=dns:coordinator,dns:coordinator.presto,dns:coordinator.presto.svc,dns:coordinator.presto.svc.cluster.local,dns:coordinator-headless,dns:coordinator-headless.presto,dns:coordinator-headless.presto.svc,dns:coordinator-headless.presto.svc.cluster.local,dns:localhost,dns:trino-proxy,ip:127.0.0.1,ip:192.168.64.5,ip:192.168.64.6 \ 51 | -storepass password 52 | 53 | keytool -exportcert -file certs/trino.cer -alias trino -keystore certs/keystore.jks -storepass password 54 | 55 | keytool -import -v -trustcacerts -alias trino_trust -file certs/trino.cer -keystore certs/truststore.jks -storepass password -keypass password -noprompt 56 | 57 | 58 | keytool -keystore certs/keystore.jks -exportcert -alias trino -storepass password| openssl x509 -inform der -text 59 | 60 | keytool -importkeystore -srckeystore certs/keystore.jks -destkeystore certs/trino.p12 -srcstoretype jks -deststoretype pkcs12 -srcstorepass password -deststorepass password 61 | 62 | openssl pkcs12 -in certs/trino.p12 -out certs/trino.pem 63 | 64 | openssl x509 -in certs/trino.cer -inform DER -out certs/trino.crt 65 | 66 | ``` 67 | 68 | ### Configure Environment/Secrets 69 | Let's start presto coordinator, presto worker, hive, postgres using docker-compose. 70 | Before starting it, please perform the following 71 | 72 | ``` 73 | cp .env.template .env 74 | ``` 75 | and put the secrets in .env file 76 | 77 | Run ```./env.sh``` to substitute S3 and Postgres password in hive-site.xml 78 | 79 | It is highly advisable that you update your /etc/hosts or equivalent file with the following host entries so we can use 80 | same DNS names inside docker as well as in your cli. 81 | 82 | /etc/hosts: 83 | 84 | ``` 85 | 127.0.0.1 coordinator worker-1 worker-0 trino-proxy 86 | 127.0.0.1 postgres 87 | ``` 88 | 89 | docker-compose up can be run using ```make up``` 90 | Check the progress using ```make ps``` and once all containers are up, navigate to https://localhost:8443/ui/ with credentials admin/password to view the presto UI. 91 | 92 | ### Test Trino Connectivity 93 | Run ```make trino``` with password as "password" to login to trino using the trino executable jar. 94 | Here issue command like ```show catalogs;``` to see all available catalogs for query. 95 | 96 | ```sql 97 | show catalogs; 98 | show schemas in tpch; 99 | show tables in tpch.sf1; 100 | select * from tpch.sf1.nation; 101 | ``` 102 | 103 | The same query can be run through rest URI 104 | 105 | ```shell 106 | curl -k https://coordinator:8443/v1/statement/ \ 107 | --header "Authorization: Basic YWRtaW46cGFzc3dvcmQ=" \ 108 | --header "X-Trino-User: admin" \ 109 | --header "X-Trino-Schema: sf1" \ 110 | --header "X-Trino-Source: somesource" \ 111 | --header "X-Trino-Time-Zone: UTC" \ 112 | --header "X-Trino-Catalog: tpch" \ 113 | --header "User-Agent: trino-cli" \ 114 | --data "select * from nation" 115 | 116 | ``` 117 | 118 | ### Setup S3 environment 119 | create a s3 bucket for this project and upload test data to s3 bucket 120 | 121 | 122 | replace endpoint-url if not running minio at all. 123 | ``` 124 | while read line; do if [[ ! "$line" =~ ^# ]]; then export $line; fi; done < .env; 125 | AWS_ACCESS_KEY_ID=${STORE_KEY} AWS_SECRET_ACCESS_KEY=${STORE_SECRET} \ 126 | aws s3 --endpoint-url http://localhost:9005 mb s3://my-trino-dataset 127 | ``` 128 | I used by google fit app data for this 129 | 130 | ``` 131 | AWS_ACCESS_KEY_ID=${STORE_KEY} AWS_SECRET_ACCESS_KEY=${STORE_SECRET} \ 132 | aws s3 --endpoint-url http://localhost:9005 cp hive/sampledata.csv s3://my-trino-dataset/data/fit/load_date=2021-03-06/ 133 | ``` 134 | 135 | also upload sample finance data to another slice in the same bucket 136 | 137 | ``` 138 | AWS_ACCESS_KEY_ID=${STORE_KEY} AWS_SECRET_ACCESS_KEY=${STORE_SECRET} \ 139 | aws s3 --endpoint-url http://localhost:9005 cp --recursive hive/output s3://my-trino-dataset/data/finance/output 140 | ``` 141 | 142 | You should see something like this when you visit http://localhost:9001/ 143 | By default, you can provide minio/minio123 as credentials when accessing minio. 144 | 145 | ![minio](./docs/minio-ui.png) 146 | Study .env.template file or .env to check whether you are using minio 147 | in data or gateway mode. 148 | 149 | 150 | Let's list the data in object storage to verify that the upload is successful 151 | ``` 152 | AWS_ACCESS_KEY_ID=${STORE_KEY} AWS_SECRET_ACCESS_KEY=${STORE_SECRET} \ 153 | aws s3 --endpoint-url http://localhost:9005 ls s3://my-trino-dataset/data --recursive 154 | ``` 155 | 156 | ### S3 Credentials 157 | In order to provide maximum flexibility and avoid secret sprawl, we are implementing a custom credential provider 158 | and reusing it for both Hive and Trino usage. This ensures our hive-site.xml and trino hive properties do not need to have sensitive information. 159 | 160 | To avoid conflicts with environment variables and the default behaviour, we are using the following four environment variables to determine what our 161 | secret strategy is going to be. We could also provide these as XML string property in hive resource xml file. 162 | 163 | ``` 164 | String accessKey = System.getenv("STORE_KEY"); 165 | String secretKey = System.getenv("STORE_SECRET"); 166 | String identityFile = System.getenv("STORE_TOKEN_FILE"); 167 | String roleArn = System.getenv("STORE_ROLE_ARN"); 168 | ``` 169 | 170 | Refer to [CredentialsFactory.java](./hive-authz/src/main/java/com/github/skhatri/s3/CredentialsFactory.java) in hive-authz project to view/change the credentials derivation strategy. 171 | 172 | 173 | ### Create Table in Hive with S3 174 | 175 | Next, login to hive and create the activity table. Register a new partition also 176 | 177 | ```shell 178 | make hive-cli 179 | ``` 180 | 181 | #set bucket name as variable 182 | 183 | ```hiveql 184 | set hivevar:bucket_name=my-trino-dataset; 185 | CREATE DATABASE fitness; 186 | use fitness; 187 | 188 | CREATE EXTERNAL TABLE activity ( 189 | activity_date date, 190 | average_weight string, 191 | max_weight string, 192 | min_weight string, 193 | calories float, 194 | heart_points float, 195 | heart_minutes float, 196 | low_latitude float, 197 | low_longitude float, 198 | high_latitude float, 199 | high_longitude float, 200 | step_count int, 201 | distance float, 202 | average_speed float, 203 | max_speed float, 204 | min_speed float, 205 | move_minutes_count int, 206 | cycling_duration float, 207 | inactive_duration float, 208 | walking_duration float, 209 | running_duration float 210 | ) PARTITIONED BY (load_date date) 211 | ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' 212 | LOCATION 's3a://${hivevar:bucket_name}/data/fit' 213 | TBLPROPERTIES ("skip.header.line.count"="1"); 214 | 215 | ALTER TABLE activity 216 | ADD PARTITION (load_date='2021-03-06') LOCATION 's3a://${hivevar:bucket_name}/data/fit/load_date=2021-03-06'; 217 | ``` 218 | 219 | Create Delta Lake Table 220 | 221 | ```hiveql 222 | CREATE DATABASE finance; 223 | 224 | 225 | DROP TABLE IF EXISTS finance.activity; 226 | 227 | #Using spark delta generated manifest 228 | CREATE EXTERNAL TABLE finance.activity( 229 | account string, 230 | txn_id string, 231 | merchant string, 232 | category string, 233 | last_updated timestamp, 234 | deleted boolean, 235 | txn_date date, 236 | amount float 237 | ) partitioned by (version date) 238 | ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' 239 | STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat' 240 | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 241 | LOCATION 's3a://${hivevar:bucket_name}/data/finance/output/activity/_symlink_format_manifest/'; 242 | 243 | MSCK REPAIR TABLE finance.activity; 244 | 245 | DROP TABLE IF EXISTS finance.activity_snapshot; 246 | 247 | CREATE EXTERNAL TABLE finance.activity_snapshot( 248 | account string, 249 | txn_id string, 250 | merchant string, 251 | category string, 252 | last_updated timestamp, 253 | deleted boolean, 254 | txn_date date, 255 | amount float 256 | ) partitioned by (version date) 257 | ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' 258 | STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat' 259 | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 260 | LOCATION 's3a://${hivevar:bucket_name}/data/finance/output/activity/_delta_manifest/'; 261 | 262 | MSCK REPAIR TABLE finance.activity_snapshot; 263 | 264 | ``` 265 | 266 | ### Queries using Hive 267 | Just using hive, we can do the following queries as well. 268 | 269 | #### How much have I walked since I installed the fit app? 270 | ```sql 271 | use fitness; 272 | select sum(step_count) as steps, sum(distance)/1000 as distance_in_kms from activity; 273 | ``` 274 | 275 | #### Monthly activity summary 276 | ```sql 277 | select year(activity_date), month(activity_date), sum(heart_points) as heart_pt, sum(step_count) as steps, sum(distance)/1000 as km 278 | from activity 279 | group by year(activity_date), month(activity_date); 280 | ``` 281 | 282 | ### Queries using Trino 283 | 284 | ```shell 285 | make trino 286 | ``` 287 | 288 | ```sql 289 | show catalogs; 290 | show schemas in hive; 291 | show tables in hive.fitness; 292 | 293 | select sum(step_count) as steps, sum(distance)/1000 as distance_in_kms from hive.fitness.activity; 294 | 295 | use hive.fitness; 296 | select year(activity_date) as year, month(activity_date) as month, sum(heart_points) as heart_pt, sum(step_count) as steps, sum(distance)/1000 as km 297 | from activity 298 | group by year(activity_date), month(activity_date); 299 | ``` 300 | 301 | ### Query Delta Data with Trino 302 | 303 | View https://github.com/skhatri/spark-delta-by-example for relevant spark delta code 304 | There is pre delta 2.0 code and higher versions may have different API now. 305 | 306 | #### Show available snapshots 307 | 308 | ```sql 309 | select version, count(*) 310 | from hive.finance.activity_snapshot 311 | group by version 312 | order by version asc; 313 | ``` 314 | 315 | |version | _col1| 316 | |----------|------| 317 | |2021-03-02| 15| 318 | |2021-03-03| 30| 319 | |2021-03-05| 45| 320 | |2021-03-08| 47| 321 | |2021-03-09| 50| 322 | 323 | There are 5 versions of the activity dataset for 5 date partitions. The last version has 50 records. 324 | 325 | 326 | #### Show latest 327 | 328 | ```sql 329 | select count(*) 330 | from hive.finance.activity; 331 | ``` 332 | 333 | |_col0| 334 | |-----| 335 | | 50| 336 | 337 | The activity table also contains the same number of records as the latest version. These records would be merged. 338 | 339 | #### Find total number of activities by account 340 | 341 | ```sql 342 | select account, count(*) as activity_count from hive.finance.activity 343 | group by account; 344 | ``` 345 | 346 | #### As of a specific snapshot 347 | As of version 2021-03-02, what was transaction id ```txn10``` labelled as? 348 | 349 | ```sql 350 | select * from hive.finance.activity_snapshot 351 | where version=cast('2021-03-02' as date) and txn_id='txn10'; 352 | ``` 353 | 354 | |account | txn_id | merchant | category | last_updated | deleted | txn_date | amount | version| 355 | |---------|--------|------------------|-----------|-------------------------|---------|------------|--------|-----------| 356 | |acc4 | txn10 | Prouds Jewellery | Jewellery | 2021-03-01 13:00:00.000 | false | 2021-03-02 | 189.0 | 2021-03-02 | 357 | 358 | (1 row) 359 | 360 | ```sql 361 | select * from hive.finance.activity_snapshot 362 | where version=cast('2021-03-09' as date) and txn_id='txn10'; 363 | 364 | ``` 365 | 366 | |account | txn_id | merchant | category | last_updated | deleted | txn_date | amount | version | 367 | |---------|--------|------------------|----------|-------------------------|---------|------------|--------|-----------| 368 | |acc4 | txn10 | Prouds Jewellery | Fashion | 2021-03-07 13:00:00.000 | false | 2021-03-02 | 189.0 | 2021-03-09 | 369 | 370 | 371 | #### Latest category 372 | What is the latest category of transaction id ```txn10``` and when was it last updated? 373 | 374 | ```sql 375 | select * from hive.finance.activity 376 | where txn_id='txn10'; 377 | 378 | ``` 379 | 380 | |account | txn_id | merchant | category | last_updated | deleted | txn_date | amount | version | 381 | |---------|--------|------------------|----------|-------------------------|---------|------------|--------|---------| 382 | |acc4 | txn10 | Prouds Jewellery | Fashion | 2021-03-07 13:00:00.000 | false | 2021-03-02 | 189.0 | 2021-03-02| 383 | 384 | 385 | #### Category Change over time 386 | Acc5 bought something from Apple Store Sydney on 2021-03-05, how did the category for this transaction change over time? 387 | 388 | ```sql 389 | select account, txn_date, category as latest_category, last_updated, txn_id 390 | from hive.finance.activity 391 | where account = 'acc5' and txn_date=cast('2021-03-05' as date) and merchant='Apple Store Sydney'; 392 | 393 | select account, txn_date, category as versioned_category, last_updated, txn_id 394 | from hive.finance.activity_snapshot 395 | where version=cast('2021-03-05' as date) and account = 'acc5' and txn_date=cast('2021-03-05' as date) and merchant='Apple Store Sydney'; 396 | ``` 397 | It was initially labelled as Hardware and later it got changed to Phone. 398 | 399 | ### Access Control 400 | 401 | Let's create few users whose access will be configured in trino rules file. 402 | 403 | ``` 404 | htpasswd -C 10 -B -b -c security/passwords/password.db user1 password 405 | htpasswd -C 10 -B -b security/passwords/password.db user2 password 406 | htpasswd -C 10 -B -b security/passwords/password.db user3 password 407 | htpasswd -C 10 -B -b security/passwords/password.db skhatri password 408 | htpasswd -C 10 -B -b security/passwords/password.db admin password 409 | ``` 410 | 411 | Refer to security/rules/rules.json for configuration, the following table summaries the ACL by user. 412 | 413 | | User | Access | 414 | |---------|------------------------------------------------| 415 | | user1 | finance.\*,tpch,information_schema | 416 | | user2 | fitness.activity,tpch,information_schema | 417 | | user3 | finance.activity,tpch,information_schema | 418 | | skhatri | finance.\*, fitness.\*,tpch,information_schema | 419 | | admin | * | 420 | 421 | Restart coordinator and worker-1 instances: 422 | 423 | ```shell 424 | docker-compose stop worker-1 425 | docker-compose stop coordinator 426 | docker-compose start coordinator 427 | docker-compose start worker-1 428 | ``` 429 | 430 | #### Querying as Users 431 | Let's login as user1 and explore their privileges. 432 | 433 | ```shell 434 | ./trino --debug --user=user1 --password --truststore-path=./certs/truststore.jks --truststore-password=password --server https://localhost:8443 435 | ``` 436 | 437 | While the ```show catalogs;``` command lists the available catalogs. Listing schema in one of the catalogs like tpch will only show information_schema. 438 | If we try to retrieve customer data from tpch.sf1, it will throw exception: 439 | 440 | ```sql 441 | select * from tpch.sf1.customer limit 2; 442 | ``` 443 | As user1 does not have access to tpch, the query will return error 444 | 445 | ``` 446 | Query 20220507_183600_00046_z589m failed: Access Denied: Cannot select from table tpch.sf1.customer 447 | io.trino.spi.security.AccessDeniedException: Access Denied: Cannot select from table tpch.sf1.customer 448 | ``` 449 | 450 | In rules.json we have the following permission: 451 | 452 | ```json 453 | { 454 | "user": "user1", 455 | "catalog": "hive", 456 | "schema": "finance", 457 | "table": ".*", 458 | "privileges": [ 459 | "SELECT" 460 | ] 461 | } 462 | ``` 463 | We have granted user1 the access to query any table in schema finance of the catalog hive. Let's test this out. 464 | 465 | ```sql 466 | 467 | show schemas in hive; 468 | ``` 469 | 470 | | Schema | 471 | |------------------| 472 | |finance | 473 | |information_schema| 474 | 475 | ```sql 476 | show tables in hive.finance; 477 | ``` 478 | 479 | So we can see finance schema for hive in the list of schema available here. While you can check the available tables with ```show tables in hive.finance```, 480 | let's use our prior knowledge of hive finance tables and query activity table. 481 | 482 | ```sql 483 | select * from hive.finance.activity limit 2; 484 | ``` 485 | Yes, we can retrieve the data as user1 486 | 487 | |account | txn_id | merchant | category | last_updated | deleted | txn_date | amount | version| 488 | |--------|--------|-------------------|----------|-------------------------|---------|------------|--------|---------| 489 | |acc2 | txn36 | Kathmandu Miranda | Clothing | 2021-03-04 13:00:00.000 | false | 2021-03-05 | 69.95 | 2021-03-05| 490 | |acc3 | txn22 | Optus Phone | Phone | 2021-03-02 13:00:00.000 | false | 2021-03-03 | 40.0 | 2021-03-03| 491 | 492 | The rules.json can be reloaded periodically using ```security.refresh-period``` when using File system access control. If using this 493 | ACL in Kubernetes environment, it is possible to run a sidecar container along with coordinator which can fetch the file periodically 494 | from git or remote resource. Refresh option is available for access control file as well as the passwords file. 495 | 496 | Refer to trino-ext-authz folder for a external authz plugin that is based on default file system access control. 497 | 498 | Possible extension areas can be calling Open Policy Agent server from the external authz plugin to determine or downloading rules json from a remote resource and caching it. 499 | A new yaml DSL can be maintained to keep rules simpler. 500 | 501 | ### Querying with Superset 502 | 503 | Setup a database in Superset using the following URL and connect params 504 | 505 | 506 | Login to http://localhost:9000/ with admin/admin 507 | 508 | Go to the screen to add a new database http://localhost:9000/databaseview/add and configure the following: 509 | 510 | |Field| Value | 511 | |---|----------------------------------------------| 512 | |Database| trino | 513 | |SQLAlchemy URI| trino://admin:password@coordinator:8443/hive | 514 | |Impersonate the logged on user| enable it | 515 | |Secure Extra| {"connect_args": {"verify": false}} | 516 | |Security| Paste content of ca.crt and trino.crt ```cat certs/ca.crt certs/trino.crt|pbcopy``` | 517 | 518 | Click Test Connection to verify the configuration is ok. Once configured, open http://localhost:9000/superset/sqllab 519 | and execute 520 | 521 | ```sql 522 | select * 523 | from hive.finance.activity 524 | limit 2 525 | ``` 526 | The query should return some data. 527 | ![superset query](./docs/superset-query.png) 528 | 529 | You can grant sql_lab role to other users using the link in superset ```http://localhost:9000/users/list/``` 530 | Log out and try to login as user1 with password "password". Select data from hive.finance.activity. 531 | 532 | ```sql 533 | select * 534 | from hive.finance.activity 535 | limit 2; 536 | ``` 537 | This should result in successful query execution and with some data being displayed. Behind the scenes, trino driver sent impersonation 538 | header for user1 even though the connection is established with admin user. 539 | 540 | Let's try query that user1 does not have access to run. 541 | 542 | ```sql 543 | select * 544 | from tpch.sf1.nation 545 | limit 2; 546 | ``` 547 | 548 | The following error should be displayed when running the above query as user1. 549 | 550 | ``` 551 | Unexpected Error 552 | base error: Access Denied: Cannot select from table tpch.sf1.nation 553 | ``` 554 | ![trino error](./docs/trino-superset-access.png) 555 | 556 | 557 | Impersonation configuration for superset queries is also present in security/rules/rules.json 558 | 559 | ```json 560 | , 561 | "impersonation": [ 562 | { 563 | "original_user": "(admin|dev)", 564 | "new_user": ".*", 565 | "allow": true 566 | }, 567 | { 568 | "original_user": "skhatri", 569 | "new_user": "admin", 570 | "allow": false 571 | }, 572 | { 573 | "original_user": "skhatri", 574 | "new_user": ".*", 575 | "allow": true 576 | } 577 | ] 578 | } 579 | ``` 580 | 581 | #### Query via REST call 582 | 583 | Note, you can also run the query via command line like so: 584 | 585 | ```shell 586 | curl --cacert ./certs/trino.crt \ 587 | --header "Authorization: Basic YWRtaW46cGFzc3dvcmQ=" \ 588 | --header "X-Trino-Source: trino-python-client" \ 589 | --header "X-Trino-Time-Zone: UTC" \ 590 | --header "User-Agent: trino-cli" \ 591 | --header "X-Trino-User: user1" \ 592 | --header "X-Trino-Catalog: hive" \ 593 | --header "X-Trino-Schema: fitness" \ 594 | --data "select * from hive.fitness.activity" \ 595 | https://coordinator:8443/v1/statement/ 596 | ``` 597 | 598 | The query runs asynchronously and result is to be collected by the client by calling url returned as nextUri. 599 | 600 | ```json 601 | {"id":"20220507_202039_00316_z589m","infoUri":"https://coordinator:8443/ui/query.html?20220507_202039_00316_z589m","nextUri":"https://coordinator:8443/v1/statement/queued/20220507_202039_00316_z589m/y66e5ba5e33aa4a2d883eb1de5b1b34a70d49fa5b/1","stats":{"state":"QUEUED","queued":true,"scheduled":false,"nodes":0,"totalSplits":0,"queuedSplits":0,"runningSplits":0,"completedSplits":0,"cpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":0,"elapsedTimeMillis":0,"processedRows":0,"processedBytes":0,"physicalInputBytes":0,"peakMemoryBytes":0,"spilledBytes":0},"warnings":[]} 602 | ``` 603 | 604 | The below query is only for demonstration purpose and will fail. You will need to copy nextUri from output of the previous command. 605 | 606 | ```shell 607 | 608 | curl --cacert ./certs/trino.crt \ 609 | --header "Authorization: Basic YWRtaW46cGFzc3dvcmQ=" \ 610 | --header "X-Trino-Source: trino-python-client" \ 611 | --header "X-Trino-Time-Zone: UTC" \ 612 | --header "User-Agent: trino-cli" \ 613 | --header "X-Trino-User: user1" \ 614 | --header "X-Trino-Catalog: hive" \ 615 | --header "X-Trino-Schema: fitness" \ 616 | https://coordinator:8443/v1/statement/queued/20220507_202039_00316_z589m/y66e5ba5e33aa4a2d883eb1de5b1b34a70d49fa5b/1 617 | ``` 618 | 619 | ```json 620 | {"id":"20220507_202039_00316_z589m","infoUri":"https://coordinator:8443/ui/query.html?20220507_202039_00316_z589m","nextUri":"https://coordinator:8443/v1/statement/queued/20220507_202039_00316_z589m/y2a208f52803793fff6721c73899be69224a8361b/2","stats":{"state":"QUEUED","queued":true,"scheduled":false,"nodes":0,"totalSplits":0,"queuedSplits":0,"runningSplits":0,"completedSplits":0,"cpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":3,"elapsedTimeMillis":5,"processedRows":0,"processedBytes":0,"physicalInputBytes":0,"peakMemoryBytes":0,"spilledBytes":0},"warnings":[]} 621 | ``` 622 | 623 | We will keep on calling nextUri 624 | 625 | ```shell 626 | curl --cacert ./certs/trino.crt \ 627 | --header "Authorization: Basic YWRtaW46cGFzc3dvcmQ=" \ 628 | --header "X-Trino-Source: trino-python-client" \ 629 | --header "X-Trino-Time-Zone: UTC" \ 630 | --header "User-Agent: trino-cli" \ 631 | --header "X-Trino-User: user1" \ 632 | --header "X-Trino-Catalog: hive" \ 633 | --header "X-Trino-Schema: fitness" \ 634 | https://coordinator:8443/v1/statement/queued/20220507_202039_00316_z589m/y2a208f52803793fff6721c73899be69224a8361b/2 635 | ``` 636 | 637 | The response is returned but it has access denied exception. This is valid as we are trying to impersonate "user1" who does not have access to hive.fitness.activity 638 | 639 | ```json 640 | {"id":"20220507_202039_00316_z589m","infoUri":"https://coordinator:8443/ui/query.html?20220507_202039_00316_z589m","stats":{"state":"FAILED","queued":false,"scheduled":false,"nodes":0,"totalSplits":0,"queuedSplits":0,"runningSplits":0,"completedSplits":0,"cpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":3,"elapsedTimeMillis":46,"processedRows":0,"processedBytes":0,"physicalInputBytes":0,"peakMemoryBytes":0,"spilledBytes":0},"error":{"message":"Access Denied: Cannot select from table hive.fitness.activity","errorCode":4,"errorName":"PERMISSION_DENIED","errorType":"USER_ERROR","failureInfo":{"type":"io.trino.spi.security.AccessDeniedException","message":"Access Denied: Cannot select from table hive.fitness.activity","suppressed":[]} 641 | ``` 642 | 643 | We will now remove the impersonation user and try the query again. 644 | 645 | ```shell 646 | curl --cacert ./certs/trino.crt \ 647 | --header "Authorization: Basic YWRtaW46cGFzc3dvcmQ=" \ 648 | --header "X-Trino-Source: trino-python-client" \ 649 | --header "X-Trino-Time-Zone: UTC" \ 650 | --header "User-Agent: trino-cli" \ 651 | --header "X-Trino-Catalog: hive" \ 652 | --header "X-Trino-Schema: fitness" \ 653 | --data "select * from hive.fitness.activity limit 1" \ 654 | https://coordinator:8443/v1/statement/ 655 | ``` 656 | 657 | Clients like superset have the configuration to poll for response. In the background, they would also be calling nextUri just like we are. 658 | 659 | ```json 660 | {"id":"20220507_204009_00318_z589m","infoUri":"https://coordinator:8443/ui/query.html?20220507_204009_00318_z589m","nextUri":"https://coordinator:8443/v1/statement/queued/20220507_204009_00318_z589m/yd279ea2a17c75c545a33a7e2ca6ef153fbb8959d/1","stats":{"state":"QUEUED","queued":true,"scheduled":false,"nodes":0,"totalSplits":0,"queuedSplits":0,"runningSplits":0,"completedSplits":0,"cpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":0,"elapsedTimeMillis":0,"processedRows":0,"processedBytes":0,"physicalInputBytes":0,"peakMemoryBytes":0,"spilledBytes":0},"warnings":[]} 661 | ``` 662 | 663 | call retrieve what is in nextUri 664 | ```shell 665 | curl --cacert ./certs/trino.crt \ 666 | --header "Authorization: Basic YWRtaW46cGFzc3dvcmQ=" \ 667 | --header "X-Trino-Source: trino-python-client" \ 668 | --header "X-Trino-Time-Zone: UTC" \ 669 | --header "User-Agent: trino-cli" \ 670 | --header "X-Trino-Catalog: hive" \ 671 | --header "X-Trino-Schema: fitness" \ 672 | https://coordinator:8443/v1/statement/queued/20220507_204009_00318_z589m/yd279ea2a17c75c545a33a7e2ca6ef153fbb8959d/1 673 | ``` 674 | 675 | ```json 676 | {"id":"20220507_204009_00318_z589m","infoUri":"https://coordinator:8443/ui/query.html?20220507_204009_00318_z589m","nextUri":"https://coordinator:8443/v1/statement/queued/20220507_204009_00318_z589m/ye6a1823d14841fe8b51fe72c8128097f23cbfd2c/2","stats":{"state":"QUEUED","queued":true,"scheduled":false,"nodes":0,"totalSplits":0,"queuedSplits":0,"runningSplits":0,"completedSplits":0,"cpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":8,"elapsedTimeMillis":9,"processedRows":0,"processedBytes":0,"physicalInputBytes":0,"peakMemoryBytes":0,"spilledBytes":0},"warnings":[]} 677 | ``` 678 | 679 | Let's keep on calling nextUri for completeness. 680 | ```shell 681 | curl --cacert ./certs/trino.crt \ 682 | --header "Authorization: Basic YWRtaW46cGFzc3dvcmQ=" \ 683 | --header "X-Trino-Source: trino-python-client" \ 684 | --header "X-Trino-Time-Zone: UTC" \ 685 | --header "User-Agent: trino-cli" \ 686 | --header "X-Trino-Catalog: hive" \ 687 | --header "X-Trino-Schema: fitness" \ 688 | https://coordinator:8443/v1/statement/queued/20220507_204009_00318_z589m/ye6a1823d14841fe8b51fe72c8128097f23cbfd2c/2 689 | ``` 690 | 691 | ```json 692 | {"id":"20220507_204009_00318_z589m","infoUri":"https://coordinator:8443/ui/query.html?20220507_204009_00318_z589m","nextUri":"https://coordinator:8443/v1/statement/executing/20220507_204009_00318_z589m/y210ffea7ebcb7b54ef598335bf18e2b331427888/0","stats":{"state":"QUEUED","queued":true,"scheduled":false,"nodes":0,"totalSplits":0,"queuedSplits":0,"runningSplits":0,"completedSplits":0,"cpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":8,"elapsedTimeMillis":21373,"processedRows":0,"processedBytes":0,"physicalInputBytes":0,"peakMemoryBytes":0,"spilledBytes":0},"warnings":[]} 693 | ``` 694 | ```shell 695 | 696 | curl --cacert ./certs/trino.crt \ 697 | --header "Authorization: Basic YWRtaW46cGFzc3dvcmQ=" \ 698 | --header "X-Trino-Source: trino-python-client" \ 699 | --header "X-Trino-Time-Zone: UTC" \ 700 | --header "User-Agent: trino-cli" \ 701 | --header "X-Trino-Catalog: hive" \ 702 | --header "X-Trino-Schema: fitness" \ 703 | https://coordinator:8443/v1/statement/executing/20220507_204009_00318_z589m/y210ffea7ebcb7b54ef598335bf18e2b331427888/0 704 | ``` 705 | 706 | The final response has the columns and data attributes. 707 | 708 | ```json 709 | {"id":"20220507_204009_00318_z589m","infoUri":"https://coordinator:8443/ui/query.html?20220507_204009_00318_z589m","partialCancelUri":"https://coordinator:8443/v1/statement/executing/partialCancel/20220507_204009_00318_z589m/1/yc1e5c2e839286131959d53aec4c005e10c352b6b/1","nextUri":"https://coordinator:8443/v1/statement/executing/20220507_204009_00318_z589m/yc1e5c2e839286131959d53aec4c005e10c352b6b/1","columns":[{"name":"activity_date","type":"date","typeSignature":{"rawType":"date","arguments":[]}},{"name":"average_weight","type":"varchar","typeSignature":{"rawType":"varchar","arguments":[{"kind":"LONG","value":2147483647}]}},{"name":"max_weight","type":"varchar","typeSignature":{"rawType":"varchar","arguments":[{"kind":"LONG","value":2147483647}]}},{"name":"min_weight","type":"varchar","typeSignature":{"rawType":"varchar","arguments":[{"kind":"LONG","value":2147483647}]}},{"name":"calories","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"heart_points","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"heart_minutes","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"low_latitude","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"low_longitude","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"high_latitude","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"high_longitude","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"step_count","type":"integer","typeSignature":{"rawType":"integer","arguments":[]}},{"name":"distance","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"average_speed","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"max_speed","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"min_speed","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"move_minutes_count","type":"integer","typeSignature":{"rawType":"integer","arguments":[]}},{"name":"cycling_duration","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"inactive_duration","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"walking_duration","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"running_duration","type":"real","typeSignature":{"rawType":"real","arguments":[]}},{"name":"load_date","type":"date","typeSignature":{"rawType":"date","arguments":[]}}],"data":[["2019-10-03","","","",2909.2954,72.0,63.0,33.9449,-122.476845,37.809143,-118.40078,19363,13745.628,1.3299762,18.032257,0.045891136,222,null,6.4861504E7,7164427.0,6170764.0,"2021-03-06"]],"stats":{"state":"RUNNING","queued":false,"scheduled":true,"nodes":2,"totalSplits":7,"queuedSplits":0,"runningSplits":1,"completedSplits":6,"cpuTimeMillis":32,"wallTimeMillis":143,"queuedTimeMillis":8,"elapsedTimeMillis":39369,"processedRows":521,"processedBytes":96423,"physicalInputBytes":96423,"peakMemoryBytes":638,"spilledBytes":0,"rootStage":{"stageId":"0","state":"RUNNING","done":false,"nodes":1,"totalSplits":5,"queuedSplits":0,"runningSplits":0,"completedSplits":5,"cpuTimeMillis":3,"wallTimeMillis":4,"processedRows":1,"processedBytes":574,"physicalInputBytes":0,"failedTasks":0,"coordinatorOnly":false,"subStages":[{"stageId":"1","state":"PENDING","done":false,"nodes":2,"totalSplits":2,"queuedSplits":0,"runningSplits":1,"completedSplits":1,"cpuTimeMillis":29,"wallTimeMillis":139,"processedRows":521,"processedBytes":96423,"physicalInputBytes":96423,"failedTasks":0,"coordinatorOnly":false,"subStages":[]}]},"progressPercentage":85.71428571428571},"warnings":[]} 710 | ``` 711 | 712 | ### Running with Envoy 713 | Although trino has its plugin system for observability and acl, it is also possible to put envoy proxy on top of coordinator. 714 | 715 | The benefits of this can be 716 | - free observability through envoy 717 | - possible ACL or additional rules applied enforced in envoy filter 718 | - zero-downtime trino upgrades 719 | - traffic load shedding based on user, time etc 720 | 721 | Optionally, create trino-proxy certificate which can be configured for trino-proxy envoy instance. 722 | 723 | ``` 724 | ./ca.sh trino-proxy 725 | ``` 726 | 727 | Install jq if you do not have it ```brew install jq``` 728 | 729 | The below script uses trino-proxy to queue up the query and retrieves data by polling the response. 730 | 731 | A file ```bash curl-query.sh``` has been created with the below content to make it easier for you to query against trino using curl and nextUri in a loop until data elements show up. 732 | 733 | ```shell 734 | query="select * from hive.fitness.activity limit 1" 735 | user="user2" 736 | next_uri=$(curl -s -k --cacert ./certs/trino-proxy/bundle.crt \ 737 | --header "Authorization: Basic $(echo -n admin:password|base64)" \ 738 | --header "X-Trino-User: ${user}" --data "${query}" \ 739 | https://trino-proxy:8453/v1/statement/|jq -r '.nextUri') 740 | while true; 741 | do 742 | curl -s -o out.json -k --cacert ./certs/trino-proxy/bundle.crt \ 743 | --header "Authorization: Basic $(echo -n admin:password|base64)" \ 744 | --header "X-Trino-User: ${user}" \ 745 | ${next_uri} 746 | cat out.json 747 | data_val=$(cat out.json|jq -r '.data') 748 | if [[ "${data_val}" != "null" ]]; 749 | then 750 | echo "-----DATA------" 751 | echo ${data_val} 752 | echo "---------------" 753 | break; 754 | fi; 755 | next_uri=$(cat out.json|jq -r '.nextUri') 756 | if [[ "${next_uri}" == "null" ]]; 757 | then 758 | break; 759 | fi; 760 | sleep 1; 761 | done; 762 | 763 | ``` 764 | 765 | Great, we have come to the end of the example. We created a data lake setup with trino, hive, minio and then queried it using superset. We also added an envoy proxy layer on top for observability. Obviously it works on my local laptop :), but if for some reason, things do not on yours, please raise an issue or get in touch. 766 | --------------------------------------------------------------------------------