├── .coveragerc ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yml │ ├── config.yml │ └── feature-request.yml ├── scripts │ ├── create_new_user.sh │ └── install_oracle_instantclient.sh └── workflows │ └── dbt-oracle-adapter-tests.yml ├── .gitignore ├── CONTRIBUTING.md ├── HISTORY.md ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── NOTICE.txt ├── README.md ├── SECURITY.md ├── THIRD_PARTY_LICENSES.txt ├── bin └── create-pem-from-p12 ├── dbt ├── __init__.py ├── adapters │ ├── __init__.py │ └── oracle │ │ ├── __init__.py │ │ ├── __version__.py │ │ ├── column.py │ │ ├── connection_helper.py │ │ ├── connections.py │ │ ├── impl.py │ │ ├── keyword_catalog.py │ │ ├── python_submissions.py │ │ ├── relation.py │ │ ├── relation_configs │ │ ├── __init__.py │ │ ├── base.py │ │ ├── materialized_view.py │ │ └── policies.py │ │ └── sample_profiles.yml └── include │ ├── __init__.py │ └── oracle │ ├── __init__.py │ ├── dbt_project.yml │ ├── macros │ ├── adapters.sql │ ├── apply_grants.sql │ ├── catalog.sql │ ├── columns.sql │ ├── freshness.sql │ ├── materializations │ │ ├── incremental │ │ │ ├── incremental.sql │ │ │ └── strategies.sql │ │ ├── materialized_view │ │ │ └── materialized_view.sql │ │ ├── python_model │ │ │ └── python.sql │ │ ├── seed │ │ │ └── seed.sql │ │ ├── snapshot │ │ │ ├── snapshot.sql │ │ │ ├── snapshot_merge.sql │ │ │ └── strategies.sql │ │ ├── table │ │ │ └── table.sql │ │ ├── tests │ │ │ └── helpers.sql │ │ └── view │ │ │ └── view.sql │ ├── relations │ │ └── drop.sql │ ├── schema_tests.sql │ ├── show.sql │ ├── update_legacy_snapshots.sql │ └── utils │ │ ├── cast_bool_to_text.sql │ │ ├── data_types.sql │ │ ├── date_spine.sql │ │ ├── dateadd.sql │ │ ├── datediff.sql │ │ ├── datetrunc.sql │ │ ├── except.sql │ │ ├── generate_series.sql │ │ ├── hash.sql │ │ ├── last_day.sql │ │ ├── position.sql │ │ ├── right.sql │ │ └── timestamps.sql │ └── profile_template.yml ├── dbt_adbs_test_project ├── README.md ├── analysis │ └── eu_customers.sql ├── data │ └── seed.csv ├── dbt_project.yml ├── macros │ ├── create_index.sql │ ├── datatypes.sql │ ├── demo_multiple_statements.sql │ ├── execute_statements.sql │ ├── generate_schema_name.sql │ ├── hash_arguments.sql │ └── readme.md ├── models │ ├── demo │ │ ├── promotion_costs_for_direct_sales_channel.sql │ │ ├── promotion_costs_for_direct_sales_channel_incr_insert.sql │ │ ├── promotion_costs_for_direct_sales_channel_incr_merge.sql │ │ └── promotion_costs_for_direct_sales_channel_incr_merge_unique_keys.sql │ ├── direct_sales_channel_promo_cost.sql │ ├── eu │ │ ├── countries.sql │ │ └── eu_direct_sales_channels_promo_costs.sql │ ├── exposures.yml │ ├── income_levels.sql │ ├── internet_sales_channel_customers.sql │ ├── kafka.sql │ ├── people.sql │ ├── promotion_costs.sql │ ├── properties.yml │ ├── sales_cost.sql │ ├── sales_cost_incremental.py │ ├── sales_internet_channel.sql │ ├── sales_internet_mv.sql │ ├── sales_py.py │ ├── schema.yml │ ├── test_py_ref.py │ ├── test_py_source.py │ ├── union_customer_sales.sql │ ├── us_product_delete_insert.sql │ ├── us_product_sales_channel_ranking.sql │ ├── us_product_sales_channel_ranking_append.sql │ └── us_seed_customers.sql ├── package-lock.yml ├── packages.yml ├── profiles.yml ├── seeds │ ├── kafka_message.csv │ ├── seed.csv │ └── seed_with_empty_col.csv ├── snapshots │ ├── README.md │ └── promotion_costs.sql └── test │ └── test_count_employees.sql ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── requirements_dev.txt ├── sbom_generation.yaml ├── setup.cfg ├── setup.py ├── tests ├── README.md ├── __init__.py ├── conftest.py └── functional │ └── adapter │ ├── constraints │ ├── __init__.py │ ├── fixtures.py │ └── test_constraints.py │ ├── incremental_materialization │ ├── __init__.py │ ├── quotes │ │ ├── __init__.py │ │ ├── test_quote_special_characters_and_keywords.py │ │ ├── test_quoted_columns_incremental_insert.py │ │ └── test_quotes_enabled_in_model.py │ ├── sync_schema │ │ ├── __init__.py │ │ └── test_quote_special_characters_and_keywords.py │ ├── test_incremental_predicates.py │ ├── test_merge_update_columns.py │ ├── test_quotes_with_merge_update_columns.py │ └── test_unique_id.py │ ├── macros │ ├── __init__.py │ └── test_alter_column_type.py │ ├── materialized_view │ ├── __init__.py │ ├── test_materialized_view.py │ └── utils.py │ ├── simple_seed │ ├── __init__.py │ └── test_simple_seed.py │ ├── snapshots │ ├── __init__.py │ └── test_invalidate_deletes.py │ ├── test_basic.py │ ├── test_caching.py │ ├── test_concurrency.py │ ├── test_config.py │ ├── test_data_types.py │ ├── test_dbt_show.py │ ├── test_docs_generate.py │ ├── test_docs_genreferences.py │ ├── test_ephemeral.py │ ├── test_generictests_where.py │ ├── test_get_last_relation_modified.py │ ├── test_grants.py │ ├── test_quoted_relations.py │ └── utils │ ├── test_common_utils.py │ ├── test_date_spine.py │ ├── test_dateutils.py │ └── test_generate_series.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = dbt 3 | 4 | omit = 5 | /*/tests/* 6 | .dbt-oracle-env/* 7 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: Bug 2 | description: Report a bug or an issue you've found with dbt-oracle 3 | title: "[Bug] " 4 | labels: ["bug", "triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this bug report! 10 | - type: checkboxes 11 | attributes: 12 | label: Is there an existing issue for this? 13 | description: Please search to see if an issue already exists for the bug you encountered. 14 | options: 15 | - label: I have searched the existing issues 16 | required: true 17 | - type: textarea 18 | attributes: 19 | label: Current Behavior 20 | description: A concise description of what you're experiencing. 21 | validations: 22 | required: false 23 | - type: textarea 24 | attributes: 25 | label: Expected Behavior 26 | description: A concise description of what you expected to happen. 27 | validations: 28 | required: false 29 | - type: textarea 30 | attributes: 31 | label: Steps To Reproduce 32 | description: Steps to reproduce the behavior. 33 | placeholder: | 34 | 1. In this environment... 35 | 2. With this config... 36 | 3. Run '...' 37 | 4. See error... 38 | validations: 39 | required: false 40 | - type: textarea 41 | id: logs 42 | attributes: 43 | label: Relevant log output using `--debug` flag enabled 44 | description: | 45 | If applicable, log output to help explain your problem. 46 | render: shell 47 | validations: 48 | required: false 49 | - type: textarea 50 | attributes: 51 | label: Environment 52 | description: | 53 | examples: 54 | - **OS**: Ubuntu 20.04 55 | - **Python**: 3.7.2 (`python --version`) 56 | - **dbt**: 0.21.0 (`dbt --version`) 57 | value: | 58 | - OS: 59 | - Python: 60 | - dbt: 61 | render: markdown 62 | validations: 63 | required: false 64 | - type: textarea 65 | id: database 66 | attributes: 67 | label: What Oracle database version are you using dbt with? 68 | description: | 69 | examples: 70 | - 19c 71 | - 21c 72 | validations: 73 | required: false 74 | - type: textarea 75 | attributes: 76 | label: Additional Context 77 | description: | 78 | Links? References? Anything that will give us more context about the issue you are encountering! 79 | 80 | Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. 81 | validations: 82 | required: false 83 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yml: -------------------------------------------------------------------------------- 1 | name: Feature 2 | description: Suggest an idea for dbt-oracle 3 | title: "[Feature] <title>" 4 | labels: ["enhancement", "triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this feature request! 10 | - type: textarea 11 | attributes: 12 | label: Describe the Feature 13 | description: A clear and concise description of what you want to happen. 14 | validations: 15 | required: true 16 | - type: textarea 17 | attributes: 18 | label: Describe alternatives you've considered 19 | description: | 20 | A clear and concise description of any alternative solutions or features you've considered. 21 | validations: 22 | required: false 23 | - type: textarea 24 | attributes: 25 | label: Who will this benefit? 26 | description: | 27 | What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly. 28 | validations: 29 | required: false 30 | - type: textarea 31 | attributes: 32 | label: Anything else? 33 | description: | 34 | Links? References? Anything that will give us more context about the feature you are suggesting! 35 | validations: 36 | required: false 37 | -------------------------------------------------------------------------------- /.github/scripts/create_new_user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -Exeuo pipefail 4 | 5 | # Parameters 6 | DB_USER="${1}" 7 | DB_PASSWORD="${2}" 8 | TARGET_PDB="${3:-FREEPDB1}" 9 | 10 | # Prepare container switch statement to create user in PDB. 11 | ALTER_SESSION_CMD="ALTER SESSION SET CONTAINER=${TARGET_PDB};" 12 | 13 | # 11g XE does not support PDBs, set container switch statement to empty string. 14 | ORACLE_VERSION=$(sqlplus -version | grep "Release" | awk '{ print $3 }') 15 | if [[ "${ORACLE_VERSION}" = "11.2"* ]]; then 16 | ALTER_SESSION_CMD=""; 17 | fi; 18 | 19 | # Create new user in target PDB 20 | sqlplus -s / as sysdba << EOF 21 | -- Exit on any errors 22 | WHENEVER SQLERROR EXIT SQL.SQLCODE 23 | ${ALTER_SESSION_CMD} 24 | CREATE USER ${DB_USER} IDENTIFIED BY "${DB_PASSWORD}" QUOTA UNLIMITED ON SYSTEM; 25 | GRANT DB_DEVELOPER_ROLE TO ${DB_USER}; 26 | exit; 27 | EOF 28 | -------------------------------------------------------------------------------- /.github/scripts/install_oracle_instantclient.sh: -------------------------------------------------------------------------------- 1 | sudo apt-get update 2 | sudo apt-get install wget libaio1t64 3 | sudo mkdir -p /opt/oracle 4 | wget https://download.oracle.com/otn_software/linux/instantclient/2370000/instantclient-basiclite-linux.x64-23.7.0.25.01.zip -P /tmp 5 | sudo unzip /tmp/instantclient-basiclite-linux.x64-23.7.0.25.01.zip -d /opt/oracle 6 | export PATH="$PATH:/opt/oracle/instantclient_23_7" 7 | export LD_LIBRARY_PATH="/opt/oracle/instantclient_23_7:$LD_LIBRARY_PATH" 8 | sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 9 | sudo mkdir -p /opt/tns_admin 10 | echo "DISABLE_OOB=ON" >> /opt/tns_admin/sqlnet.ora 11 | 12 | -------------------------------------------------------------------------------- /.github/workflows/dbt-oracle-adapter-tests.yml: -------------------------------------------------------------------------------- 1 | name: dbt-tests-adapter 2 | on: push 3 | 4 | jobs: 5 | dbt_oracle_adapter_tests: 6 | runs-on: ${{ matrix.os }} 7 | strategy: 8 | fail-fast: true 9 | matrix: 10 | os: [ ubuntu-latest ] 11 | python-version: ['3.9', '3.10', '3.11', '3.12'] 12 | 13 | services: 14 | oracle_db: 15 | image: container-registry.oracle.com/database/free:latest-lite 16 | env: 17 | ORACLE_PWD: ${{ secrets.DBT_ORACLE_PASSWORD }} 18 | options: --name oracle_db 19 | ports: 20 | - 1521:1521 21 | 22 | steps: 23 | - name: Check out dbt-oracle repository code 24 | uses: actions/checkout@v3 25 | 26 | - name: Set up Python ${{ matrix.python-version }} 27 | uses: actions/setup-python@v3 28 | with: 29 | python-version: ${{ matrix.python-version }} 30 | 31 | - name: Install Oracle Instantclient 32 | run: | 33 | chmod +x ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh 34 | ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh 35 | 36 | - name: Copy Create User script 37 | run: | 38 | chmod +x ${{ github.workspace }}/.github/scripts/create_new_user.sh 39 | docker cp ${{ github.workspace }}/.github/scripts/create_new_user.sh oracle_db:/home/oracle/create_new_user.sh 40 | 41 | - name: Create dbt test users 42 | run: | 43 | docker exec oracle_db /home/oracle/create_new_user.sh dbt_test ${{ secrets.DBT_ORACLE_PASSWORD }} 44 | docker exec oracle_db /home/oracle/create_new_user.sh dbt_test_user_1 ${{ secrets.DBT_ORACLE_PASSWORD }} 45 | docker exec oracle_db /home/oracle/create_new_user.sh dbt_test_user_2 ${{ secrets.DBT_ORACLE_PASSWORD }} 46 | docker exec oracle_db /home/oracle/create_new_user.sh dbt_test_user_3 ${{ secrets.DBT_ORACLE_PASSWORD }} 47 | 48 | - name: Install dbt-oracle with core dependencies 49 | run: | 50 | python -m pip install --upgrade pip 51 | pip install pytest 'dbt-tests-adapter~=1.11,<1.12' 52 | pip install -r requirements.txt 53 | pip install -e . 54 | 55 | - name: Check create-pem-from-p12 script is installed in bin 56 | run: | 57 | create-pem-from-p12 --help 58 | 59 | - name: Run adapter tests - ORA_PYTHON_DRIVER_TYPE => THICK 60 | run: | 61 | pytest -v 62 | env: 63 | ORA_PYTHON_DRIVER_TYPE: THICK 64 | DBT_ORACLE_USER: DBT_TEST 65 | DBT_ORACLE_HOST: localhost 66 | DBT_ORACLE_PORT: 1521 67 | DBT_ORACLE_SCHEMA: DBT_TEST 68 | DBT_ORACLE_PASSWORD: ${{ secrets.DBT_ORACLE_PASSWORD }} 69 | DBT_ORACLE_DATABASE: FREEPDB1 70 | DBT_ORACLE_SERVICE: FREEPDB1 71 | DBT_ORACLE_PROTOCOL: tcp 72 | LD_LIBRARY_PATH: /opt/oracle/instantclient_23_7 73 | TNS_ADMIN: /opt/tns_admin 74 | DBT_TEST_USER_1: DBT_TEST_USER_1 75 | DBT_TEST_USER_2: DBT_TEST_USER_2 76 | DBT_TEST_USER_3: DBT_TEST_USER_3 77 | 78 | - name: Run adapter tests - ORA_PYTHON_DRIVER_TYPE => THIN 79 | run: | 80 | pytest -v 81 | env: 82 | ORA_PYTHON_DRIVER_TYPE: THIN 83 | DBT_ORACLE_USER: DBT_TEST 84 | DBT_ORACLE_HOST: localhost 85 | DBT_ORACLE_PORT: 1521 86 | DBT_ORACLE_SCHEMA: DBT_TEST 87 | DBT_ORACLE_PASSWORD: ${{ secrets.DBT_ORACLE_PASSWORD }} 88 | DBT_ORACLE_DATABASE: FREEPDB1 89 | DBT_ORACLE_SERVICE: FREEPDB1 90 | DBT_ORACLE_PROTOCOL: tcp 91 | DISABLE_OOB: on 92 | TNS_ADMIN: /opt/tns_admin 93 | DBT_TEST_USER_1: DBT_TEST_USER_1 94 | DBT_TEST_USER_2: DBT_TEST_USER_2 95 | DBT_TEST_USER_3: DBT_TEST_USER_3 96 | 97 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | dbt_modules 3 | logs 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # dotenv 87 | .env 88 | 89 | # virtualenv 90 | .venv 91 | venv/ 92 | ENV 93 | ,dbt-oracle-env/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | 108 | dataml_engine/.vscode/settings.json 109 | 110 | ./profile.yaml 111 | *./profile.yaml 112 | /test_code 113 | .dbt-oracle-env/ 114 | ./dbt_test_project/profiles.yml 115 | ./tests/oracle.dbtspec 116 | 117 | 118 | .vscode/ 119 | build/ 120 | .egg-info/ 121 | dist 122 | 123 | .user.yml 124 | 125 | 126 | dbt_integration_test 127 | dbtest.py 128 | list_relations.sql 129 | list_relations_without_caching.sql 130 | Untitled 131 | .idea 132 | dbt_adbs_test_project/dbt_packages 133 | dbt_oracle_xe_test_project 134 | .dockerignore 135 | dockerfile 136 | jaffle_shop 137 | tests/oracle_1TLS.dbtspec 138 | doc/.docenv 139 | doc/build.gitbak 140 | .gitbak/ 141 | .venvpy37 142 | .venv1.1.0 143 | .venv 144 | .bldenv 145 | .venv1.2/ 146 | .venv1.3/ 147 | .venv1.4/ 148 | .venv1.5/ 149 | .venv1.6/ 150 | .venv1.7/ 151 | .venv*/ 152 | dbt_adbs_py_test_project 153 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | # History 2 | 3 | ## v0.1.0 (2020-06-02) 4 | 5 | - First release on PyPI. 6 | 7 | ## v0.2.0 (2020-06-02) 8 | 9 | - Added full macro implementation. Thanks Fabrice Etanchaud 10 | 11 | ## v0.4.3 (2021-06-28) 12 | 13 | - Last stable release of [dbt-oracle](https://github.com/techindicium/dbt-oracle) by Techindicium 14 | 15 | ## v1.0.0 (2022-05-04) 16 | 17 | - First release of vendor-built adapter by Oracle. 18 | - This version supports Python 3.6, 3.7, 3.8 and 3.9 19 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include dbt/include *.sql *.yml *.md 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Configuration variables 2 | VERSION=1.9.2 3 | PROJ_DIR?=$(shell pwd) 4 | VENV_DIR?=${PROJ_DIR}/.bldenv 5 | BUILD_DIR=${PROJ_DIR}/build 6 | DIST_DIR=${PROJ_DIR}/dist 7 | PYTHON_3=python3.12 8 | 9 | 10 | clean_venv: 11 | rm -fr ${VENV_DIR} 12 | 13 | clean: clean_venv 14 | rm -fr ${DIST_DIR} 15 | rm -fr ${BUILD_DIR} 16 | 17 | wheel: clean 18 | ${PYTHON_3} -m venv ${VENV_DIR} 19 | ${VENV_DIR}/bin/pip install --upgrade wheel dataclasses build 20 | ${VENV_DIR}/bin/python3 -m build 21 | 22 | # Target to test dbt-oracle package in development environment. 23 | # This builds a wheel pkg from source in the current project directory and tests all dbt functionalities. 24 | adbs_local_env_test: wheel clean_venv 25 | ${PYTHON_3} -m venv ${VENV_DIR} 26 | ${VENV_DIR}/bin/pip install ${DIST_DIR}/dbt_oracle-${VERSION}-py3-none-any.whl 27 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt --version 28 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt debug --profiles-dir ./ 29 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt deps --profiles-dir ./ 30 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run-operation drop_schema --args 'relation: ${DBT_ORACLE_SCHEMA}' --profiles-dir ./ 31 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt deps --profiles-dir ./ 32 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt seed --profiles-dir ./ 33 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run --profiles-dir ./ 34 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt test --profiles-dir ./ 35 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt test --profiles-dir ./ 36 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run --profiles-dir ./ 37 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt snapshot --profiles-dir ./ 38 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt snapshot --profiles-dir ./ 39 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt show --limit 4 --select people --profiles-dir ./ 40 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt docs generate --profiles-dir ./ 41 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run-operation drop_schema --args 'relation: ${DBT_ORACLE_SCHEMA}' --profiles-dir ./ 42 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt clean --profiles-dir ./ 43 | 44 | 45 | # Target to test a dbt-oracle package from PyPI. 46 | # This installs a dbt-oracle from PyPI and tests all dbt functionalities 47 | adbs_pypi_test: clean_venv 48 | ${PYTHON_3} -m venv ${VENV_DIR} 49 | ${VENV_DIR}/bin/pip install dbt-oracle==${VERSION} 50 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt --version 51 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt debug --profiles-dir ./ 52 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt deps --profiles-dir ./ 53 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run-operation drop_schema --args 'relation: ${DBT_ORACLE_SCHEMA}' --profiles-dir ./ 54 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt seed --profiles-dir ./ 55 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run --profiles-dir ./ 56 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt test --profiles-dir ./ 57 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt test --profiles-dir ./ 58 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run --profiles-dir ./ 59 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt snapshot --profiles-dir ./ 60 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt snapshot --profiles-dir ./ 61 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt show --limit 4 --select people --profiles-dir ./ 62 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt docs generate --profiles-dir ./ 63 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run-operation drop_schema --args 'relation: ${DBT_ORACLE_SCHEMA}' --profiles-dir ./ 64 | cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt clean --profiles-dir ./ 65 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022, Oracle and/or its affiliates 2 | Based on original contribution from Indicium; Copyright (c) 2020, Vitor Avancini -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dbt-oracle 2 | 3 | [![PyPI version](https://badge.fury.io/py/dbt-oracle.svg)](https://pypi.python.org/pypi/dbt-oracle) 4 | [![dbt-tests-adapter](https://github.com/oracle/dbt-oracle/actions/workflows/oracle-xe-adapter-tests.yml/badge.svg)](https://github.com/oracle/dbt-oracle/actions/workflows/oracle-xe-adapter-tests.yml) 5 | [![dbt-oracle docs](https://img.shields.io/badge/docs-read-blue)](https://docs.getdbt.com/reference/warehouse-setups/oracle-setup) 6 | [![dbt-oracle license](https://img.shields.io/badge/license-Apache%202.0-blue)][4] 7 | 8 | `dbt-oracle` implements [dbt (data build tool)](https://docs.getdbt.com/docs/introduction) functionalities for Oracle Autonomous Database. 9 | 10 | > Prior to version 1.0.0, dbt-oracle was created and maintained by [Indicium](https://indicium.tech/) on [their GitHub repo](https://github.com/techindicium/dbt-oracle). Contributors in this repo are credited for laying the groundwork and maintaining the adapter till version 0.4.3. 11 | From version 1.0.0, dbt-oracle is maintained and distributed by Oracle. 12 | 13 | 14 | ## Installation 15 | 16 | ```bash 17 | pip install dbt-oracle 18 | ``` 19 | 20 | 21 | ## Documentation 22 | 23 | Please refer to the [Oracle setup on dbt docs website][1] for documentation. 24 | 25 | ## Help 26 | 27 | Questions can be asked either in [db-oracle community Slack channel][6] or in [GitHub Discussions][7] 28 | 29 | Bugs reports or feature requests can be raised in [GitHub Issues][8] 30 | 31 | ## Sample project 32 | 33 | To get started, a sample dbt project can be found in the directory [/dbt_adbs_test_project][5]. 34 | 35 | ## Contributing 36 | 37 | This project welcomes contributions from the community. Before submitting a pull request, please [review our contribution guide](./CONTRIBUTING.md). 38 | 39 | ## Security 40 | 41 | Please consult the [security guide](./SECURITY.md) for our responsible security vulnerability disclosure process. 42 | 43 | ## License 44 | dbt-oracle is licensed under Apache 2.0 License which you can find [here][4]. 45 | 46 | [1]: https://docs.getdbt.com/reference/warehouse-profiles/oracle-profile 47 | [2]: https://github.com/oracle/dbt-oracle/blob/main/CONTRIBUTING.md 48 | [3]: https://github.com/oracle/dbt-oracle/blob/main/SECURITY.md 49 | [4]: https://github.com/oracle/dbt-oracle/blob/main/LICENSE.txt 50 | [5]: https://github.com/oracle/dbt-oracle/tree/main/dbt_adbs_test_project 51 | [6]: https://getdbt.slack.com/archives/C01PWH4TXLY 52 | [7]: https://github.com/oracle/dbt-oracle/discussions 53 | [8]: https://github.com/oracle/dbt-oracle/issues 54 | 55 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting security vulnerabilities 2 | 3 | Oracle values the independent security research community and believes that 4 | responsible disclosure of security vulnerabilities helps us ensure the security 5 | and privacy of all our users. 6 | 7 | Please do NOT raise a GitHub Issue to report a security vulnerability. If you 8 | believe you have found a security vulnerability, please submit a report to 9 | [secalert_us@oracle.com][1] preferably with a proof of concept. Please review 10 | some additional information on [how to report security vulnerabilities to Oracle][2]. 11 | We encourage people who contact Oracle Security to use email encryption using 12 | [our encryption key][3]. 13 | 14 | We ask that you do not use other channels or contact the project maintainers 15 | directly. 16 | 17 | Non-vulnerability related security issues including ideas for new or improved 18 | security features are welcome on GitHub Issues. 19 | 20 | ## Security updates, alerts and bulletins 21 | 22 | Security updates will be released on a regular cadence. Many of our projects 23 | will typically release security fixes in conjunction with the 24 | Oracle Critical Patch Update program. Additional 25 | information, including past advisories, is available on our [security alerts][4] 26 | page. 27 | 28 | ## Security-related information 29 | 30 | We will provide security related information such as a threat model, considerations 31 | for secure use, or any known security issues in our documentation. Please note 32 | that labs and sample code are intended to demonstrate a concept and may not be 33 | sufficiently hardened for production use. 34 | 35 | [1]: mailto:secalert_us@oracle.com 36 | [2]: https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html 37 | [3]: https://www.oracle.com/security-alerts/encryptionkey.html 38 | [4]: https://www.oracle.com/security-alerts/ 39 | -------------------------------------------------------------------------------- /bin/create-pem-from-p12: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Script to generate PEM from PKCS12 5 | 6 | https://python-oracledb.readthedocs.io/en/latest/user_guide/connection_handling.html#creating-a-pem-file-for-python-oracledb-thin-mode 7 | 8 | """ 9 | 10 | import argparse 11 | import getpass 12 | import os 13 | 14 | from cryptography.hazmat.primitives.serialization import (pkcs12, 15 | Encoding, 16 | PrivateFormat, 17 | BestAvailableEncryption, 18 | NoEncryption) 19 | 20 | # parse command line 21 | parser = argparse.ArgumentParser(description="Generates PEM from PKCS#12") 22 | parser.add_argument("wallet_location", 23 | help="the directory in which the PKCS#12 encoded " 24 | "wallet file ewallet.p12 is found") 25 | parser.add_argument("--wallet-password", 26 | help="the password for the wallet which is used to " 27 | "decrypt the PKCS#12 encoded wallet file; if not " 28 | "specified, it will be requested securely") 29 | parser.add_argument("--no-encrypt", 30 | dest="encrypt", action="store_false", default=True, 31 | help="do not encrypt the converted PEM file with the " 32 | "wallet password") 33 | args = parser.parse_args() 34 | 35 | # validate arguments and acquire password if one was not specified 36 | pkcs12_file_name = os.path.join(args.wallet_location, "ewallet.p12") 37 | if not os.path.exists(pkcs12_file_name): 38 | msg = f"wallet location {args.wallet_location} does not contain " \ 39 | "ewallet.p12" 40 | raise Exception(msg) 41 | if args.wallet_password is None: 42 | args.wallet_password = getpass.getpass() 43 | 44 | pem_file_name = os.path.join(args.wallet_location, "ewallet.pem") 45 | pkcs12_data = open(pkcs12_file_name, "rb").read() 46 | result = pkcs12.load_key_and_certificates(pkcs12_data, 47 | args.wallet_password.encode()) 48 | private_key, certificate, additional_certificates = result 49 | if args.encrypt: 50 | encryptor = BestAvailableEncryption(args.wallet_password.encode()) 51 | else: 52 | encryptor = NoEncryption() 53 | with open(pem_file_name, "wb") as f: 54 | f.write(private_key.private_bytes(Encoding.PEM, 55 | PrivateFormat.PKCS8, 56 | encryptor)) 57 | f.write(certificate.public_bytes(Encoding.PEM)) 58 | for cert in additional_certificates: 59 | f.write(cert.public_bytes(Encoding.PEM)) 60 | print("PEM file", pem_file_name, "written.") 61 | -------------------------------------------------------------------------------- /dbt/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | from pkgutil import extend_path 18 | 19 | __path__ = extend_path(__path__, __name__) 20 | -------------------------------------------------------------------------------- /dbt/adapters/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | __path__ = __import__('pkgutil').extend_path(__path__, __name__) 18 | -------------------------------------------------------------------------------- /dbt/adapters/oracle/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2024, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | from dbt.adapters.oracle.connections import OracleAdapterConnectionManager 18 | from dbt.adapters.oracle.connections import OracleAdapterCredentials 19 | from dbt.adapters.oracle.impl import OracleAdapter 20 | from dbt.adapters.base import AdapterPlugin 21 | from dbt.include import oracle 22 | 23 | Plugin = AdapterPlugin( 24 | adapter=OracleAdapter, 25 | credentials=OracleAdapterCredentials, 26 | include_path=oracle.PACKAGE_PATH 27 | ) 28 | -------------------------------------------------------------------------------- /dbt/adapters/oracle/__version__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2024, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | version = "1.9.1" 18 | -------------------------------------------------------------------------------- /dbt/adapters/oracle/column.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | from dataclasses import dataclass 18 | from typing import Dict, ClassVar 19 | 20 | 21 | from dbt.adapters.base.column import Column 22 | from dbt.adapters.oracle.keyword_catalog import KEYWORDS 23 | 24 | 25 | @dataclass 26 | class OracleColumn(Column): 27 | # https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html#GUID-A3C0D836-BADB-44E5-A5D4-265BA5968483 28 | 29 | TYPE_LABELS: ClassVar[Dict[str, str]] = { 30 | "STRING": "VARCHAR2(4000)", 31 | "TIMESTAMP": "TIMESTAMP", 32 | "FLOAT": "NUMBER", 33 | "INTEGER": "INTEGER", 34 | } 35 | 36 | STRING_DATATYPES = {'char', 'nchar', 'varchar', 'varchar2', 'nvarchar2'} 37 | NUMBER_DATATYPES = {'number', 'float'} 38 | 39 | @property 40 | def data_type(self) -> str: 41 | if self.is_string(): 42 | return self.oracle_string_type(self.dtype, self.string_size()) 43 | elif self.is_numeric(): 44 | return self.numeric_type(self.dtype, self.numeric_precision, self.numeric_scale) 45 | else: 46 | return self.dtype 47 | 48 | @classmethod 49 | def oracle_string_type(cls, dtype: str, size: int = None): 50 | """ 51 | - CHAR(SIZE) 52 | - VARCHAR2(SIZE) 53 | - NCHAR(SIZE) or NCHAR 54 | - NVARCHAR2(SIZE) 55 | """ 56 | if size is None: 57 | return dtype 58 | else: 59 | return "{}({})".format(dtype, size) 60 | 61 | def is_numeric(self) -> bool: 62 | if self.dtype.lower() in self.NUMBER_DATATYPES: 63 | return True 64 | return super().is_numeric() 65 | 66 | def is_string(self) -> bool: 67 | if self.dtype.lower() in self.STRING_DATATYPES: 68 | return True 69 | return super().is_string() 70 | -------------------------------------------------------------------------------- /dbt/adapters/oracle/relation_configs/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | from dbt.adapters.oracle.relation_configs.materialized_view import ( 18 | OracleMaterializedViewConfig, 19 | OracleMaterializedViewConfigChangeset, 20 | OracleBuildModeConfigChange, 21 | OracleRefreshModeConfigChange, 22 | OracleQueryConfigChange, 23 | OracleQueryRewriteConfigChange, 24 | OracleRefreshMethodConfigChange) 25 | 26 | from dbt.adapters.oracle.relation_configs.policies import ( 27 | OracleQuotePolicy, 28 | OracleIncludePolicy) 29 | -------------------------------------------------------------------------------- /dbt/adapters/oracle/relation_configs/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | from dataclasses import dataclass 18 | from typing import Any, Dict, Optional 19 | 20 | import agate 21 | from dbt.adapters.base.relation import Policy 22 | from dbt.adapters.relation_configs import ( 23 | RelationConfigBase, 24 | RelationResults, 25 | ) 26 | from dbt.contracts.graph.nodes import ModelNode 27 | from dbt.adapters.contracts.relation import ComponentName 28 | 29 | from dbt.adapters.oracle.relation_configs.policies import ( 30 | OracleQuotePolicy, 31 | OracleIncludePolicy, 32 | ) 33 | 34 | 35 | @dataclass(frozen=True, eq=True, unsafe_hash=True) 36 | class OracleRelationConfigBase(RelationConfigBase): 37 | """ 38 | This base class implements a few boilerplate methods and provides some light structure for Oracle relations. 39 | """ 40 | 41 | @classmethod 42 | def include_policy(cls) -> Policy: 43 | return OracleIncludePolicy() 44 | 45 | @classmethod 46 | def quote_policy(cls) -> Policy: 47 | return OracleQuotePolicy() 48 | 49 | @classmethod 50 | def from_model_node(cls, model_node: ModelNode): 51 | relation_config = cls.parse_model_node(model_node) 52 | relation = cls.from_dict(relation_config) 53 | return relation 54 | 55 | @classmethod 56 | def parse_model_node(cls, model_node: ModelNode) -> Dict[str, Any]: 57 | raise NotImplementedError( 58 | "`parse_model_node()` needs to be implemented on this RelationConfigBase instance" 59 | ) 60 | 61 | @classmethod 62 | def from_relation_results(cls, relation_results: RelationResults): 63 | relation_config = cls.parse_relation_results(relation_results) 64 | relation = cls.from_dict(relation_config) 65 | return relation 66 | 67 | @classmethod 68 | def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str, Any]: 69 | raise NotImplementedError( 70 | "`parse_relation_results()` needs to be implemented on this RelationConfigBase instance" 71 | ) 72 | 73 | @classmethod 74 | def _render_part(cls, component: ComponentName, value: Optional[str]) -> Optional[str]: 75 | if cls.include_policy().get_part(component) and value: 76 | if cls.quote_policy().get_part(component): 77 | return f'"{value}"' 78 | return value.lower() 79 | return None 80 | 81 | @classmethod 82 | def _get_first_row(cls, results: agate.Table) -> agate.Row: 83 | try: 84 | return results.rows[0] 85 | except IndexError: 86 | return agate.Row(values=set()) 87 | -------------------------------------------------------------------------------- /dbt/adapters/oracle/relation_configs/policies.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2024, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | from dataclasses import dataclass 18 | 19 | from dbt.adapters.base.relation import Policy 20 | from dbt_common.dataclass_schema import StrEnum 21 | 22 | 23 | @dataclass 24 | class OracleQuotePolicy(Policy): 25 | database: bool = False 26 | schema: bool = False 27 | identifier: bool = False 28 | 29 | 30 | @dataclass 31 | class OracleIncludePolicy(Policy): 32 | database: bool = False 33 | schema: bool = True 34 | identifier: bool = True 35 | -------------------------------------------------------------------------------- /dbt/adapters/oracle/sample_profiles.yml: -------------------------------------------------------------------------------- 1 | default: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: oracle 6 | protocol: tcps 7 | host: "{{ env_var('DBT_ORACLE_HOST') }}" 8 | user: "{{ env_var('DBT_ORACLE_USER') }}" 9 | password: "{{ env_var('DBT_ORACLE_PASSWORD') }}" 10 | database: "{{ env_var('DBT_ORACLE_DATABASE') }}" 11 | port: 1522 12 | service: "{{ env_var('DBT_ORACLE_SERVICE') }}" 13 | schema: "{{ env_var('DBT_ORACLE_SCHEMA') }}" 14 | threads: 1 15 | -------------------------------------------------------------------------------- /dbt/include/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | from pkgutil import extend_path 18 | 19 | __path__ = extend_path(__path__, __name__) 20 | -------------------------------------------------------------------------------- /dbt/include/oracle/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | import os 18 | PACKAGE_PATH = os.path.dirname(__file__) 19 | -------------------------------------------------------------------------------- /dbt/include/oracle/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | name: dbt_oracle 3 | config-version: 2 4 | version: 1.0 5 | 6 | quoting: 7 | database: false 8 | schema: false 9 | identifier: false 10 | 11 | macro-paths: ["macros"] 12 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/apply_grants.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | 18 | {% macro oracle__get_show_grant_sql(relation) %} 19 | {# SQL that returns the current grants (grantee-privilege pairs) #} 20 | SELECT grantee as "grantee", privilege as "privilege_type" 21 | FROM SYS.ALL_TAB_PRIVS 22 | WHERE UPPER(table_name) = UPPER('{{ relation.identifier }}') 23 | {% if relation.schema %} 24 | AND UPPER(table_schema) = UPPER('{{ relation.schema }}') 25 | {% endif %} 26 | {% endmacro %} 27 | 28 | {% macro oracle__call_dcl_statements(dcl_statement_list) %} 29 | {# Run each grant/revoke statement against the database. This is the culmination of apply_grants() #} 30 | {% for dcl_statement in dcl_statement_list %} 31 | {% do run_query(dcl_statement) %} 32 | {% endfor %} 33 | {% endmacro %} 34 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/columns.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {% macro oracle__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} 18 | 19 | {% if add_columns is none %} 20 | {% set add_columns = [] %} 21 | {% endif %} 22 | {% if remove_columns is none %} 23 | {% set remove_columns = [] %} 24 | {% endif %} 25 | {# To avoid ORA-12987: cannot combine drop column with other operations, we execute 2 different SQL for add and drop respectively #} 26 | 27 | {% if add_columns|length > 0 %} 28 | {% set add_sql %} 29 | ALTER {{ relation.type }} {{ relation }} 30 | ADD ( 31 | {% for column in add_columns %} 32 | {{ adapter.check_and_quote_identifier(column.name, model.columns) }} {{ column.data_type }}{{ ',' if not loop.last }} 33 | {% endfor %} 34 | ) 35 | {% endset %} 36 | {% do run_query(add_sql)%} 37 | {% endif %} 38 | 39 | {% if remove_columns|length > 0 %} 40 | {% set remove_sql %} 41 | ALTER {{ relation.type }} {{ relation }} 42 | DROP ( 43 | {% for column in remove_columns %} 44 | {{ adapter.check_and_quote_identifier(column.name, model.columns) }}{{ ',' if not loop.last }} 45 | {% endfor %} 46 | ) CASCADE CONSTRAINTS 47 | {% endset %} 48 | {% do run_query(remove_sql)%} 49 | {% endif %} 50 | {% endmacro %} 51 | 52 | {% macro get_quoted_column_csv(model, column_names) %} 53 | {%- set quoted = [] -%} 54 | {% for col in column_names %} 55 | {%- do quoted.append(adapter.check_and_quote_identifier(col, model.columns)) -%} 56 | {% endfor %} 57 | {%- set cols_csv = quoted | join(', ') -%} 58 | {{ return(cols_csv) }} 59 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/include/oracle/macros/freshness.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% macro oracle__collect_freshness(source, loaded_at_field, filter) %} 17 | {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%} 18 | select 19 | {% if loaded_at_field | upper == 'ORA_ROWSCN' %} 20 | SCN_TO_TIMESTAMP(max(ORA_ROWSCN)) as max_loaded_at, 21 | {% else %} 22 | max({{ loaded_at_field }}) as max_loaded_at, 23 | {% endif %} 24 | {{ current_timestamp() }} as snapshotted_at 25 | from {{ source }} 26 | {% if filter %} 27 | where {{ filter }} 28 | {% endif %} 29 | {% endcall %} 30 | {{ return(load_result('collect_freshness')) }} 31 | {% endmacro %} 32 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/materializations/incremental/incremental.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {% materialization incremental, adapter='oracle', supported_languages=['sql', 'python'] %} 18 | 19 | {% set unique_key = config.get('unique_key') %} 20 | {% set full_refresh_mode = should_full_refresh() %} 21 | {%- set language = model['language'] -%} 22 | {% set target_relation = this.incorporate(type='table') %} 23 | {% set existing_relation = load_relation(this) %} 24 | {% set tmp_relation = make_temp_relation(this) %} 25 | {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %} 26 | {% set grant_config = config.get('grants') %} 27 | 28 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 29 | 30 | -- `BEGIN` happens here: 31 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 32 | 33 | {% set to_drop = [] %} 34 | {% if existing_relation is none %} 35 | {% set build_sql = create_table_as(False, target_relation, sql, language) %} 36 | {% elif existing_relation.is_view or full_refresh_mode %} 37 | {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #} 38 | {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %} 39 | {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %} 40 | {% do adapter.drop_relation(backup_relation) %} 41 | {% if existing_relation.is_view %} 42 | {% do adapter.drop_relation(existing_relation) %} 43 | {% else %} 44 | {% do adapter.rename_relation(existing_relation, backup_relation) %} 45 | {% endif %} 46 | {% set build_sql = create_table_as(False, target_relation, sql, language) %} 47 | {% do to_drop.append(backup_relation) %} 48 | {% else %} 49 | {% set tmp_relation = make_temp_relation(target_relation) %} 50 | {% do to_drop.append(tmp_relation) %} 51 | {% call statement("make_tmp_relation", language=language) %} 52 | {{create_table_as(True, tmp_relation, sql, language)}} 53 | {% endcall %} 54 | {#-- After this language should be SQL --#} 55 | {% set language = 'sql' %} 56 | {% do adapter.expand_target_column_types( 57 | from_relation=tmp_relation, 58 | to_relation=target_relation) %} 59 | {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %} 60 | {% if not dest_columns %} 61 | {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} 62 | {% endif %} 63 | 64 | {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} 65 | {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} 66 | {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} 67 | {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} 68 | {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} 69 | {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} 70 | 71 | {% endif %} 72 | 73 | {% call statement("main", language=language) %} 74 | {{ build_sql }} 75 | {% endcall %} 76 | 77 | {% do persist_docs(target_relation, model) %} 78 | 79 | {{ run_hooks(post_hooks, inside_transaction=True) }} 80 | 81 | -- `COMMIT` happens here 82 | {% do adapter.commit() %} 83 | 84 | {% for rel in to_drop %} 85 | {% do adapter.truncate_relation(rel) %} 86 | {% do adapter.drop_relation(rel) %} 87 | {% endfor %} 88 | 89 | {{ run_hooks(post_hooks, inside_transaction=False) }} 90 | 91 | {% set should_revoke = should_revoke(existing_relation.is_table, full_refresh_mode) %} 92 | {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} 93 | 94 | {{ return({'relations': [target_relation]}) }} 95 | 96 | {%- endmaterialization %} 97 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/materializations/materialized_view/materialized_view.sql: -------------------------------------------------------------------------------- 1 | {% macro oracle__refresh_materialized_view(relation) -%} 2 | BEGIN 3 | DBMS_MVIEW.REFRESH('{{ relation }}'); 4 | END; 5 | {%- endmacro %} 6 | 7 | {% macro oracle__drop_materialized_view(relation) -%} 8 | DROP MATERIALIZED VIEW {{ relation }} 9 | {%- endmacro %} 10 | 11 | 12 | {% macro oracle__get_create_materialized_view_as_sql(relation, sql) %} 13 | {%- set materialized_view = relation.from_runtime_config(config) -%} 14 | create materialized view {{ relation }} 15 | BUILD {{ materialized_view.build_mode }} 16 | REFRESH {{ materialized_view.refresh_method }} ON {{ materialized_view.refresh_mode }} 17 | {{ materialized_view.query_rewrite }} QUERY REWRITE 18 | AS {{ materialized_view.query }} 19 | {% endmacro %} 20 | 21 | {% macro oracle__get_alter_materialized_view_as_sql( 22 | relation, 23 | configuration_changes, 24 | sql, 25 | existing_relation, 26 | backup_relation, 27 | intermediate_relation 28 | ) %} 29 | {% if configuration_changes.requires_full_refresh %} 30 | {{- log('Applying REPLACE to: ' ~ existing_relation ) -}} 31 | {{ oracle__drop_relation(existing_relation) }} 32 | {{ oracle__get_create_materialized_view_as_sql(relation, sql) }} 33 | {% else %} 34 | 35 | {%- set refresh_method = configuration_changes.refresh_method -%} 36 | {%- set refresh_mode = configuration_changes.refresh_mode -%} 37 | {%- set build_mode = configuration_changes.build_mode -%} 38 | {%- set query_rewrite = configuration_changes.query_rewrite -%} 39 | 40 | ALTER MATERIALIZED VIEW {{ relation }} 41 | BUILD {{ build_mode.context }} 42 | REFRESH {{ refresh_method.context }} ON {{ refresh_mode.context }} 43 | {{ query_rewrite.context }} QUERY REWRITE 44 | 45 | {%- endif -%} 46 | 47 | {% endmacro %} 48 | 49 | {% macro oracle__get_materialized_view_configuration_changes(existing_relation, new_config) %} 50 | {% set _existing_materialized_view = oracle__describe_materialized_view_config(existing_relation) %} 51 | {% set _configuration_changes = existing_relation.materialized_view_config_changeset(_existing_materialized_view, new_config) %} 52 | {% do return(_configuration_changes) %} 53 | {% endmacro %} 54 | 55 | 56 | {% macro oracle__describe_materialized_view_config(relation) -%} 57 | {%- set _materialized_view_sql -%} 58 | SELECT query, 59 | owner, 60 | mview_name, 61 | rewrite_enabled, 62 | refresh_mode, 63 | refresh_method, 64 | build_mode, 65 | fast_refreshable, 66 | last_refresh_date, 67 | compile_state, 68 | last_refresh_type 69 | FROM sys.all_mviews 70 | WHERE mview_name = '{{ relation.identifier }}' 71 | {%- endset %} 72 | {% set _materialized_view = run_query(_materialized_view_sql) %} 73 | {% do return({'materialized_view': _materialized_view}) %} 74 | {%- endmacro %} 75 | 76 | 77 | {% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %} 78 | 79 | {% set full_refresh_mode = should_full_refresh() %} 80 | 81 | -- determine the scenario we're in: create, full_refresh, alter, refresh data 82 | {% if existing_relation is none %} 83 | {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %} 84 | {% elif full_refresh_mode or not existing_relation.is_materialized_view %} 85 | {{- log('Applying REPLACE to: ' ~ existing_relation ) -}} 86 | {{ oracle__drop_relation(existing_relation) }} 87 | {% set build_sql = oracle__get_create_materialized_view_as_sql(target_relation, sql) %} 88 | {% else %} 89 | 90 | -- get config options 91 | {% set on_configuration_change = config.get('on_configuration_change') %} 92 | {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %} 93 | 94 | {% if configuration_changes is none %} 95 | {% set build_sql = refresh_materialized_view(target_relation) %} 96 | 97 | {% elif on_configuration_change == 'apply' %} 98 | {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %} 99 | {% elif on_configuration_change == 'continue' %} 100 | {% set build_sql = '' %} 101 | {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }} 102 | {% elif on_configuration_change == 'fail' %} 103 | {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation ~ "`") }} 104 | 105 | {% else %} 106 | -- this only happens if the user provides a value other than `apply`, 'skip', 'fail' 107 | {{ exceptions.raise_compiler_error("Unexpected configuration scenario") }} 108 | 109 | {% endif %} 110 | 111 | {% endif %} 112 | 113 | {% do return(build_sql) %} 114 | 115 | {% endmacro %} 116 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/materializations/python_model/python.sql: -------------------------------------------------------------------------------- 1 | {% macro build_ref_function(model) %} 2 | {%- set ref_dict = {} -%} 3 | {%- for _ref in model.refs -%} 4 | {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %} 5 | {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%} 6 | {%- if _ref.get('version') -%} 7 | {% do _ref_args.extend(["v" ~ _ref['version']]) %} 8 | {%- endif -%} 9 | {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%} 10 | {%- endfor -%} 11 | 12 | def ref(*args, **kwargs): 13 | refs = {{ ref_dict | tojson }} 14 | key = ".".join(args) 15 | version = kwargs.get("v") or kwargs.get("version") 16 | if version: 17 | key += f".v{version}" 18 | schema, table = refs[key].split(".") 19 | # Use oml.sync(schema=schema, table=table) 20 | dbt_load_df_function = kwargs.get("dbt_load_df_function") 21 | return dbt_load_df_function(schema=schema.upper(), table=table.upper()) 22 | 23 | {% endmacro %} 24 | 25 | {% macro build_source_function(model) %} 26 | 27 | {%- set source_dict = {} -%} 28 | {%- for _source in model.sources -%} 29 | {%- set resolved = source(*_source) -%} 30 | {%- do source_dict.update({_source | join("."): resolve_model_name(resolved)}) -%} 31 | {%- endfor -%} 32 | 33 | def source(*args, dbt_load_df_function): 34 | sources = {{ source_dict | tojson }} 35 | key = ".".join(args) 36 | schema, table = sources[key].split(".") 37 | # Use oml.sync(schema=schema, table=table) 38 | return dbt_load_df_function(schema=schema.upper(), table=table.upper()) 39 | 40 | {% endmacro %} 41 | 42 | {% macro build_config_dict(model) %} 43 | {%- set config_dict = {} -%} 44 | {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %} 45 | {%- for key, default in config_dbt_used -%} 46 | {# weird type testing with enum, would be much easier to write this logic in Python! #} 47 | {%- if key == 'language' -%} 48 | {%- set value = 'python' -%} 49 | {%- endif -%} 50 | {%- set value = model.config.get(key, default) -%} 51 | {%- do config_dict.update({key: value}) -%} 52 | {%- endfor -%} 53 | config_dict = {{ config_dict }} 54 | {% endmacro %} 55 | 56 | {% macro py_script_postfix(model) %} 57 | def main(action, client_identifier, clientinfo, module): 58 | import oml 59 | def set_connection_attributes(): 60 | try: 61 | connection = oml.core.methods._get_conn() 62 | except Exception: 63 | raise 64 | else: 65 | session_info = {"action": action, 66 | "client_identifier": client_identifier, 67 | "clientinfo": clientinfo, 68 | "module": module} 69 | for k, v in session_info.items(): 70 | try: 71 | setattr(connection, k, v) 72 | except AttributeError: 73 | pass # ok to be silent, ADB-S Python runtime, complains about print statements 74 | 75 | set_connection_attributes() 76 | 77 | import pandas as pd 78 | {{ build_ref_function(model ) }} 79 | {{ build_source_function(model ) }} 80 | {{ build_config_dict(model) }} 81 | 82 | class config: 83 | def __init__(self, *args, **kwargs): 84 | pass 85 | 86 | @staticmethod 87 | def get(key, default=None): 88 | return config_dict.get(key, default) 89 | 90 | class this: 91 | """dbt.this() or dbt.this.identifier""" 92 | database = "{{ this.database }}" 93 | schema = "{{ this.schema }}" 94 | identifier = "{{ this.identifier }}" 95 | def __repr__(self): 96 | return "{{ this }}" 97 | 98 | 99 | class dbtObj: 100 | def __init__(self, load_df_function) -> None: 101 | self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function) 102 | self.ref = lambda *args: ref(*args, dbt_load_df_function=load_df_function) 103 | self.config = config 104 | self.this = this() 105 | self.is_incremental = {{ is_incremental() }} 106 | 107 | def materialize(df, table, session): 108 | if isinstance(df, pd.core.frame.DataFrame): 109 | oml.create(df, table=table) 110 | elif isinstance(df, oml.core.frame.DataFrame): 111 | df.materialize(table=table) 112 | 113 | {{ model.raw_code | indent(width=4, first=False, blank=True)}} 114 | 115 | 116 | {{py_script_comment()}} 117 | {% endmacro %} 118 | 119 | {#-- entry point for add instructions for running compiled_code --#} 120 | {%macro py_script_comment()%} 121 | {%endmacro%} 122 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/materializations/seed/seed.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {% macro oracle_basic_load_csv_rows(model, batch_size, agate_table) %} 18 | 19 | {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} 20 | {% set bindings = [] %} 21 | 22 | {% set statements = [] %} 23 | 24 | {% for chunk in agate_table.rows | batch(batch_size) %} 25 | {% set bindings = [] %} 26 | 27 | {% for row in chunk %} 28 | {% do bindings.extend(row) %} 29 | {% endfor %} 30 | 31 | {% set sql %} 32 | insert all 33 | {% for row in chunk -%} 34 | into {{ this.render() }} ({{ cols_sql }}) values( 35 | {%- for column in agate_table.column_names -%} 36 | :p{{ loop.index }} 37 | {%- if not loop.last%},{%- endif %} 38 | {%- endfor %}) 39 | {% endfor %} 40 | select * from dual 41 | {% endset %} 42 | 43 | {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} 44 | 45 | {% if loop.index0 == 0 %} 46 | {% do statements.append(sql) %} 47 | {% endif %} 48 | {% endfor %} 49 | 50 | {# Return SQL so we can render it out into the compiled files #} 51 | {{ return(statements[0]) }} 52 | {% endmacro %} 53 | 54 | {% macro oracle__load_csv_rows(model, agate_table) %} 55 | {{ return(oracle_basic_load_csv_rows(model, 100, agate_table) )}} 56 | {% endmacro %} 57 | 58 | 59 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/materializations/snapshot/snapshot_merge.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {% macro oracle__snapshot_merge_sql(target, source, insert_cols) -%} 18 | {%- set insert_cols_csv = [] -%} 19 | 20 | {% for column in insert_cols %} 21 | {% do insert_cols_csv.append("DBT_INTERNAL_SOURCE." + column) %} 22 | {% endfor %} 23 | 24 | {%- set dest_cols_csv = [] -%} 25 | 26 | {% for column in insert_cols %} 27 | {% do dest_cols_csv.append("DBT_INTERNAL_DEST." + column) %} 28 | {% endfor %} 29 | 30 | {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} 31 | 32 | merge into {{ target }} DBT_INTERNAL_DEST 33 | using {{ source }} DBT_INTERNAL_SOURCE 34 | on (DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}) 35 | 36 | when matched 37 | then update 38 | set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} 39 | where 40 | {% if config.get("dbt_valid_to_current") %} 41 | (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or 42 | DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null) 43 | {% else %} 44 | DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null 45 | {% endif %} 46 | and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') 47 | when not matched 48 | then insert ({{ dest_cols_csv | join(', ') }}) 49 | values ({{ insert_cols_csv | join(', ') }}) 50 | where DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' 51 | {% endmacro %} 52 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/materializations/tests/helpers.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} 17 | {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }} 18 | {%- endmacro %} 19 | 20 | {% macro oracle__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} 21 | select 22 | {{ fail_calc }} as failures, 23 | case when {{ fail_calc }} {{ warn_if }} then 1 else 0 end as should_warn, 24 | case when {{ fail_calc }} {{ error_if }} then 1 else 0 end as should_error 25 | from ( 26 | {{ main_sql }} 27 | {{ "limit " ~ limit if limit != none }} 28 | ) dbt_internal_test 29 | {%- endmacro %} 30 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/materializations/view/view.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright 2021 dbt Labs, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {%- materialization view, adapter='oracle' -%} 18 | 19 | {%- set identifier = model['alias'] -%} 20 | {%- set grant_config = config.get('grants') -%} 21 | {%- set backup_identifier = model['alias'] + '__dbt_backup' -%} 22 | 23 | {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} 24 | {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, 25 | type='view') -%} 26 | /* 27 | This relation (probably) doesn't exist yet. If it does exist, it's a leftover from 28 | a previous run, and we're going to try to drop it immediately. At the end of this 29 | materialization, we're going to rename the "old_relation" to this identifier, 30 | and then we're going to drop it. In order to make sure we run the correct one of: 31 | - drop view ... 32 | - drop table ... 33 | 34 | We need to set the type of this relation to be the type of the old_relation, if it exists, 35 | or else "view" as a sane default if it does not. Note that if the old_relation does not 36 | exist, then there is nothing to move out of the way and subsequentally drop. In that case, 37 | this relation will be effectively unused. 38 | */ 39 | {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%} 40 | {%- set backup_relation = api.Relation.create(identifier=backup_identifier, 41 | schema=schema, database=database, 42 | type=backup_relation_type) -%} 43 | -- as above, the backup_relation should not already exist 44 | {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier, 45 | schema=schema, 46 | database=database) -%} 47 | 48 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 49 | 50 | {{ drop_relation_if_exists(preexisting_backup_relation) }} 51 | 52 | -- `BEGIN` happens here: 53 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 54 | 55 | -- if old_relation was a table 56 | {% if old_relation is not none and old_relation.type == 'table' %} 57 | {{ adapter.rename_relation(old_relation, backup_relation) }} 58 | {% endif %} 59 | 60 | -- build model 61 | {% call statement('main') -%} 62 | {{ create_view_as(target_relation, sql) }} 63 | {%- endcall %} 64 | 65 | {% do persist_docs(target_relation, model) %} 66 | 67 | {{ run_hooks(post_hooks, inside_transaction=True) }} 68 | 69 | {{ adapter.commit() }} 70 | 71 | {{ drop_relation_if_exists(backup_relation) }} 72 | 73 | {{ run_hooks(post_hooks, inside_transaction=False) }} 74 | 75 | {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %} 76 | {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} 77 | 78 | {{ return({'relations': [target_relation]}) }} 79 | 80 | {%- endmaterialization -%} 81 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/relations/drop.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2024, Oracle and/or its affiliates. 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | #} 15 | {%- macro oracle__get_drop_sql(relation) -%} 16 | DECLARE 17 | dne_942 EXCEPTION; 18 | PRAGMA EXCEPTION_INIT(dne_942, -942); 19 | attempted_ddl_on_in_use_GTT EXCEPTION; 20 | pragma EXCEPTION_INIT(attempted_ddl_on_in_use_GTT, -14452); 21 | mv_dne_12003 EXCEPTION; 22 | PRAGMA EXCEPTION_INIT(mv_dne_12003, -12003); 23 | BEGIN 24 | SAVEPOINT start_transaction; 25 | {%- if relation.is_materialized_view -%} 26 | EXECUTE IMMEDIATE '{{ oracle__drop_materialized_view(relation) }}'; 27 | {%- elif relation.is_table -%} 28 | EXECUTE IMMEDIATE 'DROP table {{ relation }} cascade constraints purge'; 29 | {%- else -%} 30 | EXECUTE IMMEDIATE 'DROP {{ relation.type }} {{ relation }} cascade constraints'; 31 | {%- endif -%} 32 | COMMIT; 33 | EXCEPTION 34 | WHEN attempted_ddl_on_in_use_GTT THEN 35 | NULL; -- if it its a global temporary table, leave it alone. 36 | WHEN dne_942 THEN 37 | NULL; 38 | WHEN mv_dne_12003 THEN 39 | NULL; 40 | END; 41 | {%- endmacro -%} 42 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/schema_tests.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {% macro oracle__test_accepted_values(model, column_name, values, quote=True) %} 18 | 19 | with all_values as ( 20 | 21 | select distinct 22 | {{ column_name }} as value_field 23 | 24 | from {{ model }} 25 | 26 | ), 27 | 28 | validation_errors as ( 29 | 30 | select 31 | value_field 32 | 33 | from all_values 34 | where value_field not in ( 35 | {% for value in values -%} 36 | {% if quote -%} 37 | '{{ value }}' 38 | {%- else -%} 39 | {{ value }} 40 | {%- endif -%} 41 | {%- if not loop.last -%},{%- endif %} 42 | {%- endfor %} 43 | ) 44 | ) 45 | 46 | select * from( 47 | select count(*) as not_accepted_values from validation_errors 48 | ) c where c.not_accepted_values != 0 49 | 50 | {% endmacro %} 51 | 52 | {% macro oracle__test_not_null(model, column_name) %} 53 | 54 | select * from ( 55 | select count(*) as null_count 56 | from {{ model }} 57 | where {{ column_name }} is null) c where c.null_count != 0 58 | 59 | {% endmacro %} 60 | 61 | {% macro oracle__test_relationships(model, column_name, to, field) %} 62 | 63 | select * from ( 64 | select count(*) as validation_errors 65 | from ( 66 | select {{ column_name }} as id from {{ model }} 67 | ) child 68 | left join ( 69 | select {{ field }} as id from {{ to }} 70 | ) parent on parent.id = child.id 71 | where child.id is not null 72 | and parent.id is null) c where c.validation_errors != 0 73 | 74 | {% endmacro %} 75 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/show.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2024, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | 17 | {% macro oracle__get_limit_subquery_sql(sql, limit) %} 18 | select * 19 | from ( 20 | {{ sql }} 21 | ) 22 | fetch first {{ limit }} rows only 23 | {% endmacro %} 24 | 25 | 26 | {% macro oracle__get_limit_sql(sql, limit) %} 27 | {{ compiled_code }} 28 | {% if limit is not none %} 29 | fetch first {{ limit }} rows only 30 | {%- endif -%} 31 | {% endmacro %} 32 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/update_legacy_snapshots.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2024, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | 17 | {# 18 | Legacy hash function ORA_HASH is known to have collisions in large datasets 19 | causing errors in snapshot merge statement. Please check the below Github 20 | issues: 21 | 22 | https://github.com/oracle/dbt-oracle/issues/52 23 | https://github.com/oracle/dbt-oracle/issues/102 24 | 25 | This hash function is used in the marcro oracle__snapshot_hash_arguments 26 | 27 | dbt-oracle 1.9 will switch to a stronger hash function - SHA256. Changing the 28 | hash function will invalidate existing snapshots.These helper macros will 29 | ensure a smoother transition to dbt-oracle 1.9. 30 | 31 | It is recommended for teams to switch to SHA256 hash function before 32 | dbt-oracle 1.9 using a 2-step process: 33 | 1. Create a macro oracle__snapshot_hash_arguments(args) in your dbt project 34 | Copy paste the contents of macro 35 | oracle__snapshot_standard_hash_arguments(args) shown below. This will become 36 | the default from dbt-oracle 1.9 37 | 38 | 2. Run the following operation on your snapshot table 39 | 40 | dbt --debug run-operation update_legacy_dbt_scd_id \ 41 | --args '{snapshot_table: PROMOTION_COSTS_SNAPSHOT, cols: ["promo_id", "dbt_updated_at"]}' 42 | 43 | #} 44 | 45 | {% macro oracle__snapshot_standard_hash_arguments(args) -%} 46 | STANDARD_HASH({%- for arg in args -%} 47 | coalesce(cast({{ arg }} as varchar(4000) ), '') 48 | {% if not loop.last %} || '|' || {% endif %} 49 | {%- endfor -%}, 'SHA256') 50 | {%- endmacro %} 51 | 52 | 53 | {% macro update_legacy_dbt_scd_id(snapshot_table, cols) -%} 54 | 55 | {%- call statement('update_legacy_dbt_scd_id_dtype') -%} 56 | BEGIN 57 | UPDATE {{ snapshot_table }} SET DBT_SCD_ID = NULL; 58 | COMMIT; 59 | EXECUTE IMMEDIATE 'ALTER TABLE {{ snapshot_table }} MODIFY (dbt_scd_id RAW(32))'; 60 | END; 61 | {%- endcall -%} 62 | 63 | {%- call statement('update_legacy_dbt_scd_id') -%} 64 | BEGIN 65 | UPDATE {{ snapshot_table }} 66 | SET dbt_scd_id = {{ oracle__snapshot_standard_hash_arguments(cols) }}; 67 | COMMIT; 68 | END; 69 | {%- endcall -%} 70 | {%- endmacro %} 71 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/cast_bool_to_text.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | 18 | {% macro oracle__cast_bool_to_text(bool_expression) %} 19 | CASE 20 | WHEN {{ bool_expression }} THEN 'true' 21 | ELSE 'false' 22 | END 23 | {% endmacro %} 24 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/data_types.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | 18 | {# bigint = int64 #} 19 | {%- macro oracle__type_bigint() -%} 20 | NUMBER(19) 21 | {%- endmacro -%} 22 | 23 | {# int = int32 #} 24 | {%- macro oracle__type_int() -%} 25 | NUMBER(10, 0) 26 | {%- endmacro -%} 27 | 28 | 29 | {% macro oracle__type_boolean() -%} 30 | NUMBER(1) 31 | {%- endmacro %} 32 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/date_spine.sql: -------------------------------------------------------------------------------- 1 | {% macro oracle__get_intervals_between(start_date, end_date, datepart) -%} 2 | {%- call statement('get_intervals_between', fetch_result=True) %} 3 | 4 | select {{ dbt.datediff(start_date, end_date, datepart) }} from dual 5 | 6 | {%- endcall -%} 7 | 8 | {%- set value_list = load_result('get_intervals_between') -%} 9 | 10 | {%- if value_list and value_list['data'] -%} 11 | {%- set values = value_list['data'] | map(attribute=0) | list %} 12 | {{ return(values[0]) }} 13 | {%- else -%} 14 | {{ return(1) }} 15 | {%- endif -%} 16 | 17 | {%- endmacro %} 18 | 19 | {% macro oracle__date_spine_all_periods(datepart, start_date, end_date) %} 20 | select ( 21 | {{ 22 | dbt.dateadd( 23 | datepart, 24 | "row_number() over (order by 1) - 1", 25 | start_date 26 | ) 27 | }} 28 | ) as date_{{datepart}} 29 | from ({{dbt.generate_series( 30 | dbt.get_intervals_between(start_date, end_date, datepart) 31 | )}}) 32 | {% endmacro %} 33 | 34 | {% macro oracle__date_spine(datepart, start_date, end_date) %} 35 | 36 | {# call as follows: 37 | 38 | date_spine( 39 | "day", 40 | "to_date('01/01/2016', 'mm/dd/yyyy')", 41 | "dbt.dateadd(week, 1, current_date)" 42 | ) #} 43 | 44 | select * 45 | from ({{oracle__date_spine_all_periods(datepart, start_date, end_date)}}) 46 | where date_{{datepart}} <= {{ end_date }} 47 | 48 | {% endmacro %} 49 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/dateadd.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | 18 | {% macro oracle__dateadd(datepart, interval, from_date_or_timestamp) %} 19 | {%- set single_quote = "\'" -%} 20 | {%- set D2S_INTERVAL_UNITS = ['DAY', 'HOUR', 'MINUTE', 'SECOND'] -%} 21 | {%- set M2Y_INTERVAL_UNITS = ['YEAR','MONTH'] -%} 22 | {%- if datepart.upper() in D2S_INTERVAL_UNITS -%} 23 | {{ from_date_or_timestamp }} + NUMTODSINTERVAL({{ interval }}, {{single_quote ~ datepart ~ single_quote}}) 24 | {%- elif datepart.upper() in M2Y_INTERVAL_UNITS -%} 25 | {{ from_date_or_timestamp }} + NUMTOYMINTERVAL({{ interval }}, {{single_quote ~ datepart ~ single_quote}}) 26 | {%- elif datepart.upper() == 'QUARTER' -%} 27 | ADD_MONTHS({{ from_date_or_timestamp }}, 3*{{ interval }}) 28 | {% elif datepart.upper() == 'WEEK' %} 29 | {{ from_date_or_timestamp }} + 7*{{ interval }} 30 | {%- endif -%} 31 | {% endmacro %} 32 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/datediff.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {# Returns difference (as an integer) in between 2 dates #} 18 | {% macro oracle__datediff(first_date, second_date, datepart) %} 19 | {% if datepart.upper() == 'YEAR' %} 20 | ROUND(MONTHS_BETWEEN(TRUNC(CAST({{second_date}} AS DATE), 'YEAR'), TRUNC(CAST({{first_date}} AS DATE), 'YEAR'))/12) 21 | {% elif datepart.upper() == 'QUARTER' %} 22 | ROUND(MONTHS_BETWEEN(TRUNC(CAST({{second_date}} AS DATE), 'Q'), TRUNC(CAST({{first_date}} AS DATE), 'Q'))/3) 23 | {% elif datepart.upper() == 'MONTH'%} 24 | ROUND(MONTHS_BETWEEN(TRUNC(CAST({{second_date}} AS DATE), 'MONTH'), TRUNC(CAST({{first_date}} AS DATE), 'MONTH'))) 25 | {% elif datepart.upper() == 'WEEK' %} 26 | ROUND((TRUNC(CAST({{ second_date }} AS DATE), 'DAY') - TRUNC(CAST({{ first_date }} AS DATE), 'DAY'))/7) 27 | {% elif datepart.upper() == 'DAY' %} 28 | ROUND(TRUNC(CAST({{ second_date }} AS DATE), 'DD') - TRUNC(CAST({{ first_date }} AS DATE), 'DD')) 29 | {% elif datepart.upper() == 'HOUR' %} 30 | ROUND((TRUNC(CAST({{ second_date }} AS DATE), 'HH') - TRUNC(CAST({{ first_date }} AS DATE), 'HH'))*24) 31 | {% elif datepart.upper() == 'MINUTE' %} 32 | ROUND((TRUNC(CAST({{ second_date }} AS DATE), 'MI') - TRUNC(CAST({{ first_date }} AS DATE), 'MI'))*24*60) 33 | {% elif datepart.upper() == 'SECOND' %} 34 | EXTRACT(DAY FROM (CAST({{ second_date }} AS TIMESTAMP) - CAST({{ first_date }} AS TIMESTAMP)))*24*60*60 35 | +EXTRACT(HOUR FROM (CAST({{ second_date }} AS TIMESTAMP) - CAST({{ first_date }} AS TIMESTAMP)))*60*60 36 | +EXTRACT(MINUTE FROM (CAST({{ second_date }} AS TIMESTAMP) - CAST({{ first_date }} AS TIMESTAMP)))*60 37 | +EXTRACT(SECOND FROM (CAST({{ second_date }} AS TIMESTAMP) - CAST({{ first_date }} AS TIMESTAMP))) 38 | {% endif %} 39 | {% endmacro %} 40 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/datetrunc.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {# Returns date with the 'datepart' portion truncated #} 18 | {% macro oracle__date_trunc(datepart, date) %} 19 | {% if datepart.upper() == 'QUARTER' %} 20 | {% set datepart = 'Q' %} 21 | {% endif %} 22 | {% if datepart.upper() == 'WEEK' %} 23 | {% set datepart = 'WW' %} 24 | {% endif %} 25 | {%- set single_quote = "\'" -%} 26 | TRUNC({{date}}, {{single_quote ~ datepart ~ single_quote}}) 27 | {% endmacro %} 28 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/except.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | 18 | {% macro oracle__except() %} 19 | MINUS 20 | {% endmacro %} 21 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/generate_series.sql: -------------------------------------------------------------------------------- 1 | {% macro oracle__generate_series(upper_bound) %} 2 | select to_number(column_value) as generated_number from xmltable('1 to {{upper_bound}}') 3 | {% endmacro %} 4 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/hash.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | 18 | {% macro oracle__hash(field, method='MD5') %} 19 | {%- set single_quote = "\'" -%} 20 | STANDARD_HASH({{field}}, {{single_quote ~ method ~ single_quote }}) 21 | {% endmacro %} 22 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/last_day.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {# Returns last day of the quarter, year or month #} 18 | {% macro oracle__last_day(date, datepart) %} 19 | {{dbt.dateadd('day', '-1', dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date)))}} 20 | {% endmacro %} 21 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/position.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | 18 | {% macro oracle__position(substring_text, string_text) %} 19 | INSTR({{ string_text }}, {{ substring_text }}) 20 | {% endmacro %} 21 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/right.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | {# Returns N rightmost characters from a string #} 18 | {% macro oracle__right(string_text, length_expression) %} 19 | 20 | case when {{ length_expression }} = 0 21 | then '' 22 | else 23 | substr( 24 | {{ string_text }}, 25 | -1 * ({{ length_expression }}) 26 | ) 27 | end 28 | 29 | {%- endmacro -%} 30 | -------------------------------------------------------------------------------- /dbt/include/oracle/macros/utils/timestamps.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | #} 17 | 18 | {% macro oracle__current_timestamp() -%} 19 | SYS_EXTRACT_UTC(current_timestamp) 20 | {%- endmacro %} 21 | 22 | {% macro oracle__snapshot_string_as_time(timestamp) -%} 23 | {%- set result = "TO_TIMESTAMP('"~ timestamp ~ "','yyyy/mm/dd hh24:mi:ss.FF')" -%} 24 | {{ return(result) }} 25 | {%- endmacro %} 26 | 27 | {% macro get_snapshot_get_time_data_type() %} 28 | {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %} 29 | {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time from dual' %} 30 | {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %} 31 | {% set time_data_type = snapshot_time_column_schema[0].dtype %} 32 | {{ return(time_data_type or none) }} 33 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/include/oracle/profile_template.yml: -------------------------------------------------------------------------------- 1 | fixed: 2 | type: oracle 3 | prompts: 4 | protocol: 5 | hint: 'tcp or tcps' 6 | type: string 7 | default: tcps 8 | host: 9 | hint: 'adb.<oci-region>.oraclecloud.com' 10 | default: "{{ env_var('DBT_ORACLE_HOST') }}" 11 | port: 12 | type: int 13 | default: 1522 14 | user: 15 | type: string 16 | default: "{{ env_var('DBT_ORACLE_USER') }}" 17 | password: 18 | hide_input: true 19 | default: "{{ env_var('DBT_ORACLE_PASSWORD') }}" 20 | service: 21 | hint: 'service name in tnsnames.ora' 22 | type: string 23 | default: "{{ env_var('DBT_ORACLE_SERVICE') }}" 24 | schema: 25 | hint: 'database schema in which dbt objects should be created' 26 | type: string 27 | default: "{{ env_var('DBT_ORACLE_SCHEMA') }}" 28 | threads: 29 | hint: '1 or more' 30 | type: int 31 | default: 1 32 | 33 | 34 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/README.md: -------------------------------------------------------------------------------- 1 | # Integration testing with Oracle's Autonomous Database Service (ADBS) 2 | 3 | ## Always Free Autonomous Database 4 | 5 | To test the integration with ADBS, you can use OCI's [Always Free Autonomous Database](https://docs.oracle.com/en-us/iaas/Content/Database/Concepts/adbfreeoverview.htm). The database is provided free of charge. 6 | - Processor: 1 Oracle CPU processor (cannot be scaled) 7 | - Database Storage: 20 GB storage (cannot be scaled) 8 | - Workload Type could be either ATP (Autonomous Transaction Processing) or ADW (Autonomous Data Warehouse) 9 | 10 | The database also provides a read-only Sales History data set. Any user can start querying the tables in this Sales History `sh` schema. Models in this test project refer the `sh` schema. You do not need to load any other dataset. 11 | 12 | ## Setup 13 | 14 | [Install][1] dbt-oracle and setup [profiles.yml](profiles.yml) for ADBS 15 | 16 | ### Verify the installation 17 | 18 | ```bash 19 | dbt --version 20 | ``` 21 | 22 | ```text 23 | Core: 24 | - installed: 1.3.0 25 | - latest: 1.3.0 - Up to date! 26 | ``` 27 | 28 | ### Check database connectivity 29 | 30 | ```bash 31 | dbt debug --profiles-dir ./ 32 | ``` 33 | 34 | ```text 35 | dbt version: 1.3.0 36 | python version: 3.8.13 37 | .. 38 | .. 39 | Connection: 40 | user: 41 | database: 42 | schema: 43 | ... 44 | Connection test: [OK connection ok] 45 | 46 | All checks passed! 47 | 48 | ``` 49 | After this, you can test various dbt features included in this project 50 | 51 | ## Features tested in this project 52 | 53 | | Feature | Command | Corresponding File | 54 | | --------|---------|----- | 55 | | Connection | `dbt debug` | [profiles.yml](profiles.yml) 56 | | Data Sources | `dbt run` or `dbt test` | [schema.yml](./models/schema.yml) 57 | | Seeds | `dbt seed` | [seed.csv](./data/seed.csv) 58 | | View Materialization | `dbt run` | [direct_sales_channel_promo_cost.sql](./models/direct_sales_channel_promo_cost.sql) 59 | | Table Materialization | `dbt run` | [sales_internet_channel.sql](./models/sales_internet_channel.sql) 60 | | Ephemeral Materialization | `dbt run` | [income_levels.sql](./models/income_levels.sql) & [us_seed_customers.sql](./models/us_seed_customers.sql) 61 | | Incremental Materialization | `dbt run` | [us_product_sales_channel_ranking.sql](./models/us_product_sales_channel_ranking.sql) 62 | | Singular Test | `dbt test` | [test_count_employees.sql](./test/test_count_employees.sql) 63 | | Generic Test - Not null | `dbt test` | [schema.yml](./models/schema.yml) 64 | | Generic Test - Unique values | `dbt test` | [schema.yml](./models/schema.yml) 65 | | Generic Test - Accepted values | `dbt test` | [schema.yml](./models/schema.yml) 66 | | Generic Test - Relationships | `dbt test` | [schema.yml](./models/schema.yml) 67 | | Operations | `dbt run-operation` | Check [macros](macros) 68 | | Snapshots | `dbt snapshot` | Check [snapshots](snapshots) 69 | | Analyses | `dbt compile` | [eu_customers.sql](./analysis/eu_customers.sql) 70 | | Exposures | `dbt run` or `dbt test` | [exposures.yml](./models/exposures.yml) 71 | | Generate documentation | `dbt docs generate` | 72 | | Serve project documentation on port 8080 | `dbt docs serve` 73 | 74 | 75 | ## Tests [TODO] 76 | - Metrics - Experimental feature introduced in dbt-core==1.0.0 77 | 78 | [1]: https://docs.getdbt.com/reference/warehouse-profiles/oracle-profile 79 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/analysis/eu_customers.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | with eu_countries as ( 17 | select * from {{ ref('countries') }} 18 | ), internet_sales as ( 19 | select * from {{ ref('sales_internet_channel') }} 20 | ), customer as ( 21 | select * from {{ source('sh_database', 'customers')}} 22 | ) 23 | 24 | select c.cust_first_name, c.cust_last_name, s.time_id, s.prod_id, c.cust_id, s.quantity_sold 25 | from customer c, internet_sales s, eu_countries ct 26 | where c.cust_id = s.cust_id 27 | and c.country_id = ct.country_id 28 | 29 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/data/seed.csv: -------------------------------------------------------------------------------- 1 | id,first_name,last_name,email,gender,ip_address 2 | 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 3 | 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 4 | 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 5 | 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 6 | 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 -------------------------------------------------------------------------------- /dbt_adbs_test_project/dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: dbt_adbs_test_project 2 | config-version: 2 3 | version: 1.0 4 | profile: dbt_test 5 | analysis-paths: ['analysis'] 6 | test-paths: ['test'] 7 | clean-targets: # directories to be removed by `dbt clean` 8 | - "target" 9 | - "dbt_packages" 10 | - "dbt_modules" 11 | - "logs" 12 | 13 | quoting: 14 | database: false 15 | identifier: false 16 | schema: false 17 | 18 | seeds: 19 | dbt_adbs_test_project: 20 | quote_columns: false 21 | kafka_message: 22 | +column_types: 23 | message: CLOB 24 | blob_message: BLOB 25 | seed_with_empty_col: 26 | +column_types: 27 | id: number 28 | first_name: varchar2(16) 29 | last_name: varchar2(16) 30 | email: varchar2(26) 31 | gender: varchar2(16) 32 | age: number 33 | ip_address: varchar2(16) 34 | 35 | snapshots: 36 | dbt_adbs_test_project: 37 | target_schema: "{{ env_var('DBT_ORACLE_USER') }}" 38 | 39 | on-run-start: 40 | - "select 'hook start' from dual" 41 | 42 | on-run-end: 43 | - "select 'hook ended' from dual" 44 | 45 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/macros/create_index.sql: -------------------------------------------------------------------------------- 1 | {%- macro random_digits(n) -%} 2 | {%- for _ in range(n) -%} 3 | {{ range(10) | random }} 4 | {%- endfor -%} 5 | {%- endmacro -%} 6 | 7 | {% macro create_index(idx_prefix, columns) %} 8 | {%- set columns_name = columns | replace(', ', '__') | replace(',', '__') -%} 9 | {%- set random_part = random_digits(16) -%} 10 | {%- set index_name = idx_prefix ~ "__idx_" ~ random_part ~ "_on__" ~ columns_name -%} 11 | {%- set sql -%} 12 | CREATE INDEX "{{ index_name }}" ON {{ this }} ({{ columns }}) 13 | {%- endset -%} 14 | {%- if execute -%} 15 | {{- log("Creating index...", info = true) -}} 16 | {% do run_query(sql) %} 17 | {%- endif -%} 18 | {%- endmacro -%} 19 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/macros/datatypes.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {%- macro oracle__type_string() -%} 17 | VARCHAR2(4000) 18 | {%- endmacro -%} 19 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/macros/demo_multiple_statements.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% macro demo_multiple_statements() -%} 17 | {%- call statement('demo_multiple_statements') -%} 18 | DECLARE 19 | p_id NUMBER(4) := 412; 20 | p_amount NUMBER(7, 2) := 1233.00; 21 | sql_stmt VARCHAR2(200); 22 | BEGIN 23 | EXECUTE IMMEDIATE 'CREATE TABLE product (id NUMBER, amt NUMBER)'; 24 | sql_stmt := 'INSERT into product values (:1, :2)'; 25 | EXECUTE IMMEDIATE sql_stmt USING p_id, p_amount; 26 | EXECUTE IMMEDIATE 'DELETE FROM product WHERE id = :num' USING p_id; 27 | EXECUTE IMMEDIATE 'DROP TABLE product'; 28 | END; 29 | {%- endcall -%} 30 | {% endmacro %} 31 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/macros/execute_statements.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% macro execute_statements(statements) -%} 17 | {%- call statement('execute_statements') -%} 18 | BEGIN 19 | {% for statement in statements %} 20 | execute immediate '{{ statement }}'; 21 | {% endfor %} 22 | END; 23 | {%- endcall -%} 24 | {% endmacro %} 25 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/macros/generate_schema_name.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% macro generate_schema_name(custom_schema_name, node) -%} 17 | 18 | {%- set default_schema = target.schema -%} 19 | {%- if custom_schema_name is none -%} 20 | {{ default_schema }} 21 | {%- else -%} 22 | {{ custom_schema_name | trim }} 23 | {%- endif -%} 24 | 25 | {%- endmacro %} -------------------------------------------------------------------------------- /dbt_adbs_test_project/macros/hash_arguments.sql: -------------------------------------------------------------------------------- 1 | {% macro hash_arguments(args) -%} 2 | ORA_HASH({%- for arg in args -%} 3 | coalesce(cast({{ arg }} as varchar(50) ), '') 4 | {% if not loop.last %} || '|' || {% endif %} 5 | {%- endfor -%}) 6 | {%- endmacro %} -------------------------------------------------------------------------------- /dbt_adbs_test_project/macros/readme.md: -------------------------------------------------------------------------------- 1 | # Macros 2 | 3 | Macros are reusable SQL, which can be invoked using `dbt run-operation` commands 4 | 5 | ## Demo Multiple statements 6 | 7 | This addresses `ORA-00933: SQL command not properly ended` which was raised in https://github.com/techindicium/dbt-oracle/issues/26 8 | 9 | ```sql 10 | {% macro demo_multiple_statements() -%} 11 | {%- call statement('demo_multiple_statements') -%} 12 | DECLARE 13 | p_id NUMBER(4) := 412; 14 | p_amount NUMBER(7, 2) := 1233.00; 15 | sql_stmt VARCHAR2(200); 16 | BEGIN 17 | EXECUTE IMMEDIATE 'CREATE TABLE product (id NUMBER, amt NUMBER)'; 18 | sql_stmt := 'INSERT into product values (:1, :2)'; 19 | EXECUTE IMMEDIATE sql_stmt USING p_id, p_amount; 20 | EXECUTE IMMEDIATE 'DELETE FROM product WHERE id = :num' USING p_id; 21 | EXECUTE IMMEDIATE 'DROP TABLE product'; 22 | END; 23 | {%- endcall -%} 24 | {% endmacro %} 25 | ``` 26 | 27 | Invoke the macro operation using `dbt run-operation` command from the project root directory 28 | 29 | ```bash 30 | dbt --debug run-operation demo_multiple_statements --profiles-dir ./ 31 | ``` 32 | 33 | The following SQL is run 34 | 35 | ```sql 36 | DECLARE 37 | p_id NUMBER(4) := 412; 38 | p_amount NUMBER(7, 2) := 1233.00; 39 | sql_stmt VARCHAR2(200); 40 | BEGIN 41 | EXECUTE IMMEDIATE 'CREATE TABLE product (id NUMBER, amt NUMBER)'; 42 | sql_stmt := 'INSERT into product values (:1, :2)'; 43 | EXECUTE IMMEDIATE sql_stmt USING p_id, p_amount; 44 | EXECUTE IMMEDIATE 'DELETE FROM product WHERE id = :num' USING p_id; 45 | EXECUTE IMMEDIATE 'DROP TABLE product'; 46 | END; 47 | 48 | ``` 49 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/demo/promotion_costs_for_direct_sales_channel.sql: -------------------------------------------------------------------------------- 1 | {{config(materialized='table')}} 2 | WITH direct_sales_promo_cost AS ( 3 | SELECT s.prod_id, 4 | s.quantity_sold, 5 | s.amount_sold, 6 | s.time_id, 7 | c.channel_desc, 8 | p.promo_name, 9 | p.promo_cost 10 | FROM {{ source('sh_database', 'sales') }} s, 11 | {{ source('sh_database', 'promotions') }} p, 12 | {{ source('sh_database', 'channels') }} c 13 | WHERE s.channel_id = 3 14 | AND s.promo_id = p.promo_id 15 | AND s.channel_id = c.channel_id 16 | ) 17 | SELECT * FROM direct_sales_promo_cost -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/demo/promotion_costs_for_direct_sales_channel_incr_insert.sql: -------------------------------------------------------------------------------- 1 | {{config(materialized='incremental')}} 2 | WITH direct_sales_promo_cost AS ( 3 | SELECT s.prod_id, 4 | s.quantity_sold, 5 | s.amount_sold, 6 | s.time_id, 7 | c.channel_desc, 8 | p.promo_name, 9 | p.promo_cost 10 | FROM {{ source('sh_database', 'sales') }} s, 11 | {{ source('sh_database', 'promotions') }} p, 12 | {{ source('sh_database', 'channels') }} c 13 | WHERE s.channel_id = 3 14 | AND s.promo_id = p.promo_id 15 | AND s.channel_id = c.channel_id 16 | {% if is_incremental() %} 17 | AND s.time_id > (SELECT MAX(time_id) FROM {{ this }}) 18 | {% endif %} 19 | ) 20 | SELECT * FROM direct_sales_promo_cost -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/demo/promotion_costs_for_direct_sales_channel_incr_merge.sql: -------------------------------------------------------------------------------- 1 | {{config(materialized='incremental', unique_key='group_id')}} 2 | WITH direct_sales_promo_cost AS ( 3 | SELECT s.prod_id, 4 | s.quantity_sold, 5 | s.amount_sold, 6 | s.time_id, 7 | c.channel_desc, 8 | p.promo_name, 9 | p.promo_cost, 10 | {{ hash_arguments(['s.prod_id', 's.quantity_sold', 's.time_id', 'p.promo_name']) }} AS group_id 11 | FROM {{ source('sh_database', 'sales') }} s, 12 | {{ source('sh_database', 'promotions') }} p, 13 | {{ source('sh_database', 'channels') }} c 14 | WHERE s.channel_id = 3 15 | AND s.promo_id = p.promo_id 16 | AND s.channel_id = c.channel_id 17 | {% if is_incremental() %} 18 | AND s.time_id > (SELECT MAX(time_id) FROM {{ this }}) 19 | {% endif %} 20 | ) 21 | SELECT * FROM direct_sales_promo_cost -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/demo/promotion_costs_for_direct_sales_channel_incr_merge_unique_keys.sql: -------------------------------------------------------------------------------- 1 | {{config(materialized='incremental', 2 | on_schema_change='append_new_columns', 3 | unique_key=['prod_id', 'quantity_sold', 'time_id', 'promo_name'])}} 4 | WITH direct_sales_promo_cost AS ( 5 | SELECT s.prod_id, 6 | s.quantity_sold, 7 | s.amount_sold, 8 | s.time_id, 9 | s.cust_id, 10 | c.channel_desc, 11 | p.promo_name, 12 | p.promo_cost 13 | FROM {{ source('sh_database', 'sales') }} s, 14 | {{ source('sh_database', 'promotions') }} p, 15 | {{ source('sh_database', 'channels') }} c 16 | WHERE s.channel_id = 3 17 | AND s.promo_id = p.promo_id 18 | AND s.channel_id = c.channel_id 19 | {% if is_incremental() %} 20 | AND s.time_id > (SELECT MAX(time_id) FROM {{ this }}) 21 | {% endif %} 22 | ) 23 | SELECT * FROM direct_sales_promo_cost 24 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/direct_sales_channel_promo_cost.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{config(materialized='view')}} 17 | with direct_sales_promo_cost as( 18 | SELECT s.prod_id, s.quantity_sold, c.channel_desc, s.amount_sold, p.promo_name, p.promo_cost 19 | FROM {{ source('sh_database', 'sales') }} s, {{ source('sh_database', 'promotions') }} p, {{ source('sh_database', 'channels') }} c 20 | WHERE s.channel_id = 3 21 | AND s.promo_id = p.promo_id 22 | AND s.channel_id = c.channel_id 23 | ) 24 | select * from direct_sales_promo_cost 25 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/eu/countries.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{config(materialized='table', 17 | schema=env_var('DBT_ORACLE_CUSTOM_SCHEMA')) }} 18 | SELECT * FROM {{ source('sh_database', 'countries')}} 19 | where country_iso_code in ('AT', 'BE', 'BG', 'DK', 'CZ', 'DE', 'IT', 20 | 'FI', 'FR', 'GR', 'NL', 'IE', 'HU', 'ES', 'SE', 21 | 'GE', 'IS', 'NO', 'CH', 'GB', 'VA') 22 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/eu/eu_direct_sales_channels_promo_costs.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{config(materialized='table')}} 17 | with eu_direct_sales_promo_cost as( 18 | SELECT s.prod_id, s.quantity_sold, ch.channel_desc, ch.channel_class, s.amount_sold, p.promo_name, p.promo_cost, cu.country_id 19 | FROM {{ source('sh_database', 'sales') }} s, {{ source('sh_database', 'promotions') }} p, 20 | {{ source('sh_database', 'channels') }} ch, {{ ref('countries') }} ct, {{ source('sh_database', 'customers')}} cu 21 | WHERE s.channel_id in (3, 9) 22 | AND s.promo_id = p.promo_id 23 | AND s.channel_id = ch.channel_id 24 | AND s.cust_id = cu.cust_id 25 | AND cu.country_id = ct.country_id 26 | ) 27 | select * from eu_direct_sales_promo_cost 28 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/exposures.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | exposures: 4 | - name: internet_sales_data_science_team 5 | type: ml 6 | description: > 7 | Data Science jobs using Internet Sales dataset 8 | depends_on: 9 | - ref('internet_sales_channel_customers') 10 | - source('sh_database', 'customers') 11 | owner: 12 | email: abc@xyz.com 13 | 14 | - name: product_sales_ranking_in_us 15 | type: dashboard 16 | description: > 17 | A monthly updating dashboard which shows rank-ordering of product sales in US market. 18 | depends_on: 19 | - ref('us_product_sales_channel_ranking') 20 | - ref('us_seed_customers') 21 | owner: 22 | email: abc@xyz.com 23 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/income_levels.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ 17 | config(materialized='ephemeral') 18 | }} 19 | select 'G: 130,000 - 149,999' as income_level from dual 20 | union all 21 | select 'K: 250,000 - 299,999' as income_level from dual 22 | union all 23 | select 'H: 150,000 - 169,999' as income_level from dual 24 | union all 25 | select 'J: 190,000 - 249,999' as income_level from dual 26 | union all 27 | select 'A: Below 30,000' as income_level from dual 28 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/internet_sales_channel_customers.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{config(materialized='table', parallel=4, table_compression_clause='COLUMN STORE COMPRESS FOR QUERY')}} 17 | select c.cust_id, c.cust_first_name, c.cust_last_name, t.country_iso_code, t.country_name, t.country_region 18 | from {{ ref('sales_internet_channel') }} s, {{ source('sh_database', 'countries') }} t, {{ source('sh_database', 'customers') }} c 19 | WHERE s.cust_id = c.cust_id 20 | AND c.country_id = t.country_id 21 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/kafka.sql: -------------------------------------------------------------------------------- 1 | select * from {{ ref('kafka_message') }} 2 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/people.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{config(materialized='table')}} 17 | select * from {{ ref('seed') }} 18 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/promotion_costs.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ config(materialized='table', grants = {'select': ['DBT_TEST_USER_1']})}} 17 | with all_promo_costs as( 18 | select * from {{ source('sh_database', 'promotions') }} 19 | ) 20 | select * from all_promo_costs 21 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/properties.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: sales_cost_incremental 4 | config: 5 | post-hook: "SELECT COUNT(*) FROM sales_cost_incremental" 6 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/sales_cost.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='table')}} 2 | with sales_cost_cte as( 3 | select prod_id, 4 | cast(time_id as TIMESTAMP) as cost_timestamp, 5 | promo_id, 6 | channel_id, 7 | unit_cost, 8 | unit_price 9 | from {{ source('sh_database', 'costs') }} 10 | ) 11 | select * from sales_cost_cte -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/sales_cost_incremental.py: -------------------------------------------------------------------------------- 1 | def model(dbt, session): 2 | # Must be either table or incremental 3 | dbt.config(materialized="incremental", incremental_strategy="merge") 4 | # oml.DataFrame representing a datasource 5 | sales_cost_df = dbt.ref("sales_cost") 6 | 7 | if dbt.is_incremental: 8 | cr = session.cursor() 9 | result = cr.execute(f"select max(cost_timestamp) from {dbt.this.identifier}") 10 | max_timestamp = result.fetchone()[0] 11 | sales_cost_df = sales_cost_df[sales_cost_df["COST_TIMESTAMP"] > max_timestamp] 12 | 13 | return sales_cost_df 14 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/sales_internet_channel.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ config(materialized='table')}} 17 | with sales_internet as( 18 | select * from {{ source('sh_database', 'sales') }} 19 | where channel_id = 4 20 | ) 21 | select * from sales_internet 22 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/sales_internet_mv.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ config(materialized='materialized_view', persist_docs={"relation": true, "columns": true}) }} 17 | select * from {{ source('sh_database', 'sales') }} 18 | where channel_id = 5 19 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/sales_py.py: -------------------------------------------------------------------------------- 1 | def model(dbt, session): 2 | dbt.config(materialized="table") 3 | dbt.config(async_flag=True) 4 | dbt.config(timeout=1800) 5 | # oml.core.DataFrame referencing a dbt-sql model 6 | sales = session.sync(query="SELECT * FROM SH.SALES") 7 | return sales 8 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: sh_database 5 | schema: sh 6 | quoting: 7 | database: false 8 | schema: false 9 | identifier: false 10 | tables: 11 | - name: customers 12 | - name: sales 13 | freshness: # default freshness 14 | warn_after: { count: 12, period: hour } 15 | error_after: { count: 24, period: hour } 16 | loaded_at_field: time_id 17 | - name: products 18 | - name: times 19 | - name: channels 20 | - name: countries 21 | - name: promotions 22 | - name: costs 23 | 24 | models: 25 | - name: sales_internet_mv 26 | description: Test comment for Materialized View 27 | 28 | - name: kafka 29 | description: kafka_description 30 | config: 31 | materialized: incremental 32 | incremental_strategy: append 33 | on_schema_change: fail 34 | full_refresh: false 35 | contract: 36 | enforced: true 37 | constraints: 38 | - type: not_null 39 | columns: [message] 40 | columns: 41 | - name: message 42 | description: Kafka message 43 | data_type: CLOB 44 | constraints: 45 | - type: not_null 46 | - name: blob_message 47 | description: Kafka message 48 | data_type: BLOB 49 | constraints: 50 | - type: not_null 51 | 52 | 53 | - name: people 54 | columns: 55 | - name: id 56 | tests: 57 | - dbt_constraints.primary_key 58 | - not_null 59 | - unique 60 | - name: gender 61 | tests: 62 | - accepted_values: 63 | values: ['Male', 'Female'] 64 | - name: countries 65 | columns: 66 | - name: country_id 67 | tests: 68 | - dbt_constraints.primary_key 69 | - not_null 70 | - unique 71 | - name: eu_direct_sales_channels_promo_costs 72 | columns: 73 | - name: country_id 74 | tests: 75 | - relationships: 76 | to: ref('countries') 77 | field: country_id 78 | - dbt_constraints.foreign_key: 79 | pk_table_name: ref('countries') 80 | pk_column_name: country_id 81 | 82 | 83 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/test_py_ref.py: -------------------------------------------------------------------------------- 1 | def model(dbt, session): 2 | # Must be either table or incremental (view is not currently supported) 3 | dbt.config(materialized="table") 4 | dbt.config(async_flag=True) 5 | dbt.config(timeout=900) # In seconds 6 | dbt.config(service="HIGH") # LOW, MEDIUM, HIGH 7 | # oml.core.DataFrame representing a datasource 8 | s_df = dbt.ref("sales_cost") 9 | return s_df 10 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/test_py_source.py: -------------------------------------------------------------------------------- 1 | def model(dbt, session): 2 | # Must be either table or incremental (view is not currently supported) 3 | dbt.config(materialized="table") 4 | # dbt.config(conda_env_name="dbt_py_env") 5 | # oml.core.DataFrame representing a datasource 6 | s_df = dbt.source("sh_database", "channels") 7 | return s_df 8 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/union_customer_sales.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ config(materialized='table')}} 17 | {{ 18 | dbt_utils.union_relations( 19 | relations=[source('sh_database', 'sales'), 20 | source('sh_database', 'customers')], 21 | source_column_name='dbt_source_relation') 22 | }} 23 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/us_product_delete_insert.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2024, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ 17 | config( 18 | materialized='incremental', 19 | unique_key='group_id', 20 | incremental_predicates=["DBT_INTERNAL_DEST.calendar_month_desc > TO_CHAR(sysdate, ''yyyy/mm/dd'')"], 21 | incremental_strategy='delete+insert', 22 | parallel=4, 23 | insert_mode="append", 24 | partition_config={"clause": "PARTITION BY HASH(PROD_NAME) PARTITIONS 4"}, 25 | table_compression_clause='COLUMN STORE COMPRESS FOR QUERY LOW') 26 | }} 27 | 28 | SELECT prod_name, channel_desc, calendar_month_desc, 29 | {{ snapshot_hash_arguments(['prod_name', 'channel_desc', 'calendar_month_desc']) }} AS group_id, 30 | TO_CHAR(SUM(amount_sold), '9,999,999,999') SALES$, 31 | RANK() OVER (ORDER BY SUM(amount_sold)) AS default_rank, 32 | RANK() OVER (ORDER BY SUM(amount_sold) DESC NULLS LAST) AS custom_rank 33 | FROM {{ source('sh_database', 'sales') }}, {{ source('sh_database', 'products') }}, {{ source('sh_database', 'customers') }}, 34 | {{ source('sh_database', 'times') }}, {{ source('sh_database', 'channels') }}, {{ source('sh_database', 'countries') }} 35 | WHERE sales.prod_id=products.prod_id AND sales.cust_id=customers.cust_id 36 | AND customers.country_id = countries.country_id AND sales.time_id=times.time_id 37 | AND sales.channel_id=channels.channel_id 38 | AND country_iso_code='US' 39 | 40 | {% if is_incremental() %} 41 | 42 | AND times.calendar_month_desc > (SELECT MAX(calendar_month_desc) FROM {{ this }}) 43 | 44 | {% endif %} 45 | 46 | GROUP BY prod_name, channel_desc, calendar_month_desc 47 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/us_product_sales_channel_ranking.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2024, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ 17 | config( 18 | materialized='incremental', 19 | unique_key='group_id', 20 | parallel=4, 21 | partition_config={"clause": "PARTITION BY HASH(PROD_NAME) PARTITIONS 4"}, 22 | table_compression_clause='COLUMN STORE COMPRESS FOR QUERY LOW') 23 | }} 24 | 25 | SELECT prod_name, channel_desc, calendar_month_desc, 26 | {{ snapshot_hash_arguments(['prod_name', 'channel_desc', 'calendar_month_desc']) }} AS group_id, 27 | TO_CHAR(SUM(amount_sold), '9,999,999,999') SALES$, 28 | RANK() OVER (ORDER BY SUM(amount_sold)) AS default_rank, 29 | RANK() OVER (ORDER BY SUM(amount_sold) DESC NULLS LAST) AS custom_rank 30 | FROM {{ source('sh_database', 'sales') }}, {{ source('sh_database', 'products') }}, {{ source('sh_database', 'customers') }}, 31 | {{ source('sh_database', 'times') }}, {{ source('sh_database', 'channels') }}, {{ source('sh_database', 'countries') }} 32 | WHERE sales.prod_id=products.prod_id AND sales.cust_id=customers.cust_id 33 | AND customers.country_id = countries.country_id AND sales.time_id=times.time_id 34 | AND sales.channel_id=channels.channel_id 35 | AND country_iso_code='US' 36 | 37 | {% if is_incremental() %} 38 | 39 | AND times.calendar_month_desc > (SELECT MAX(calendar_month_desc) FROM {{ this }}) 40 | 41 | {% endif %} 42 | 43 | GROUP BY prod_name, channel_desc, calendar_month_desc 44 | 45 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/us_product_sales_channel_ranking_append.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {{ 17 | config( 18 | materialized='incremental', 19 | incremental_strategy='append', 20 | parallel=4, 21 | table_compression_clause='COLUMN STORE COMPRESS FOR QUERY LOW') 22 | }} 23 | 24 | SELECT prod_name, channel_desc, calendar_month_desc, 25 | {{ snapshot_hash_arguments(['prod_name', 'channel_desc', 'calendar_month_desc']) }} AS group_id, 26 | TO_CHAR(SUM(amount_sold), '9,999,999,999') SALES$, 27 | RANK() OVER (ORDER BY SUM(amount_sold)) AS default_rank, 28 | RANK() OVER (ORDER BY SUM(amount_sold) DESC NULLS LAST) AS custom_rank 29 | FROM {{ source('sh_database', 'sales') }}, {{ source('sh_database', 'products') }}, {{ source('sh_database', 'customers') }}, 30 | {{ source('sh_database', 'times') }}, {{ source('sh_database', 'channels') }}, {{ source('sh_database', 'countries') }} 31 | WHERE sales.prod_id=products.prod_id AND sales.cust_id=customers.cust_id 32 | AND customers.country_id = countries.country_id AND sales.time_id=times.time_id 33 | AND sales.channel_id=channels.channel_id 34 | AND country_iso_code='US' 35 | 36 | {% if is_incremental() %} 37 | 38 | AND times.calendar_month_desc > (SELECT MAX(calendar_month_desc) FROM {{ this }}) 39 | 40 | {% endif %} 41 | 42 | GROUP BY prod_name, channel_desc, calendar_month_desc 43 | 44 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/models/us_seed_customers.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | with 17 | country AS (select * from {{ source('sh_database', 'countries')}}), 18 | customer AS (select * from {{ source('sh_database', 'customers')}}) 19 | 20 | select customer.cust_first_name, 21 | customer.cust_last_name, 22 | customer.cust_gender, 23 | customer.cust_marital_status, 24 | customer.cust_street_address, 25 | customer.cust_email, 26 | customer.cust_credit_limit 27 | from customer, country, {{ ref('income_levels') }} -- refer ephemeral model 28 | where customer.cust_income_level = {{ ref('income_levels') }}.income_level 29 | and country.country_iso_code = 'US' 30 | and customer.country_id = country.country_id 31 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/package-lock.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 0.8.6 4 | - package: Snowflake-Labs/dbt_constraints 5 | version: 0.4.2 6 | sha1_hash: 7664cb2e33183f39e86a72079d63607e43a50ad8 7 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 0.8.6 4 | - package: Snowflake-Labs/dbt_constraints 5 | version: [">=0.4.0", "<0.5.0"] 6 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_test: 2 | target: "{{ env_var('DBT_TARGET', 'dev') }}" 3 | outputs: 4 | dev: 5 | type: oracle 6 | user: "{{ env_var('DBT_ORACLE_USER') }}" 7 | pass: "{{ env_var('DBT_ORACLE_PASSWORD') }}" 8 | protocol: "tcps" 9 | host: "{{ env_var('DBT_ORACLE_HOST') }}" 10 | port: 1522 11 | service: "{{ env_var('DBT_ORACLE_SERVICE') }}" 12 | database: "{{ env_var('DBT_ORACLE_DATABASE') }}" 13 | schema: "{{ env_var('DBT_ORACLE_SCHEMA') }}" 14 | oml_cloud_service_url: "{{ env_var('DBT_ORACLE_OML_CLOUD_SERVICE_URL')}}" 15 | session_info: 16 | action: "dbt run" 17 | client_identifier: "dbt-mac-abhisoms" 18 | client_info: "dbt Python3.9 thin driver" 19 | module: "dbt-module-1.5.2" 20 | retry_count: 1 21 | retry_delay: 5 22 | threads: 1 23 | test: 24 | type: oracle 25 | user: "{{ env_var('DBT_ORACLE_USER') }}" 26 | pass: "{{ env_var('DBT_ORACLE_PASSWORD') }}" 27 | database: "{{ env_var('DBT_ORACLE_DATABASE') }}" 28 | schema: "{{ env_var('DBT_ORACLE_SCHEMA') }}" 29 | connection_string: "{{ env_var('DBT_ORACLE_CONNECT_STRING') }}" 30 | shardingkey: 31 | - skey 32 | supershardingkey: 33 | - sskey 34 | cclass: CONNECTIVITY_CLASS 35 | purity: self 36 | threads: 4 37 | prod: 38 | type: oracle 39 | user: "{{ env_var('DBT_ORACLE_USER') }}" 40 | pass: "{{ env_var('DBT_ORACLE_PASSWORD') }}" 41 | database: "{{ env_var('DBT_ORACLE_DATABASE') }}" 42 | tns_name: "{{ env_var('DBT_ORACLE_TNS_NAME') }}" 43 | schema: "{{ env_var('DBT_ORACLE_SCHEMA') }}" 44 | session_info: 45 | action: "dbt run" 46 | client_identifier: "dbt-mac-abhisoms" 47 | client_info: "dbt Python3.9 thin driver" 48 | module: "dbt-module-1.5.2" 49 | shardingkey: 50 | - skey 51 | supershardingkey: 52 | - sskey 53 | cclass: CONNECTIVITY_CLASS 54 | purity: self 55 | threads: 4 56 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/seeds/kafka_message.csv: -------------------------------------------------------------------------------- 1 | message,blob_message 2 | Jack-Hunter,1234 3 | Jack-Hunter,123124 4 | Jack-HunterJackHunter,123123 5 | Jack-Hunter,123143 6 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/seeds/seed.csv: -------------------------------------------------------------------------------- 1 | id,first_name,last_name,email,gender,ip_address 2 | 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 3 | 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 4 | 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 5 | 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 6 | 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 -------------------------------------------------------------------------------- /dbt_adbs_test_project/seeds/seed_with_empty_col.csv: -------------------------------------------------------------------------------- 1 | id,first_name,last_name,email,gender,age,ip_address 2 | 1,Jack,Hunter,jhunter0@pbs.org,,,59.80.20.168 3 | 2,Kathryn,Walker,kwalker1@ezinearticles.com,,,194.121.179.35 4 | 3,Gerald,Ryan,gryan2@com.com,,,11.3.212.243 5 | 4,Bonnie,Spencer,bspencer3@ameblo.jp,,,216.32.196.175 6 | 5,Harold,Taylor,htaylor4@people.com.cn,,,253.10.246.136 7 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/snapshots/promotion_costs.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | {% snapshot promotion_costs_snapshot %} 17 | {{ config( 18 | strategy='check', 19 | unique_key='promo_id', 20 | check_cols='all', 21 | hard_deletes='invalidate', 22 | snapshot_meta_column_names={ 23 | "dbt_valid_from": "promo_valid_from", 24 | "dbt_valid_to": "promo_valid_to", 25 | "dbt_scd_id": "dbt_scd_id" 26 | } 27 | ) 28 | }} 29 | select * from {{ ref('promotion_costs') }} 30 | {% endsnapshot %} 31 | -------------------------------------------------------------------------------- /dbt_adbs_test_project/test/test_count_employees.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #} 16 | SELECT * FROM ( 17 | select count(*) as count from {{ ref('people') }} 18 | ) c WHERE c.count != 5 19 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['setuptools >= 40.8.0', 'wheel'] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning 4 | ignore:unclosed file .*:ResourceWarning 5 | ignore::RuntimeWarning 6 | testpaths = 7 | tests/functional 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | dbt-common>=1.1.0,<2.0 2 | dbt-adapters>=1.2.1,<2.0 3 | dbt-core>=1.9.1,<2.0 4 | oracledb==3.1.1 5 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | pip 2 | wheel 3 | build 4 | setuptools>=40.6.0 5 | tox 6 | coverage 7 | twine 8 | pytest 9 | dbt-tests-adapter~=1.11,<1.12 10 | -------------------------------------------------------------------------------- /sbom_generation.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. 2 | 3 | # This OCI DevOps build specification file [1] generates a Software Bill of Materials (SBOM) of the repository. 4 | # The file is needed to run checks for third-party vulnerabilities and business approval according to Oracle’s GitHub policies. 5 | # [1] https://docs.oracle.com/en-us/iaas/Content/devops/using/build_specs.htm 6 | 7 | version: 0.1 8 | component: build 9 | timeoutInSeconds: 1000 10 | shell: bash 11 | env: 12 | variables: 13 | PYTHON_CMD: "python3" 14 | CDXGEN_DEBUG_MODE: "debug" 15 | steps: 16 | - type: Command 17 | name: "Download the version 10.10.0 of cdxgen globally" 18 | command: | 19 | npm install -g @cyclonedx/cdxgen@10.10.0 20 | - type: Command 21 | name: "Workaround to let cdxgen run on nodejs 16" 22 | command: | 23 | # cdxgen relies on a fourth-party dependency that cannot be executed in a Node.js environment running version 16 24 | # (as installed on the build runner instance) 25 | # This is a workaround to ensure cdxgen functions correctly, even in an older Node.js environment. 26 | cd /node/node-v16.14.2-linux-x64/lib/node_modules/@cyclonedx/cdxgen && \ 27 | npm install cheerio@v1.0.0-rc.12 28 | - type: Command 29 | name: "Generate SBOM for Python " 30 | command: | 31 | # Search the test or dev requirements files, so that test and dev py packages can be excluded in the generated SBOM 32 | files=$(find . -type f -regex ".*\(test.*requirements\|requirements.*test\|dev.*requirements\|requirements.*dev\).*\.txt") && \ 33 | if [ -n "$files" ]; then \ 34 | cdxgen -t python -o artifactSBOM.json --spec-version 1.4 \ 35 | --exclude "*{requirements,dev,test}*{requirements,dev,test}*.txt" --project-name "$(basename $OCI_PRIMARY_SOURCE_URL)" --no-recurse 36 | else \ 37 | cdxgen -t python -o artifactSBOM.json --spec-version 1.4 --project-name "$(basename $OCI_PRIMARY_SOURCE_URL)" --no-recurse 38 | fi \ 39 | outputArtifacts: 40 | - name: artifactSBOM 41 | type: BINARY 42 | location: ${OCI_PRIMARY_SOURCE_DIR}/artifactSBOM.json 43 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = dbt-oracle 3 | version = 1.9.2 4 | description = dbt (data build tool) adapter for Oracle Autonomous Database 5 | long_description = file: README.md 6 | long_description_content_type = text/markdown 7 | keywords = Oracle dbt 8 | author = Oracle 9 | license = Apache Software License 2.0 10 | classifiers = 11 | Development Status :: 5 - Production/Stable 12 | Intended Audience :: Developers 13 | License :: OSI Approved :: Apache Software License 14 | Programming Language :: Python :: 3 15 | Programming Language :: Python :: 3.9 16 | Programming Language :: Python :: 3.10 17 | Programming Language :: Python :: 3.11 18 | Programming Language :: Python :: 3.12 19 | 20 | # Map or URL names to links 21 | # Github, PyPI and documentations urls should be added below 22 | project_urls = 23 | Documentation = https://docs.getdbt.com/reference/warehouse-profiles/oracle-profile 24 | Source = https://github.com/oracle/dbt-oracle 25 | Bug Tracker = https://github.com/oracle/dbt-oracle/issues 26 | CI = https://github.com/oracle/dbt-oracle/actions 27 | Release Notes = https://github.com/oracle/dbt-oracle/releases 28 | 29 | [options] 30 | python_requires = >=3.9 31 | zip_safe = False 32 | packages = find_namespace: 33 | include_package_data = True 34 | install_requires = 35 | dbt-common>=1.1.0,<2.0 36 | dbt-adapters>=1.2.1,<2.0 37 | dbt-core~=1.9,<1.10 38 | oracledb==3.1.1 39 | test_suite=tests 40 | test_requires = 41 | dbt-tests-adapter~=1.10,<1.11 42 | pytest 43 | scripts = 44 | bin/create-pem-from-p12 45 | 46 | [options.packages.find] 47 | include = 48 | dbt 49 | dbt.* 50 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | """The setup script.""" 19 | import sys 20 | from setuptools import setup 21 | 22 | try: 23 | from setuptools import find_namespace_packages 24 | except ImportError: 25 | # the user has a downlevel version of setuptools. 26 | print("Error: dbt requires setuptools v40.1.0 or higher.") 27 | print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again") 28 | sys.exit(1) 29 | 30 | 31 | # lockstep with dbt-core which requires Python > 3.8 32 | if sys.version_info < (3, 9): 33 | print("Error: dbt-oracle does not support this version of Python.") 34 | print("Please upgrade to Python 3.9 or higher.") 35 | sys.exit(1) 36 | 37 | 38 | with open('README.md') as readme_file: 39 | readme = readme_file.read() 40 | 41 | 42 | requirements = [ 43 | "dbt-common>=1.1.0,<2.0", 44 | "dbt-adapters>=1.2.1,<2.0", 45 | "dbt-core~=1.9,<1.10", 46 | "oracledb==3.1.1" 47 | ] 48 | 49 | test_requirements = [ 50 | "dbt-tests-adapter~=1.10,<1.11", 51 | "pytest" 52 | ] 53 | 54 | project_urls = { 55 | 'Documentation': 'https://docs.getdbt.com/reference/warehouse-profiles/oracle-profile', 56 | 'Source': 'https://github.com/oracle/dbt-oracle', 57 | 'Bug Tracker': 'https://github.com/oracle/dbt-oracle/issues', 58 | 'CI': 'https://github.com/oracle/dbt-oracle/actions', 59 | "Release Notes": "https://github.com/oracle/dbt-oracle/releases" 60 | } 61 | 62 | url = 'https://github.com/oracle/dbt-oracle' 63 | 64 | VERSION = '1.9.2' 65 | setup( 66 | author="Oracle", 67 | python_requires='>=3.9', 68 | classifiers=[ 69 | 'Development Status :: 5 - Production/Stable', 70 | 'Intended Audience :: Developers', 71 | 'License :: OSI Approved :: Apache Software License', 72 | 'Natural Language :: English', 73 | 'Programming Language :: Python :: 3', 74 | 'Programming Language :: Python :: 3.9', 75 | 'Programming Language :: Python :: 3.10', 76 | 'Programming Language :: Python :: 3.11', 77 | 'Programming Language :: Python :: 3.12' 78 | ], 79 | description="dbt (data build tool) adapter for Oracle Autonomous Database", 80 | install_requires=requirements, 81 | license="Apache Software License 2.0", 82 | long_description=readme, 83 | long_description_content_type='text/markdown', 84 | include_package_data=True, 85 | keywords='Oracle dbt', 86 | name='dbt-oracle', 87 | packages=find_namespace_packages(include=["dbt", "dbt.*"]), 88 | test_suite='tests', 89 | tests_require=test_requirements, 90 | scripts=['bin/create-pem-from-p12'], 91 | url=url, 92 | project_urls=project_urls, 93 | version=VERSION, 94 | zip_safe=False 95 | ) 96 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | # Table of contents 4 | 5 | - [Environment Variables](#environment-variables) 6 | - [Adapter Plugin Tests](#adapter-plugin-tests) 7 | - [Testing for different Python versions](#different-python-versions) 8 | 9 | 10 | ## Environment variables <a name='environment-variables'></a> 11 | 12 | The following environment variables should be set to test `dbt-oracle` 13 | 14 | ```bash 15 | # cx_oracle needs lib_dir parameter pointing to the folder 16 | # containing the libraries from an unzipped Instant Client Basic or Basic Light package. 17 | # If lib_dir is not passed client libraries are looked for in the Operating system search path 18 | # or set in the following environment variables. 19 | DYLD_LIBRARY_PATH # For MacOS 20 | LD_LIBRARY_PATH # For Linux 21 | 22 | # For ADB, cx_oracle will need the path to the folder 23 | # containing client wallet, sqlnet.ora and tnsnames.ora 24 | TNS_ADMIN 25 | 26 | # Database connection config - dbt specific variables 27 | DBT_ORACLE_USER 28 | DBT_ORACLE_HOST 29 | DBT_ORACLE_PORT 30 | DBT_ORACLE_PROTOCOL 31 | DBT_ORACLE_SERVICE 32 | DBT_ORACLE_PASSWORD 33 | DBT_ORACLE_DATABASE 34 | DBT_ORACLE_SCHEMA 35 | ``` 36 | 37 | ## Adapter Plugin Tests <a name='adapter-plugin-tests'></a> 38 | 39 | dbt-labs has developed a package [dbt-tests-adapter](https://pypi.org/project/dbt-tests-adapter/) which defines a test suite for adapter plugins 40 | 41 | ### Setup 42 | 43 | Clone the project repository into a local directory. 44 | 45 | ```bash 46 | git clone git@github.com:oracle/dbt-oracle.git 47 | ``` 48 | 49 | Create a python3.7+ virtual environment 50 | ```bash 51 | cd dbt-oracle 52 | python3.8 -m venv .venv 53 | source .venv/bin/activate 54 | ``` 55 | Install `dbt-oracle` project in development mode 56 | ```bash 57 | pip install -e . 58 | ``` 59 | 60 | Install `dbt-tests-adapter` and `pytest` in your project's virtual environment 61 | 62 | ```bash 63 | pip install dbt-tests-adapter pytest 64 | ``` 65 | 66 | To run the test suite, just type the `pytest` command from the cloned `dbt-oracle` project directory 67 | 68 | ```pytest 69 | pytest 70 | ``` 71 | 72 | ```bash 73 | Run pytest 74 | ============================= test session starts ============================== 75 | platform linux -- Python 3.9.13, pytest-7.1.2, pluggy-1.0.0 76 | rootdir: /home/runner/work/dbt-oracle/dbt-oracle, configfile: pytest.ini, testpaths: tests/functional 77 | collected 24 items 78 | tests/functional/adapter/test_basic.py .......... [ 41%] 79 | tests/functional/adapter/test_config.py .... [ 58%] 80 | tests/functional/adapter/test_incremental_unique_id.py .......... [100%] 81 | ============================= 24 passed in 55.85s ============================== 82 | ``` 83 | 84 | ## Testing for different Python versions <a name='different-python-versions'></a> 85 | 86 | ### Python 3.7, 3.8, 3.9 and 3.10 87 | 88 | `tox` is a command line tool to check if the package installs correctly for different python versions. It also runs all the tests in each of the environments. Check the configuration file [tox.ini](../tox.ini) for details. 89 | 90 | To run tox, from command line type 91 | ```bash 92 | tox 93 | ``` 94 | If all tests succeed for all python environments, you will see the below output. 95 | 96 | ```text 97 | py37: commands succeeded 98 | py38: commands succeeded 99 | py39: commands succeeded 100 | py310: commands succeeded 101 | congratulations :) 102 | 103 | ``` 104 | For each environment, you can also see the test runtime as summary 105 | ```text 106 | py37 - 24 passed in 771.74s (0:12:51) 107 | py38 - 24 passed in 746.42s (0:12:26) 108 | py39 - 24 passed in 755.18s (0:12:35) 109 | py310 - 24 passed in 764.96s (0:12:44) 110 | ``` 111 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import pytest 19 | import os 20 | 21 | # Import the functional fixtures as a plugin 22 | # Note: fixtures with session scope need to be local 23 | 24 | pytest_plugins = ["dbt.tests.fixtures.project"] 25 | 26 | 27 | # The profile dictionary, used to write out profiles.yml 28 | @pytest.fixture(scope="class") 29 | def dbt_profile_target(): 30 | return { 31 | 'type': 'oracle', 32 | 'threads': 4, 33 | 'user': os.getenv('DBT_ORACLE_USER'), 34 | 'pass': os.getenv('DBT_ORACLE_PASSWORD'), 35 | 'host': os.getenv('DBT_ORACLE_HOST'), 36 | 'schema': os.getenv('DBT_ORACLE_SCHEMA'), 37 | 'database': os.getenv('DBT_ORACLE_DATABASE'), 38 | 'service': os.getenv('DBT_ORACLE_SERVICE'), 39 | 'protocol': os.getenv('DBT_ORACLE_PROTOCOL'), 40 | 'port': os.getenv('DBT_ORACLE_PORT') 41 | } 42 | 43 | 44 | @pytest.fixture(scope="class") 45 | def unique_schema(request, prefix) -> str: 46 | return os.getenv('DBT_ORACLE_SCHEMA') 47 | -------------------------------------------------------------------------------- /tests/functional/adapter/constraints/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /tests/functional/adapter/incremental_materialization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/dbt-oracle/456ceb9d19161508df67056c4187332e079cde18/tests/functional/adapter/incremental_materialization/__init__.py -------------------------------------------------------------------------------- /tests/functional/adapter/incremental_materialization/quotes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/dbt-oracle/456ceb9d19161508df67056c4187332e079cde18/tests/functional/adapter/incremental_materialization/quotes/__init__.py -------------------------------------------------------------------------------- /tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import datetime 19 | from pathlib import Path 20 | 21 | import pytest 22 | from dbt.tests.util import run_dbt 23 | 24 | # seeds/my_seed.csv 25 | seed_csv = """ 26 | _user_id,user_name,birth_date,income,last_login_date,desc 27 | 1,Easton,1981-05-20,40000,2022-04-25T08:57:59,login 28 | 2,Lillian,1978-09-03,54000,2022-04-25T08:58:59,login 29 | 3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59,login 30 | 4,Nolan,1976-05-06,900000,2022-04-25T09:58:59,login 31 | """.lstrip() 32 | 33 | # models/my_incr_model.sql 34 | my_incr_model_sql = """ 35 | {{config(materialized='incremental')}} 36 | SELECT * FROM {{ ref('seed') }} 37 | {% if is_incremental() %} 38 | WHERE last_login_date > (SELECT max(last_login_date) FROM {{ this }}) 39 | {% endif %} 40 | 41 | """ 42 | 43 | # seeds/add_new_rows.sql 44 | seeds__add_new_rows_sql = """ 45 | -- insert two new rows, both of which should be in incremental model 46 | INSERT ALL 47 | INTO {schema}.seed ("_user_id", user_name, birth_date, income, last_login_date, "desc") VALUES 48 | (2,'Lillian Sr.', TO_DATE('1982-02-03', 'YYYY-MM-DD'), 200000, TO_DATE('2022-05-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') 49 | INTO {schema}.seed ("_user_id", user_name, birth_date, income, last_login_date, "desc") VALUES 50 | (5,'John Doe',TO_DATE('1992-10-01', 'YYYY-MM-DD'), 300000, TO_DATE('2022-06-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') 51 | SELECT * FROM dual 52 | """ 53 | 54 | 55 | class TestIncrementalInsertQuotedColumnsAllCols: 56 | 57 | @pytest.fixture(scope="class") 58 | def seeds(self): 59 | return { 60 | "seed.csv": seed_csv, 61 | "add_new_rows.sql": seeds__add_new_rows_sql 62 | } 63 | 64 | @pytest.fixture(scope="class") 65 | def models(self): 66 | return { 67 | "my_incr_model.sql": my_incr_model_sql, 68 | } 69 | 70 | def test_run_dbt(self, project): 71 | """ 72 | - run seed 73 | - run incremental model 74 | - add new rows 75 | - run incremental model 76 | 77 | The following SQL is expected to run. 78 | 79 | merge into dbt_test.my_incr_model target 80 | using o$pt_my_incr_model175348 temp 81 | on ( 82 | temp."_user_id" = target."_user_id" 83 | ) 84 | when matched then 85 | update set 86 | target.USER_NAME = temp.USER_NAME, 87 | target.BIRTH_DATE = temp.BIRTH_DATE, 88 | target.INCOME = temp.INCOME, 89 | target.LAST_LOGIN_DATE = temp.LAST_LOGIN_DATE, 90 | target."desc" = temp."desc" 91 | when not matched then 92 | insert("_user_id", USER_NAME, BIRTH_DATE, INCOME, LAST_LOGIN_DATE, "desc") 93 | values( 94 | temp."_user_id", 95 | temp.USER_NAME, 96 | temp.BIRTH_DATE, 97 | temp.INCOME, 98 | temp.LAST_LOGIN_DATE, 99 | temp."desc" 100 | ) 101 | 102 | 103 | """ 104 | results = run_dbt(['seed']) 105 | assert len(results) == 1 106 | 107 | results = run_dbt(['run']) 108 | assert len(results) == 1 109 | 110 | project.run_sql_file(Path("seeds") / Path("add_new_rows.sql")) 111 | 112 | results = run_dbt(['run']) 113 | assert len(results) == 1 114 | 115 | user_id_2_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, 116 | 'my_incr_model', 117 | 2) 118 | result = project.run_sql(user_id_2_query, fetch="all") 119 | assert len(result) == 2 120 | 121 | used_id_5_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, 122 | 'my_incr_model', 123 | 5) 124 | expected_result = [(5, 'John Doe', 125 | datetime.datetime(1992, 10, 1, 0, 0), 126 | 300000, 127 | datetime.datetime(2022, 6, 1, 6, 1, 31), 128 | 'Login')] 129 | 130 | result = project.run_sql(used_id_5_query, fetch="all") 131 | assert result == expected_result 132 | 133 | 134 | -------------------------------------------------------------------------------- /tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import datetime 19 | from pathlib import Path 20 | 21 | import pytest 22 | from dbt.tests.util import run_dbt 23 | 24 | # seeds/my_seed.csv 25 | seed_csv = """ 26 | user_id,user_name,birth_date,income,last_login_date,description 27 | 1,Easton,1981-05-20,40000,2022-04-25T08:57:59,login 28 | 2,Lillian,1978-09-03,54000,2022-04-25T08:58:59,login 29 | 3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59,login 30 | 4,Nolan,1976-05-06,900000,2022-04-25T09:58:59,login 31 | """.lstrip() 32 | 33 | model_yml = """ 34 | version: 2 35 | models: 36 | - name: my_incr_model 37 | columns: 38 | - name: user_name 39 | quote: true 40 | - name: user_id 41 | quote: true 42 | - name: birth_date 43 | quote: true 44 | - name: income 45 | quote: true 46 | - name: last_login_date 47 | quote: true 48 | - name: description 49 | quote: true 50 | """ 51 | 52 | 53 | # models/my_incr_model.sql 54 | my_incr_model_sql = """ 55 | {{config(materialized='incremental', 56 | merge_update_columns=["user_name", "birth_date", "income", "last_login_date", "description"], 57 | unique_key='user_id')}} 58 | SELECT * FROM {{ ref('seed') }} 59 | {% if is_incremental() %} 60 | WHERE "last_login_date" > (SELECT max("last_login_date") FROM {{ this }}) 61 | {% endif %} 62 | 63 | """ 64 | 65 | # seeds/add_new_rows.sql 66 | seeds__add_new_rows_sql = """ 67 | -- insert two new rows, both of which should be in incremental model 68 | INSERT ALL 69 | INTO {schema}.seed ("user_id", "user_name", "birth_date", "income", "last_login_date", "description") VALUES 70 | (2,'Lillian Sr.', TO_DATE('1982-02-03', 'YYYY-MM-DD'), 200000, TO_DATE('2022-05-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') 71 | INTO {schema}.seed ("user_id", "user_name", "birth_date", "income", "last_login_date", "description") VALUES 72 | (5,'John Doe',TO_DATE('1992-10-01', 'YYYY-MM-DD'), 300000, TO_DATE('2022-06-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') 73 | SELECT * FROM dual 74 | """ 75 | 76 | 77 | class TestIncrementalMergeQuotedColumnsConfigYml: 78 | 79 | @pytest.fixture(scope="class") 80 | def seeds(self): 81 | return { 82 | "seed.csv": seed_csv, 83 | "add_new_rows.sql": seeds__add_new_rows_sql 84 | } 85 | 86 | @pytest.fixture(scope="class") 87 | def models(self): 88 | return { 89 | "my_incr_model.sql": my_incr_model_sql, 90 | "schema.yml": model_yml 91 | } 92 | 93 | @pytest.fixture(scope="class") 94 | def project_config_update(self): 95 | return { 96 | "seeds": { 97 | "quote_columns": True 98 | }, 99 | } 100 | 101 | def test_run_dbt(self, project): 102 | """ 103 | - run seed 104 | - run incremental model 105 | - add new rows 106 | - run incremental model 107 | 108 | The following SQL is expected to run. 109 | 110 | """ 111 | results = run_dbt(['seed']) 112 | assert len(results) == 1 113 | 114 | results = run_dbt(['run']) 115 | assert len(results) == 1 116 | 117 | project.run_sql_file(Path("seeds") / Path("add_new_rows.sql")) 118 | 119 | results = run_dbt(['run']) 120 | assert len(results) == 1 121 | 122 | user_id_2_query = 'SELECT * FROM {}.{} WHERE "user_id" = {}'.format(project.test_schema, 123 | 'my_incr_model', 124 | 2) 125 | expected_result = [(2, 'Lillian Sr.', 126 | datetime.datetime(1982, 2, 3, 0, 0), 127 | 200000, 128 | datetime.datetime(2022, 5, 1, 6, 1, 31), 129 | 'Login')] 130 | 131 | result = project.run_sql(user_id_2_query, fetch="all") 132 | assert result == expected_result 133 | 134 | used_id_5_query = 'SELECT * FROM {}.{} WHERE "user_id" = {}'.format(project.test_schema, 135 | 'my_incr_model', 136 | 5) 137 | expected_result = [(5, 'John Doe', 138 | datetime.datetime(1992, 10, 1, 0, 0), 139 | 300000, 140 | datetime.datetime(2022, 6, 1, 6, 1, 31), 141 | 'Login')] 142 | 143 | result = project.run_sql(used_id_5_query, fetch="all") 144 | assert result == expected_result 145 | 146 | 147 | -------------------------------------------------------------------------------- /tests/functional/adapter/incremental_materialization/sync_schema/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/dbt-oracle/456ceb9d19161508df67056c4187332e079cde18/tests/functional/adapter/incremental_materialization/sync_schema/__init__.py -------------------------------------------------------------------------------- /tests/functional/adapter/incremental_materialization/test_incremental_predicates.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | import pytest 18 | from dbt.tests.adapter.incremental.test_incremental_predicates import BaseIncrementalPredicates 19 | 20 | 21 | models__delete_insert_incremental_predicates_sql = """ 22 | {{ config( 23 | materialized = 'incremental', 24 | unique_key = 'id' 25 | ) }} 26 | 27 | {% if not is_incremental() %} 28 | 29 | select 1 as id, 'hello' as msg, 'blue' as color from dual 30 | union all 31 | select 2 as id, 'goodbye' as msg, 'red' as color from dual 32 | 33 | {% else %} 34 | 35 | -- delete will not happen on the above record where id = 2, so new record will be inserted instead 36 | select 1 as id, 'hey' as msg, 'blue' as color from dual 37 | union all 38 | select 2 as id, 'yo' as msg, 'green' as color from dual 39 | union all 40 | select 3 as id, 'anyway' as msg, 'purple' as color from dual 41 | 42 | {% endif %} 43 | """ 44 | 45 | class TestIncrementalPredicatesMergeOracle(BaseIncrementalPredicates): 46 | 47 | @pytest.fixture(scope="class") 48 | def models(self): 49 | return { 50 | "delete_insert_incremental_predicates.sql": models__delete_insert_incremental_predicates_sql 51 | } 52 | 53 | @pytest.fixture(scope="class") 54 | def project_config_update(self): 55 | return { 56 | "models": { 57 | "+incremental_predicates": [ 58 | "dbt_internal_dest.id != 2" 59 | ], 60 | "+incremental_strategy": "merge" 61 | } 62 | } 63 | 64 | 65 | class TestPredicatesMergeOracle(BaseIncrementalPredicates): 66 | 67 | @pytest.fixture(scope="class") 68 | def models(self): 69 | return { 70 | "delete_insert_incremental_predicates.sql": models__delete_insert_incremental_predicates_sql 71 | } 72 | 73 | @pytest.fixture(scope="class") 74 | def project_config_update(self): 75 | return { 76 | "models": { 77 | "+predicates": [ 78 | "dbt_internal_dest.id != 2" 79 | ], 80 | "+incremental_strategy": "merge" 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /tests/functional/adapter/incremental_materialization/test_merge_update_columns.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import datetime 19 | from pathlib import Path 20 | 21 | import pytest 22 | from dbt.tests.util import run_dbt, get_relation_columns 23 | 24 | # seeds/my_seed.csv 25 | seed_csv = """ 26 | user_id,user_name,birth_date,income,last_login_date 27 | 1,Easton,1981-05-20,40000,2022-04-25T08:57:59 28 | 2,Lillian,1978-09-03,54000,2022-04-25T08:58:59 29 | 3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59 30 | 4,Nolan,1976-05-06,900000,2022-04-25T09:58:59 31 | """.lstrip() 32 | 33 | # models/my_incr_model.sql 34 | my_incr_model_sql = """ 35 | {{config(materialized='incremental', 36 | unique_key='user_id', 37 | merge_update_columns=['user_name', 'income', 'last_login_date'])}} 38 | SELECT * FROM {{ ref('seed') }} 39 | {% if is_incremental() %} 40 | WHERE last_login_date > (SELECT max(last_login_date) FROM {{ this }}) 41 | {% endif %} 42 | 43 | """ 44 | 45 | # seeds/add_new_rows.sql 46 | seeds__add_new_rows_sql = """ 47 | -- insert two new rows, both of which should be in incremental model 48 | INSERT ALL 49 | INTO {schema}.seed (user_id, user_name, birth_date, income, last_login_date) VALUES 50 | (2,'Lillian Sr.', TO_DATE('1982-02-03', 'YYYY-MM-DD'), 200000, TO_DATE('2022-05-01 06:01:31', 'YYYY-MM-DD HH:MI:SS')) 51 | INTO {schema}.seed (user_id, user_name, birth_date, income, last_login_date) VALUES 52 | (5,'John Doe',TO_DATE('1992-10-01', 'YYYY-MM-DD'), 300000, TO_DATE('2022-06-01 06:01:31', 'YYYY-MM-DD HH:MI:SS')) 53 | SELECT * FROM dual 54 | """ 55 | 56 | 57 | class TestIncrementalMergeUpdateColumns: 58 | 59 | @pytest.fixture(scope="class") 60 | def seeds(self): 61 | return { 62 | "seed.csv": seed_csv, 63 | "add_new_rows.sql": seeds__add_new_rows_sql 64 | } 65 | 66 | @pytest.fixture(scope="class") 67 | def models(self): 68 | return { 69 | "my_incr_model.sql": my_incr_model_sql, 70 | } 71 | 72 | def test_run_dbt(self, project): 73 | """ 74 | - run seed 75 | - run incremental model 76 | - add new rows 77 | - run incremental model 78 | 79 | The following SQL is expected to run. 80 | 81 | merge into dbt_test.my_incr_model target 82 | using o$pt_my_incr_model150332 temp 83 | on ( 84 | temp.USER_ID = target.USER_ID 85 | ) 86 | when matched then 87 | update set 88 | target.user_name = temp.user_name, 89 | target.income = temp.income, 90 | target.last_login_date = temp.last_login_date 91 | when not matched then 92 | insert(USER_ID, USER_NAME, BIRTH_DATE, INCOME, LAST_LOGIN_DATE) 93 | values( 94 | temp.USER_ID, 95 | temp.USER_NAME, 96 | temp.BIRTH_DATE, 97 | temp.INCOME, 98 | temp.LAST_LOGIN_DATE 99 | ) 100 | 101 | """ 102 | results = run_dbt(['seed']) 103 | assert len(results) == 1 104 | 105 | results = run_dbt(['run']) 106 | assert len(results) == 1 107 | 108 | project.run_sql_file(Path("seeds") / Path("add_new_rows.sql")) 109 | 110 | results = run_dbt(['run']) 111 | assert len(results) == 1 112 | 113 | user_id_2_query = "SELECT * FROM {}.{} WHERE user_id = {}".format(project.test_schema, 114 | 'my_incr_model', 115 | 2) 116 | expected_result = [(2, 'Lillian Sr.', 117 | datetime.datetime(1978, 9, 3, 0, 0), 118 | 200000, 119 | datetime.datetime(2022, 5, 1, 6, 1, 31))] 120 | 121 | result = project.run_sql(user_id_2_query, fetch="all") 122 | assert result == expected_result 123 | 124 | used_id_5_query = "SELECT * FROM {}.{} WHERE user_id = {}".format(project.test_schema, 125 | 'my_incr_model', 126 | 5) 127 | expected_result = [(5, 'John Doe', 128 | datetime.datetime(1992, 10, 1, 0, 0), 129 | 300000, 130 | datetime.datetime(2022, 6, 1, 6, 1, 31))] 131 | 132 | result = project.run_sql(used_id_5_query, fetch="all") 133 | assert result == expected_result 134 | 135 | 136 | 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /tests/functional/adapter/macros/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/dbt-oracle/456ceb9d19161508df67056c4187332e079cde18/tests/functional/adapter/macros/__init__.py -------------------------------------------------------------------------------- /tests/functional/adapter/macros/test_alter_column_type.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import pytest 19 | from dbt.tests.util import run_dbt, get_relation_columns 20 | 21 | # seeds/my_seed.csv 22 | my_seed_csv = """ 23 | id,name,some_date 24 | 1,Easton,1981-05-20T06:46:51 25 | 2,Lillian,1978-09-03T18:10:33 26 | 3,Jeremiah,1982-03-11T03:59:51 27 | 4,Nolan,1976-05-06T20:21:35 28 | """.lstrip() 29 | 30 | # models/my_model.sql 31 | my_model_sql = """ 32 | {{config(materialized='table')}} 33 | select * from {{ ref('my_seed') }} 34 | """ 35 | 36 | # wrapper macro which calls the alter_column_type macro 37 | alter_column_type_wrapper_macro = """ 38 | {% macro wrap_alter_column_type() %} 39 | {%- set relation = adapter.get_relation(database=target.database, schema=schema, identifier='my_model') -%} 40 | {{ return(adapter.dispatch('alter_column_type')(relation, 'name', 'CLOB')) }} 41 | {% endmacro %} 42 | """ 43 | 44 | 45 | class TestAlterColumnDataTypeMacro: 46 | """ 47 | Tests the macro `oracle__alter_column_type` 48 | 49 | """ 50 | 51 | @pytest.fixture(scope="class") 52 | def seeds(self): 53 | return { 54 | "my_seed.csv": my_seed_csv, 55 | } 56 | 57 | @pytest.fixture(scope="class") 58 | def models(self): 59 | return { 60 | "my_model.sql": my_model_sql, 61 | } 62 | 63 | @pytest.fixture(scope="class") 64 | def macros(self): 65 | return {"wrap_alter_column_type.sql": alter_column_type_wrapper_macro} 66 | 67 | def test_run_macro(self, project): 68 | """seed, then run, then macro to alter table column datatype 69 | 70 | """ 71 | results = run_dbt(['seed']) 72 | assert len(results) == 1 73 | 74 | results = run_dbt(['run']) 75 | assert len(results) == 1 76 | 77 | run_dbt(['run-operation', 'wrap_alter_column_type']) 78 | columns = get_relation_columns(project.adapter, 'my_model') 79 | expected_columns = [('ID', 'NUMBER', 0), 80 | ('NAME', 'CLOB', 0), 81 | ('SOME_DATE', 'TIMESTAMP(6)', 0)] 82 | assert expected_columns == columns 83 | 84 | 85 | -------------------------------------------------------------------------------- /tests/functional/adapter/materialized_view/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/dbt-oracle/456ceb9d19161508df67056c4187332e079cde18/tests/functional/adapter/materialized_view/__init__.py -------------------------------------------------------------------------------- /tests/functional/adapter/materialized_view/utils.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from dbt.adapters.base.relation import BaseRelation 4 | from dbt.adapters.oracle.relation import OracleRelation 5 | 6 | 7 | def query_relation_type(project, relation: BaseRelation) -> Optional[str]: 8 | assert isinstance(relation, OracleRelation) 9 | 10 | sql = f""" 11 | with tables as 12 | (select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, 13 | owner table_schema, 14 | table_name, 15 | case 16 | when iot_type = 'Y' 17 | then 'IOT' 18 | when temporary = 'Y' 19 | then 'TEMP' 20 | else 'BASE TABLE' 21 | end table_type 22 | from sys.all_tables 23 | where upper(table_name) not in (select upper(mview_name) from sys.all_mviews) 24 | union all 25 | select SYS_CONTEXT('userenv', 'DB_NAME'), 26 | owner, 27 | view_name, 28 | 'VIEW' 29 | from sys.all_views 30 | union all 31 | select SYS_CONTEXT('userenv', 'DB_NAME'), 32 | owner, 33 | mview_name, 34 | 'MATERIALIZED VIEW' 35 | from sys.all_mviews 36 | ) 37 | select case table_type 38 | when 'BASE TABLE' then 'table' 39 | when 'VIEW' then 'view' 40 | when 'MATERIALIZED VIEW' then 'materialized_view' 41 | end as "relation_type" 42 | from tables 43 | where table_type in ('BASE TABLE', 'VIEW', 'MATERIALIZED VIEW') 44 | and upper(table_schema) = upper('{relation.schema}') 45 | and upper(table_name) = upper('{relation.identifier}') 46 | """ 47 | 48 | results = project.run_sql(sql, fetch="all") 49 | if len(results) == 0: 50 | return None 51 | elif len(results) > 1: 52 | raise ValueError(f"More than one instance of {relation.identifier} found!") 53 | else: 54 | return results[0][0] 55 | 56 | 57 | def query_refresh_method(project, relation: OracleRelation): 58 | sql = f"""SELECT refresh_method 59 | FROM sys.all_mviews 60 | WHERE mview_name = '{ relation.identifier.upper() }'""" 61 | return project.run_sql(sql, fetch="one")[0].upper() 62 | 63 | 64 | def query_refresh_mode(project, relation: OracleRelation): 65 | sql = f"""SELECT refresh_mode 66 | FROM sys.all_mviews 67 | WHERE mview_name = '{ relation.identifier.upper() }'""" 68 | return project.run_sql(sql, fetch="one")[0].upper() 69 | 70 | 71 | def query_build_mode(project, relation: OracleRelation): 72 | sql = f"""SELECT build_mode 73 | FROM sys.all_mviews 74 | WHERE mview_name = '{relation.identifier.upper()}'""" 75 | return project.run_sql(sql, fetch="one")[0].upper() 76 | 77 | 78 | def query_rewrite_enabled(project, relation: OracleRelation): 79 | sql = f"""SELECT rewrite_enabled 80 | FROM sys.all_mviews 81 | WHERE mview_name = '{relation.identifier.upper() }'""" 82 | return "enable" if project.run_sql(sql, fetch="one")[0] == "Y" else "disable" 83 | 84 | -------------------------------------------------------------------------------- /tests/functional/adapter/simple_seed/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | -------------------------------------------------------------------------------- /tests/functional/adapter/simple_seed/test_simple_seed.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | import pytest 18 | 19 | from dbt.tests.adapter.simple_seed.test_seed import SeedConfigBase 20 | from dbt.tests.util import run_dbt 21 | 22 | 23 | class TestSimpleBigSeedBatched(SeedConfigBase): 24 | @pytest.fixture(scope="class") 25 | def seeds(self): 26 | seed_data = ["seed_id"] 27 | seed_data.extend([str(i) for i in range(20_000)]) 28 | return {"big_batched_seed.csv": "\n".join(seed_data)} 29 | 30 | def test_big_batched_seed(self, project): 31 | seed_results = run_dbt(["seed"]) 32 | assert len(seed_results) == 1 33 | -------------------------------------------------------------------------------- /tests/functional/adapter/snapshots/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/dbt-oracle/456ceb9d19161508df67056c4187332e079cde18/tests/functional/adapter/snapshots/__init__.py -------------------------------------------------------------------------------- /tests/functional/adapter/snapshots/test_invalidate_deletes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2025, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import pytest 19 | from pathlib import Path 20 | 21 | from dbt.tests.util import run_dbt 22 | 23 | # seeds/my_seed.csv 24 | my_seed_csv = """ 25 | id,name,some_date 26 | 1,Easton,1981-05-20T06:46:51 27 | 2,Lillian,1978-09-03T18:10:33 28 | 3,Jeremiah,1982-03-11T03:59:51 29 | 4,Nolan,1976-05-06T20:21:35 30 | """.lstrip() 31 | 32 | 33 | cc_all_snapshot_sql = """ 34 | {% snapshot cc_all_snapshot %} 35 | {{ config( 36 | check_cols='all', 37 | unique_key=['id'], 38 | strategy='check', 39 | target_database=database, 40 | target_schema=schema, 41 | invalidate_hard_deletes=True 42 | ) }} 43 | SELECT * FROM {{ ref('seed') }} 44 | {% endsnapshot %} 45 | """.strip() 46 | 47 | # seeds/insert.sql 48 | seeds__insert_sql = """ 49 | INSERT ALL 50 | INTO {schema}.seed (id, name, some_date) VALUES 51 | (5, 'John Doe', TO_DATE('1982-02-03', 'YYYY-MM-DD')) 52 | SELECT * FROM dual 53 | """ 54 | 55 | # seeds/update.sql 56 | seeds__update_sql = """ 57 | UPDATE {schema}.seed 58 | SET name = 'Lord Easton' 59 | WHERE id = 1 60 | """ 61 | 62 | # seeds/delete.sql 63 | seeds__delete_sql = """ 64 | DELETE FROM {schema}.seed WHERE id = 2 65 | """ 66 | 67 | 68 | class TestSnapshotCheckInvalidateHardDeletes: 69 | 70 | @pytest.fixture(scope="class") 71 | def seeds(self): 72 | return { 73 | "seed.csv": my_seed_csv, 74 | "insert.sql": seeds__insert_sql, 75 | "update.sql": seeds__update_sql, 76 | "delete.sql": seeds__delete_sql 77 | 78 | } 79 | 80 | @pytest.fixture(scope="class") 81 | def snapshots(self): 82 | return { 83 | "cc_all_snapshot.sql": cc_all_snapshot_sql, 84 | } 85 | 86 | def test_run_dbt(self, project): 87 | """dbt seed 88 | dbt snapshot 89 | Perform insert/update/delete 90 | dbt snapshot 91 | 92 | MERGE INTO dbt_test.cc_all_snapshot d 93 | USING o$pt_cc_all_snapshot182811 s 94 | ON (s.dbt_scd_id = d.dbt_scd_id) 95 | WHEN MATCHED 96 | THEN UPDATE 97 | SET dbt_valid_to = s.dbt_valid_to 98 | WHERE d.dbt_valid_to IS NULL 99 | AND s.dbt_change_type IN ('update', 'delete') 100 | WHEN NOT MATCHED 101 | THEN INSERT (d.id, d.name, d.some_date, d.dbt_updated_at, d.dbt_valid_from, d.dbt_valid_to, d.dbt_scd_id) 102 | VALUES (s.id, s.name, s.some_date, s.dbt_updated_at, s.dbt_valid_from, s.dbt_valid_to, s.dbt_scd_id) 103 | WHERE s.dbt_change_type = 'insert' 104 | 105 | """ 106 | results = run_dbt(['seed']) 107 | assert len(results) == 1 108 | 109 | # snapshot command 110 | results = run_dbt(["snapshot"]) 111 | for result in results: 112 | assert result.status == "success" 113 | 114 | project.run_sql_file(Path("seeds") / Path("insert.sql")) 115 | project.run_sql_file(Path("seeds") / Path("update.sql")) 116 | project.run_sql_file(Path("seeds") / Path("delete.sql")) 117 | 118 | # run snapshot command 119 | results = run_dbt(["snapshot"]) 120 | for result in results: 121 | assert result.status == "success" 122 | 123 | snapshot_of_updated_rows = project.run_sql(f"select * from cc_all_snapshot where id=1", fetch="all") 124 | assert len(snapshot_of_updated_rows) == 2 125 | 126 | # Deleted record will be invalidated. r['dbt_valid_to'] is set to current timestamp. 127 | snapshot_of_deleted_rows = project.run_sql(f"select * from cc_all_snapshot where id=2", fetch="all") 128 | assert len(snapshot_of_deleted_rows) == 1 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_caching.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import pytest 17 | 18 | from dbt.tests.adapter.caching.test_caching import BaseCachingTest 19 | 20 | from dbt.tests.util import run_dbt 21 | 22 | model_sql = """ 23 | {{ 24 | config( 25 | materialized='table' 26 | ) 27 | }} 28 | select 1 as id from dual 29 | """ 30 | 31 | another_schema_model_sql = """ 32 | {{ 33 | config( 34 | materialized='table', 35 | schema='another_schema' 36 | ) 37 | }} 38 | select 1 as id from dual 39 | """ 40 | 41 | 42 | class OracleBaseCaching(BaseCachingTest): 43 | 44 | def run_and_inspect_cache(self, project, run_args=None): 45 | run_dbt(run_args) 46 | 47 | # the cache was empty at the start of the run. 48 | # the model materialization returned an unquoted relation and added to the cache. 49 | adapter = project.adapter 50 | assert len(adapter.cache.relations) == 1 51 | relation = list(adapter.cache.relations).pop() 52 | # assert relation.schema == project.test_schema 53 | assert relation.schema == project.test_schema.lower() 54 | 55 | # on the second run, dbt will find a relation in the database during cache population. 56 | # this relation will be quoted, because list_relations_without_caching (by default) uses 57 | # quote_policy = {"database": True, "schema": True, "identifier": True} 58 | # when adding relations to the cache. 59 | run_dbt(run_args) 60 | adapter = project.adapter 61 | assert len(adapter.cache.relations) == 1 62 | second_relation = list(adapter.cache.relations).pop() 63 | 64 | # perform a case-insensitive + quote-insensitive comparison 65 | for key in ["database", "schema", "identifier"]: 66 | assert getattr(relation, key).lower() == getattr(second_relation, key).lower() 67 | 68 | 69 | class TestCachingLowerCaseModel(OracleBaseCaching): 70 | 71 | @pytest.fixture(scope="class") 72 | def models(self): 73 | return { 74 | "model.sql": model_sql, 75 | } 76 | 77 | 78 | class TestCachingUppercaseModel(OracleBaseCaching): 79 | 80 | @pytest.fixture(scope="class") 81 | def models(self): 82 | return { 83 | "MODEL.sql": model_sql, 84 | } 85 | 86 | 87 | class TestCachingSelectedSchemaOnly(OracleBaseCaching): 88 | 89 | @pytest.fixture(scope="class") 90 | def models(self): 91 | return { 92 | "model.sql": model_sql, 93 | "another_schema_model.sql": another_schema_model_sql, 94 | } 95 | 96 | def test_cache(self, project): 97 | # this should only cache the schema containing the selected model 98 | run_args = ["--cache-selected-only", "run", "--select", "model"] 99 | self.run_and_inspect_cache(project, run_args) 100 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_concurrency.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | import pytest 18 | 19 | from dbt.tests.adapter.concurrency.test_concurrency import TestConcurenncy 20 | 21 | 22 | class TestConcurrencyOracle(TestConcurenncy): 23 | pass 24 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import pytest 19 | 20 | # dbt imports 21 | from dbt.clients.yaml_helper import load_yaml_text 22 | 23 | # dbt-oracle imports 24 | from dbt.adapters.oracle import OracleAdapterCredentials 25 | from dbt.adapters.oracle.connections import OracleConnectionMethod 26 | 27 | 28 | def get_credentials(profile_yml): 29 | " Render a YAML string profiles.yml into credentials " 30 | dicty_thing = load_yaml_text(profile_yml) 31 | dicty_thing["default"]["outputs"]["target"].pop("type") 32 | dicty_thing["default"]["outputs"]["target"].pop("threads") 33 | dicty_thing["default"]["outputs"]["target"]["database"] = "PDB1" 34 | return OracleAdapterCredentials(**dicty_thing["default"]["outputs"]["target"]) 35 | 36 | # Define data 37 | SCENARIOS = { 38 | "host": { 39 | "method": OracleConnectionMethod.HOST, 40 | "profile": """ 41 | default: 42 | target: target 43 | outputs: 44 | target: 45 | type: oracle 46 | host: localhost 47 | user: dbt_test 48 | password: dbt_test 49 | protocol: tcps 50 | service: xe 51 | schema: dbt_test 52 | port: 1522 53 | threads: 1 54 | """, 55 | "dsn": "tcps://localhost:1522/xe?retry_count=1&retry_delay=3", 56 | }, 57 | "host_service": { 58 | "method": OracleConnectionMethod.HOST, 59 | "profile": """ 60 | default: 61 | target: target 62 | outputs: 63 | target: 64 | type: oracle 65 | host: localhost 66 | user: dbt_test 67 | password: dbt_test 68 | service: xe_ha.host.tld 69 | schema: dbt_test 70 | protocol: tcps 71 | port: 1522 72 | threads: 1 73 | """, 74 | "dsn": "tcps://localhost:1522/xe_ha.host.tld?retry_count=1&retry_delay=3", 75 | }, 76 | "tns": { 77 | "method": OracleConnectionMethod.TNS, 78 | "profile": """ 79 | default: 80 | target: target 81 | outputs: 82 | target: 83 | type: oracle 84 | user: dbt_test 85 | password: dbt_test 86 | tns_name: xe 87 | schema: dbt_test 88 | port: 1522 89 | protocol: tcps 90 | threads: 1 91 | """, 92 | "dsn": "xe", 93 | }, 94 | "connection_string": { 95 | "method": OracleConnectionMethod.CONNECTION_STRING, 96 | "profile": """ 97 | default: 98 | target: target 99 | outputs: 100 | target: 101 | type: oracle 102 | host: localhost 103 | user: dbt_test 104 | password: dbt_test 105 | connection_string: "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=localhost)(PORT=1522))(CONNECT_DATA=(SERVICE_NAME=xe)))" 106 | schema: dbt_test 107 | port: 1522 108 | threads: 1 109 | """, 110 | "dsn": "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=localhost)(PORT=1522))(CONNECT_DATA=(SERVICE_NAME=xe)))", 111 | }, 112 | } 113 | 114 | 115 | @pytest.fixture(scope="module", params=SCENARIOS.keys()) 116 | def scenario(request): 117 | return SCENARIOS[request.param] 118 | 119 | 120 | def test_oracle_credentials(scenario): 121 | for method, parameters in SCENARIOS.items(): 122 | credentials = get_credentials(scenario["profile"]) 123 | assert credentials.connection_method() == scenario["method"] 124 | assert credentials.get_dsn() == scenario["dsn"] 125 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_data_types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | import pytest 18 | 19 | from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean 20 | from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt 21 | from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat 22 | from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt 23 | from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric 24 | 25 | 26 | models__bigint_expected_sql = """ 27 | select 9223372036854775800 as bigint_col from dual 28 | """.lstrip() 29 | 30 | models__bigint_actual_sql = """ 31 | select cast('9223372036854775800' as {{ type_bigint() }}) as bigint_col from dual 32 | """ 33 | 34 | models__float_actual_sql = """ 35 | select cast('1.2345' as {{ type_float() }}) as float_col from dual 36 | """ 37 | 38 | models__int_actual_sql = """ 39 | select cast('12345678' as {{ type_int() }}) as int_col from dual 40 | """ 41 | 42 | models__numeric_actual_sql = """ 43 | select cast('1.2345' as {{ type_numeric() }}) as numeric_col from dual 44 | """ 45 | 46 | 47 | seeds__boolean_expected_csv = """boolean_col 48 | 1 49 | """.lstrip() 50 | 51 | models__boolean_actual_sql = """ 52 | select cast('1' as {{ type_boolean() }}) as boolean_col from dual 53 | """ 54 | 55 | 56 | class TestTypeBigIntOracle(BaseTypeBigInt): 57 | 58 | @pytest.fixture(scope="class") 59 | def models(self): 60 | return { 61 | "expected.sql": models__bigint_expected_sql, 62 | "actual.sql": self.interpolate_macro_namespace(models__bigint_actual_sql, "type_bigint"), 63 | } 64 | 65 | 66 | class TestTypeFloatOracle(BaseTypeFloat): 67 | 68 | @pytest.fixture(scope="class") 69 | def models(self): 70 | return {"actual.sql": self.interpolate_macro_namespace(models__float_actual_sql, "type_float")} 71 | 72 | 73 | class TestTypeIntOracle(BaseTypeInt): 74 | 75 | @pytest.fixture(scope="class") 76 | def models(self): 77 | return {"actual.sql": self.interpolate_macro_namespace(models__int_actual_sql, "type_int")} 78 | 79 | 80 | class TestTypeNumericOracle(BaseTypeNumeric): 81 | 82 | @pytest.fixture(scope="class") 83 | def models(self): 84 | return {"actual.sql": self.interpolate_macro_namespace(models__numeric_actual_sql, "type_numeric")} 85 | 86 | 87 | class TestTypeBooleanOracle(BaseTypeBoolean): 88 | 89 | @pytest.fixture(scope="class") 90 | def seeds(self): 91 | return {"expected.csv": seeds__boolean_expected_csv} 92 | 93 | @pytest.fixture(scope="class") 94 | def models(self): 95 | return {"actual.sql": self.interpolate_macro_namespace(models__boolean_actual_sql, "type_boolean")} 96 | 97 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_dbt_show.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | 15 | import pytest 16 | from dbt.tests.util import run_dbt 17 | 18 | from dbt.tests.adapter.dbt_show.fixtures import models__second_ephemeral_model 19 | from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit 20 | 21 | models__sql_header = """ 22 | {% call set_sql_header(config) %} 23 | with variables as ( 24 | select 1 as my_variable from dual 25 | ) 26 | {%- endcall %} 27 | select my_variable from variables 28 | """ 29 | 30 | 31 | class TestOracleShowLimit(BaseShowLimit): 32 | 33 | @pytest.mark.parametrize( 34 | "args,expected", 35 | [ 36 | ([], 5), # default limit 37 | (["--limit", 3], 3), # fetch 3 rows 38 | (["--limit", -1], 7), # fetch all rows 39 | ], 40 | ) 41 | def test_limit(self, project, args, expected): 42 | run_dbt(["build"]) 43 | dbt_args = ["show", "--inline", models__second_ephemeral_model, *args] 44 | results = run_dbt(dbt_args) 45 | assert len(results.results[0].agate_table) == expected 46 | # ensure limit was injected in compiled_code when limit specified in command args 47 | limit = results.args.get("limit") 48 | if limit > 0: 49 | assert f"fetch first { limit } rows only" in results.results[0].node.compiled_code 50 | 51 | 52 | class TestOracleShowSqlHeader(BaseShowSqlHeader): 53 | 54 | @pytest.fixture(scope="class") 55 | def models(self): 56 | return { 57 | "sql_header.sql": models__sql_header, 58 | } 59 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_ephemeral.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | import pytest 18 | 19 | from dbt.tests.util import run_dbt, check_relations_equal 20 | from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti 21 | 22 | 23 | class TestEphemeralMultiOracle(BaseEphemeralMulti): 24 | 25 | def test_ephemeral_multi(self, project): 26 | run_dbt(["seed"]) 27 | results = run_dbt(["run"]) 28 | assert len(results) == 3 29 | 30 | check_relations_equal(project.adapter, ["seed", "dependent"]) 31 | check_relations_equal(project.adapter, ["seed", "double_dependent"]) 32 | check_relations_equal(project.adapter, ["seed", "super_dependent"]) 33 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_generictests_where.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import pytest 19 | 20 | from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests 21 | from dbt.tests.adapter.basic.files import ( 22 | seeds_base_csv, 23 | base_view_sql, 24 | base_table_sql, 25 | schema_base_yml, 26 | ) 27 | 28 | generic_test_seed_yml = """ 29 | version: 2 30 | models: 31 | - name: base 32 | columns: 33 | - name: id 34 | tests: 35 | - not_null: 36 | config: 37 | where: "name = 'Easton'" 38 | """ 39 | 40 | generic_test_view_yml = """ 41 | version: 2 42 | models: 43 | - name: view_model 44 | columns: 45 | - name: id 46 | tests: 47 | - not_null: 48 | config: 49 | where: "name = 'Easton'" 50 | """ 51 | 52 | generic_test_table_yml = """ 53 | version: 2 54 | models: 55 | - name: table_model 56 | columns: 57 | - name: id 58 | tests: 59 | - not_null: 60 | config: 61 | where: "name = 'Easton'" 62 | """ 63 | 64 | 65 | class TestGenericTestsOracle(BaseGenericTests): 66 | 67 | @pytest.fixture(scope="class") 68 | def seeds(self): 69 | return { 70 | "base.csv": seeds_base_csv, 71 | "schema.yml": generic_test_seed_yml, 72 | } 73 | 74 | @pytest.fixture(scope="class") 75 | def models(self): 76 | return { 77 | "view_model.sql": base_view_sql, 78 | "table_model.sql": base_table_sql, 79 | "schema.yml": schema_base_yml, 80 | "schema_view.yml": generic_test_view_yml, 81 | "schema_table.yml": generic_test_table_yml, 82 | } 83 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_get_last_relation_modified.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | import os 18 | import pytest 19 | 20 | from dbt.cli.main import dbtRunner 21 | 22 | freshness_via_metadata_schema_yml = """version: 2 23 | sources: 24 | - name: test_source 25 | freshness: 26 | warn_after: {count: 10, period: hour} 27 | error_after: {count: 1, period: day} 28 | schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}" 29 | loaded_at_field: ts 30 | tables: 31 | - name: test_table 32 | """ 33 | 34 | class TestGetLastRelationModified: 35 | @pytest.fixture(scope="class", autouse=True) 36 | def set_env_vars(self, project): 37 | os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema 38 | yield 39 | del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] 40 | 41 | @pytest.fixture(scope="class") 42 | def models(self): 43 | return {"schema.yml": freshness_via_metadata_schema_yml} 44 | 45 | @pytest.fixture(scope="class") 46 | def custom_schema(self, project, set_env_vars): 47 | with project.adapter.connection_named("__test"): 48 | relation = project.adapter.Relation.create( 49 | database=project.database, schema=os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] 50 | ) 51 | project.adapter.drop_schema(relation) 52 | project.adapter.create_schema(relation) 53 | 54 | yield relation.schema 55 | 56 | with project.adapter.connection_named("__test"): 57 | project.adapter.drop_schema(relation) 58 | 59 | def test_get_last_relation_modified(self, project, set_env_vars, custom_schema): 60 | project.run_sql( 61 | f"create table {custom_schema}.test_table (id integer, name varchar(100) not null, ts timestamp default systimestamp)" 62 | ) 63 | 64 | project.run_sql( 65 | f"insert into {custom_schema}.test_table (id, name) values (1, 'dbt_test')" 66 | ) 67 | 68 | project.run_sql("COMMIT") 69 | 70 | warning_or_error = False 71 | 72 | def probe(e): 73 | nonlocal warning_or_error 74 | if e.info.level in ["warning", "error"]: 75 | warning_or_error = True 76 | 77 | runner = dbtRunner(callbacks=[probe]) 78 | runner.invoke(["source", "freshness"]) 79 | 80 | # The 'source freshness' command should succeed without warnings or errors. 81 | assert not warning_or_error 82 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_grants.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022, Oracle and/or its affiliates. 3 | Copyright (c) 2020, Vitor Avancini 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | 18 | import pytest 19 | 20 | from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants, model_schema_yml 21 | from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants, incremental_model_schema_yml 22 | from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants 23 | from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants 24 | from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants, snapshot_schema_yml 25 | 26 | my_model_sql = """ 27 | select 1 as fun from dual 28 | """ 29 | 30 | my_incremental_model_sql = """ 31 | select 1 as fun from dual 32 | """ 33 | 34 | my_invalid_model_sql = """ 35 | select 1 as fun from dual 36 | """ 37 | 38 | my_snapshot_sql = """ 39 | {% snapshot my_snapshot %} 40 | {{ config( 41 | check_cols='all', unique_key=['id'], strategy='check', 42 | target_database=database, target_schema=schema 43 | ) }} 44 | select 1 as id, cast('blue' as {{ type_string() }}) as color 45 | {% endsnapshot %} 46 | """.strip() 47 | 48 | 49 | class TestSeedGrantsOracle(BaseSeedGrants): 50 | pass 51 | 52 | 53 | class TestModelGrantsOracle(BaseModelGrants): 54 | 55 | @pytest.fixture(scope="class") 56 | def models(self): 57 | updated_schema = self.interpolate_name_overrides(model_schema_yml) 58 | 59 | return { 60 | "my_model.sql": my_model_sql, 61 | "schema.yml": updated_schema, 62 | } 63 | 64 | 65 | class TestIncrementalGrantsOracle(BaseIncrementalGrants): 66 | 67 | @pytest.fixture(scope="class") 68 | def models(self): 69 | updated_schema = self.interpolate_name_overrides(incremental_model_schema_yml) 70 | return { 71 | "my_incremental_model.sql": my_incremental_model_sql, 72 | "schema.yml": updated_schema, 73 | } 74 | 75 | 76 | class TestInvalidGrantsOracle(BaseInvalidGrants): 77 | 78 | def grantee_does_not_exist_error(self): 79 | return "ORA-01917: user or role" 80 | 81 | def privilege_does_not_exist_error(self): 82 | return "ORA-00990: missing or invalid privilege" 83 | 84 | @pytest.fixture(scope="class") 85 | def models(self): 86 | return { 87 | "my_invalid_model.sql": my_invalid_model_sql, 88 | } 89 | 90 | 91 | class TestSnapshotGrantsOracle(BaseSnapshotGrants): 92 | 93 | @pytest.fixture(scope="class") 94 | def snapshots(self): 95 | return { 96 | "my_snapshot.sql": my_snapshot_sql, 97 | "schema.yml": self.interpolate_name_overrides(snapshot_schema_yml), 98 | } 99 | -------------------------------------------------------------------------------- /tests/functional/adapter/utils/test_date_spine.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | 15 | import pytest 16 | 17 | from dbt.tests.adapter.utils.base_utils import BaseUtils 18 | from dbt.tests.adapter.utils.fixture_date_spine import ( 19 | models__test_date_spine_yml, 20 | ) 21 | 22 | models__test_date_spine_sql = """ 23 | with generated_dates as ( 24 | {{ date_spine("day", "to_date('2023-09-01', 'YYYY-MM-DD')", "to_date('2023-09-10', 'YYYY-MM-DD')") }} 25 | ), expected_dates as ( 26 | select to_date('2023-09-01', 'YYYY-MM-DD') as expected from dual 27 | union all 28 | select to_date('2023-09-02', 'YYYY-MM-DD') as expected from dual 29 | union all 30 | select to_date('2023-09-03', 'YYYY-MM-DD') as expected from dual 31 | union all 32 | select to_date('2023-09-04', 'YYYY-MM-DD') as expected from dual 33 | union all 34 | select to_date('2023-09-05', 'YYYY-MM-DD') as expected from dual 35 | union all 36 | select to_date('2023-09-06', 'YYYY-MM-DD') as expected from dual 37 | union all 38 | select to_date('2023-09-07', 'YYYY-MM-DD') as expected from dual 39 | union all 40 | select to_date('2023-09-08', 'YYYY-MM-DD') as expected from dual 41 | union all 42 | select to_date('2023-09-09', 'YYYY-MM-DD') as expected from dual 43 | ) 44 | select 45 | generated_dates.date_day, 46 | expected_dates.expected 47 | from generated_dates 48 | left join expected_dates on generated_dates.date_day = expected_dates.expected 49 | """ 50 | 51 | 52 | class BaseDateSpine(BaseUtils): 53 | @pytest.fixture(scope="class") 54 | def models(self): 55 | return { 56 | "test_date_spine.yml": models__test_date_spine_yml, 57 | "test_date_spine.sql": self.interpolate_macro_namespace( 58 | models__test_date_spine_sql, "date_spine" 59 | ), 60 | } 61 | 62 | 63 | class TestDateSpine(BaseDateSpine): 64 | pass 65 | 66 | -------------------------------------------------------------------------------- /tests/functional/adapter/utils/test_generate_series.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2023, Oracle and/or its affiliates. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import pytest 17 | from dbt.tests.adapter.utils.base_utils import BaseUtils 18 | from dbt.tests.adapter.utils.fixture_generate_series import ( 19 | models__test_generate_series_sql, 20 | models__test_generate_series_yml, 21 | ) 22 | 23 | models__test_generate_series_sql = """ 24 | with generated_numbers as ( 25 | {{ dbt.generate_series(10) }} 26 | ), expected_numbers as ( 27 | select 1 as expected from dual 28 | union all 29 | select 2 as expected from dual 30 | union all 31 | select 3 as expected from dual 32 | union all 33 | select 4 as expected from dual 34 | union all 35 | select 5 as expected from dual 36 | union all 37 | select 6 as expected from dual 38 | union all 39 | select 7 as expected from dual 40 | union all 41 | select 8 as expected from dual 42 | union all 43 | select 9 as expected from dual 44 | union all 45 | select 10 as expected from dual 46 | ), joined as ( 47 | select 48 | generated_numbers.generated_number, 49 | expected_numbers.expected 50 | from generated_numbers 51 | left join expected_numbers on generated_numbers.generated_number = expected_numbers.expected 52 | ) 53 | 54 | SELECT * from joined 55 | """ 56 | 57 | class BaseGenerateSeries(BaseUtils): 58 | @pytest.fixture(scope="class") 59 | def models(self): 60 | return { 61 | "test_generate_series.yml": models__test_generate_series_yml, 62 | "test_generate_series.sql": self.interpolate_macro_namespace( 63 | models__test_generate_series_sql, "generate_series" 64 | ), 65 | } 66 | 67 | 68 | class TestGenerateSeries(BaseGenerateSeries): 69 | pass 70 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py3{9,10,11,12} 3 | 4 | [testenv] 5 | passenv = 6 | TNS_ADMIN 7 | DBT_ORACLE_USER 8 | DBT_ORACLE_HOST 9 | DBT_ORACLE_PROTOCOL 10 | DBT_ORACLE_PORT 11 | DBT_ORACLE_SERVICE 12 | DBT_ORACLE_PASSWORD 13 | DBT_ORACLE_DATABASE 14 | DBT_ORACLE_SCHEMA 15 | 16 | deps = 17 | -rrequirements.txt 18 | dbt-tests-adapter~=1.10,<1.11 19 | pytest 20 | 21 | commands = pytest 22 | --------------------------------------------------------------------------------