├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── BUG_REPORT.md │ ├── FEATURE_REQUEST.md │ ├── QUESTION.md │ └── config.yml ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── ci.yaml │ └── pythonpublish.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── flow-diagram.jpg ├── flow-diagram.svg ├── pylintrc ├── sample_logging.conf ├── setup.py ├── target_snowflake ├── __init__.py ├── db_sync.py ├── exceptions.py ├── file_format.py ├── file_formats │ ├── __init__.py │ ├── csv.py │ └── parquet.py ├── flattening.py ├── stream_utils.py └── upload_clients │ ├── __init__.py │ ├── base_upload_client.py │ ├── s3_upload_client.py │ └── snowflake_upload_client.py └── tests ├── __init__.py ├── integration ├── .env.sample ├── __init__.py ├── resources │ ├── invalid-json.json │ ├── invalid-message-order.json │ ├── messages-pg-logical-streams-no-records.json │ ├── messages-pg-logical-streams.json │ ├── messages-pg-with-invalid-records.json │ ├── messages-pg-with-valid-records.json │ ├── messages-simple-table.json │ ├── messages-with-binary-columns.json │ ├── messages-with-changing-pk.json │ ├── messages-with-falsy-pk-values.json │ ├── messages-with-invalid-records.json │ ├── messages-with-multi-schemas.json │ ├── messages-with-nested-schema.json │ ├── messages-with-new-pk.json │ ├── messages-with-non-db-friendly-columns.json │ ├── messages-with-null-pk.json │ ├── messages-with-reserved-name-as-table-name.json │ ├── messages-with-space-in-table-name.json │ ├── messages-with-three-streams-modified-column.json │ ├── messages-with-three-streams.json │ ├── messages-with-unexpected-types.json │ └── messages-with-unicode-characters.json ├── test_target_snowflake.py └── utils.py └── unit ├── __init__.py ├── file_formats ├── __init__.py ├── test_csv.py └── test_parquet.py ├── resources ├── logical-streams.json ├── messages-simple-table.json ├── same-schemas-multiple-times.json └── streams_only_state.json ├── test_db_sync.py ├── test_file_format.py ├── test_flattening.py ├── test_stream_utils.py └── test_target_snowflake.py /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @transferwise/analytics-platform 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/BUG_REPORT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/.github/ISSUE_TEMPLATE/BUG_REPORT.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/QUESTION.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/.github/ISSUE_TEMPLATE/QUESTION.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/.github/ISSUE_TEMPLATE/config.yml -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/.github/PULL_REQUEST_TEMPLATE.md -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/.github/workflows/ci.yaml -------------------------------------------------------------------------------- /.github/workflows/pythonpublish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/.github/workflows/pythonpublish.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/.gitignore -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/README.md -------------------------------------------------------------------------------- /flow-diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/flow-diagram.jpg -------------------------------------------------------------------------------- /flow-diagram.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/flow-diagram.svg -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/pylintrc -------------------------------------------------------------------------------- /sample_logging.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/sample_logging.conf -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/setup.py -------------------------------------------------------------------------------- /target_snowflake/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/__init__.py -------------------------------------------------------------------------------- /target_snowflake/db_sync.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/db_sync.py -------------------------------------------------------------------------------- /target_snowflake/exceptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/exceptions.py -------------------------------------------------------------------------------- /target_snowflake/file_format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/file_format.py -------------------------------------------------------------------------------- /target_snowflake/file_formats/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /target_snowflake/file_formats/csv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/file_formats/csv.py -------------------------------------------------------------------------------- /target_snowflake/file_formats/parquet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/file_formats/parquet.py -------------------------------------------------------------------------------- /target_snowflake/flattening.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/flattening.py -------------------------------------------------------------------------------- /target_snowflake/stream_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/stream_utils.py -------------------------------------------------------------------------------- /target_snowflake/upload_clients/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /target_snowflake/upload_clients/base_upload_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/upload_clients/base_upload_client.py -------------------------------------------------------------------------------- /target_snowflake/upload_clients/s3_upload_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/upload_clients/s3_upload_client.py -------------------------------------------------------------------------------- /target_snowflake/upload_clients/snowflake_upload_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/target_snowflake/upload_clients/snowflake_upload_client.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integration/.env.sample: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/.env.sample -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integration/resources/invalid-json.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/invalid-json.json -------------------------------------------------------------------------------- /tests/integration/resources/invalid-message-order.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/invalid-message-order.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-pg-logical-streams-no-records.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-pg-logical-streams-no-records.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-pg-logical-streams.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-pg-logical-streams.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-pg-with-invalid-records.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-pg-with-invalid-records.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-pg-with-valid-records.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-pg-with-valid-records.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-simple-table.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-simple-table.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-binary-columns.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-binary-columns.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-changing-pk.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-changing-pk.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-falsy-pk-values.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-falsy-pk-values.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-invalid-records.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-invalid-records.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-multi-schemas.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-multi-schemas.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-nested-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-nested-schema.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-new-pk.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-new-pk.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-non-db-friendly-columns.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-non-db-friendly-columns.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-null-pk.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-null-pk.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-reserved-name-as-table-name.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-reserved-name-as-table-name.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-space-in-table-name.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-space-in-table-name.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-three-streams-modified-column.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-three-streams-modified-column.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-three-streams.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-three-streams.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-unexpected-types.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-unexpected-types.json -------------------------------------------------------------------------------- /tests/integration/resources/messages-with-unicode-characters.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/resources/messages-with-unicode-characters.json -------------------------------------------------------------------------------- /tests/integration/test_target_snowflake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/test_target_snowflake.py -------------------------------------------------------------------------------- /tests/integration/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/integration/utils.py -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/file_formats/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/file_formats/test_csv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/file_formats/test_csv.py -------------------------------------------------------------------------------- /tests/unit/file_formats/test_parquet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/file_formats/test_parquet.py -------------------------------------------------------------------------------- /tests/unit/resources/logical-streams.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/resources/logical-streams.json -------------------------------------------------------------------------------- /tests/unit/resources/messages-simple-table.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/resources/messages-simple-table.json -------------------------------------------------------------------------------- /tests/unit/resources/same-schemas-multiple-times.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/resources/same-schemas-multiple-times.json -------------------------------------------------------------------------------- /tests/unit/resources/streams_only_state.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/resources/streams_only_state.json -------------------------------------------------------------------------------- /tests/unit/test_db_sync.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/test_db_sync.py -------------------------------------------------------------------------------- /tests/unit/test_file_format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/test_file_format.py -------------------------------------------------------------------------------- /tests/unit/test_flattening.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/test_flattening.py -------------------------------------------------------------------------------- /tests/unit/test_stream_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/test_stream_utils.py -------------------------------------------------------------------------------- /tests/unit/test_target_snowflake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise-target-snowflake/HEAD/tests/unit/test_target_snowflake.py --------------------------------------------------------------------------------