├── .DS_Store ├── .gitattributes ├── .gitignore ├── .plandex-dev └── projects-v2.json ├── CHANGELOG.md ├── RELEASE_NOTES.md ├── config ├── config.yaml └── tables_to_transfer.csv ├── license.txt ├── main.py ├── output └── .gitkeep ├── pytest.ini ├── readme.md ├── requirements.txt ├── setup.py ├── src ├── .DS_Store ├── __init__.py ├── config.py ├── data_ingestion_manager.py ├── data_sync.py ├── logger.py ├── schema_sync.py ├── source_setup.py ├── source_sql_generator.py ├── utils │ ├── __init__.py │ ├── logger.py │ ├── table_config.py │ └── type_conversions.py └── warehouses │ ├── __init__.py │ ├── abstract_warehouse.py │ ├── duckdb_warehouse.py │ ├── snowflake_warehouse.py │ ├── type_mappings.py │ └── warehouse_factory.py └── tests ├── __init__.py ├── config ├── config.py ├── snowflake_to_duckdb.yaml └── tables_to_transfer.csv ├── conftest.py ├── data_generators └── snowflake │ ├── __init__.py │ ├── snowflake_data_generator.py │ └── test_table_schemas │ ├── full_refresh_no_pk.csv │ ├── full_refresh_one_pk.csv │ ├── full_refresh_with_geo.csv │ ├── no_pk_append_only_stream.csv │ ├── no_pk_append_only_stream_with_geo.csv │ ├── no_pk_standard_stream.csv │ ├── one_pk_append_only_stream.csv │ ├── one_pk_standard_stream.csv │ ├── two_pk_append_only_stream.csv │ └── two_pk_standard_stream.csv ├── integration └── snowflake_to_duckdb.py ├── unit ├── test_config.py ├── test_data_ingestion_manager.py ├── test_source_sql_generator.py ├── test_table_config.py └── warehouses │ ├── config_files │ ├── test_minimal_snowflake_connect.toml │ └── test_snowflake_connect.toml │ ├── test_duckdb_warehouse.py │ ├── test_snowflake_warehouse.py │ └── test_type_mappings.py └── utils └── helpers.py /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/.DS_Store -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/.gitattributes -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/.gitignore -------------------------------------------------------------------------------- /.plandex-dev/projects-v2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/.plandex-dev/projects-v2.json -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /RELEASE_NOTES.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/RELEASE_NOTES.md -------------------------------------------------------------------------------- /config/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/config/config.yaml -------------------------------------------------------------------------------- /config/tables_to_transfer.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/config/tables_to_transfer.csv -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/license.txt -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/main.py -------------------------------------------------------------------------------- /output/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/pytest.ini -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/readme.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/setup.py -------------------------------------------------------------------------------- /src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/.DS_Store -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.0.0" -------------------------------------------------------------------------------- /src/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/config.py -------------------------------------------------------------------------------- /src/data_ingestion_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/data_ingestion_manager.py -------------------------------------------------------------------------------- /src/data_sync.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/data_sync.py -------------------------------------------------------------------------------- /src/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/logger.py -------------------------------------------------------------------------------- /src/schema_sync.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/schema_sync.py -------------------------------------------------------------------------------- /src/source_setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/source_setup.py -------------------------------------------------------------------------------- /src/source_sql_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/source_sql_generator.py -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/utils/logger.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/utils/table_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/utils/table_config.py -------------------------------------------------------------------------------- /src/utils/type_conversions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/utils/type_conversions.py -------------------------------------------------------------------------------- /src/warehouses/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/warehouses/abstract_warehouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/warehouses/abstract_warehouse.py -------------------------------------------------------------------------------- /src/warehouses/duckdb_warehouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/warehouses/duckdb_warehouse.py -------------------------------------------------------------------------------- /src/warehouses/snowflake_warehouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/warehouses/snowflake_warehouse.py -------------------------------------------------------------------------------- /src/warehouses/type_mappings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/warehouses/type_mappings.py -------------------------------------------------------------------------------- /src/warehouses/warehouse_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/src/warehouses/warehouse_factory.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/config/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/config/config.py -------------------------------------------------------------------------------- /tests/config/snowflake_to_duckdb.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/config/snowflake_to_duckdb.yaml -------------------------------------------------------------------------------- /tests/config/tables_to_transfer.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/config/tables_to_transfer.csv -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/data_generators/snowflake/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/data_generators/snowflake/snowflake_data_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/snowflake_data_generator.py -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/full_refresh_no_pk.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/full_refresh_no_pk.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/full_refresh_one_pk.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/full_refresh_one_pk.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/full_refresh_with_geo.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/full_refresh_with_geo.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/no_pk_append_only_stream.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/no_pk_append_only_stream.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/no_pk_append_only_stream_with_geo.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/no_pk_append_only_stream_with_geo.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/no_pk_standard_stream.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/no_pk_standard_stream.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/one_pk_append_only_stream.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/one_pk_append_only_stream.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/one_pk_standard_stream.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/one_pk_standard_stream.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/two_pk_append_only_stream.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/two_pk_append_only_stream.csv -------------------------------------------------------------------------------- /tests/data_generators/snowflake/test_table_schemas/two_pk_standard_stream.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/data_generators/snowflake/test_table_schemas/two_pk_standard_stream.csv -------------------------------------------------------------------------------- /tests/integration/snowflake_to_duckdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/integration/snowflake_to_duckdb.py -------------------------------------------------------------------------------- /tests/unit/test_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/test_config.py -------------------------------------------------------------------------------- /tests/unit/test_data_ingestion_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/test_data_ingestion_manager.py -------------------------------------------------------------------------------- /tests/unit/test_source_sql_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/test_source_sql_generator.py -------------------------------------------------------------------------------- /tests/unit/test_table_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/test_table_config.py -------------------------------------------------------------------------------- /tests/unit/warehouses/config_files/test_minimal_snowflake_connect.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/warehouses/config_files/test_minimal_snowflake_connect.toml -------------------------------------------------------------------------------- /tests/unit/warehouses/config_files/test_snowflake_connect.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/warehouses/config_files/test_snowflake_connect.toml -------------------------------------------------------------------------------- /tests/unit/warehouses/test_duckdb_warehouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/warehouses/test_duckdb_warehouse.py -------------------------------------------------------------------------------- /tests/unit/warehouses/test_snowflake_warehouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/warehouses/test_snowflake_warehouse.py -------------------------------------------------------------------------------- /tests/unit/warehouses/test_type_mappings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/unit/warehouses/test_type_mappings.py -------------------------------------------------------------------------------- /tests/utils/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanwithawhy/melchi/HEAD/tests/utils/helpers.py --------------------------------------------------------------------------------