├── .env ├── .gitignore ├── CHECKLIST.md ├── INGESTION_MAIN_CONSIDERATIONS.md ├── MakeFile ├── README.md ├── contracts └── data_contracts_template.py ├── infrastructure ├── docker-compose.yml └── setup-quickelt-infra.sh ├── ingestion ├── pandas_templates │ ├── api_template.py │ ├── databases_template.py │ ├── s3_template.py │ ├── sharepoint_csv_template.py │ ├── sharepoint_xls_template.py │ └── web_scraping_template.py └── polars_templates │ ├── api_template.py │ ├── csv_template.py │ ├── databases_template.py │ ├── s3_template.py │ ├── sharepoint_xls_template.py │ └── web_scraping_template.py ├── requirements.txt ├── setup_env.py ├── tests ├── conftest.py ├── test_ingestion_databases_pandas_functions.py ├── test_ingestion_databases_polars_functions.py ├── test_ingestion_pandas.py └── test_ingestion_polars.py ├── transformation └── to_silver │ ├── cleaning_template_duckdb.py │ ├── cleaning_template_pandas.py │ └── cleaning_template_polars.py └── utils ├── logger.py ├── pydantic_validation_template_pandas.py ├── pydantic_validation_template_polars.py └── s3_utils.py /.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/.env -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/.gitignore -------------------------------------------------------------------------------- /CHECKLIST.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/CHECKLIST.md -------------------------------------------------------------------------------- /INGESTION_MAIN_CONSIDERATIONS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/INGESTION_MAIN_CONSIDERATIONS.md -------------------------------------------------------------------------------- /MakeFile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/MakeFile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/README.md -------------------------------------------------------------------------------- /contracts/data_contracts_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/contracts/data_contracts_template.py -------------------------------------------------------------------------------- /infrastructure/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/infrastructure/docker-compose.yml -------------------------------------------------------------------------------- /infrastructure/setup-quickelt-infra.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/infrastructure/setup-quickelt-infra.sh -------------------------------------------------------------------------------- /ingestion/pandas_templates/api_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/pandas_templates/api_template.py -------------------------------------------------------------------------------- /ingestion/pandas_templates/databases_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/pandas_templates/databases_template.py -------------------------------------------------------------------------------- /ingestion/pandas_templates/s3_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/pandas_templates/s3_template.py -------------------------------------------------------------------------------- /ingestion/pandas_templates/sharepoint_csv_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/pandas_templates/sharepoint_csv_template.py -------------------------------------------------------------------------------- /ingestion/pandas_templates/sharepoint_xls_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/pandas_templates/sharepoint_xls_template.py -------------------------------------------------------------------------------- /ingestion/pandas_templates/web_scraping_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/pandas_templates/web_scraping_template.py -------------------------------------------------------------------------------- /ingestion/polars_templates/api_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/polars_templates/api_template.py -------------------------------------------------------------------------------- /ingestion/polars_templates/csv_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/polars_templates/csv_template.py -------------------------------------------------------------------------------- /ingestion/polars_templates/databases_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/polars_templates/databases_template.py -------------------------------------------------------------------------------- /ingestion/polars_templates/s3_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/polars_templates/s3_template.py -------------------------------------------------------------------------------- /ingestion/polars_templates/sharepoint_xls_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/polars_templates/sharepoint_xls_template.py -------------------------------------------------------------------------------- /ingestion/polars_templates/web_scraping_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/ingestion/polars_templates/web_scraping_template.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/setup_env.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/test_ingestion_databases_pandas_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/tests/test_ingestion_databases_pandas_functions.py -------------------------------------------------------------------------------- /tests/test_ingestion_databases_polars_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/tests/test_ingestion_databases_polars_functions.py -------------------------------------------------------------------------------- /tests/test_ingestion_pandas.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/tests/test_ingestion_pandas.py -------------------------------------------------------------------------------- /tests/test_ingestion_polars.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/tests/test_ingestion_polars.py -------------------------------------------------------------------------------- /transformation/to_silver/cleaning_template_duckdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/transformation/to_silver/cleaning_template_duckdb.py -------------------------------------------------------------------------------- /transformation/to_silver/cleaning_template_pandas.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/transformation/to_silver/cleaning_template_pandas.py -------------------------------------------------------------------------------- /transformation/to_silver/cleaning_template_polars.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/utils/logger.py -------------------------------------------------------------------------------- /utils/pydantic_validation_template_pandas.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/utils/pydantic_validation_template_pandas.py -------------------------------------------------------------------------------- /utils/pydantic_validation_template_polars.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/utils/pydantic_validation_template_polars.py -------------------------------------------------------------------------------- /utils/s3_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpraes/quickelt/HEAD/utils/s3_utils.py --------------------------------------------------------------------------------