├── .github ├── CHANGELOG.md ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── codacy-analysis.yml │ ├── python-app.yml │ ├── semantic-release.yml │ └── terraform.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── _config.yml ├── cloud_config ├── buyers.json ├── generate_megalist_token.py └── scheduler_sample.json ├── documentation └── Megalista - Technical User Guide - EXTERNAL.pdf ├── generate_megalist_token.sh ├── megalist_dataflow ├── main.py ├── mappers │ ├── __init__.py │ ├── ads_ssd_hashing_mapper.py │ ├── ads_user_list_pii_hashing_mapper.py │ └── ads_user_list_pii_hashing_mapper_test.py ├── megalist_metadata ├── models │ ├── __init__.py │ ├── execution.py │ ├── oauth_credentials.py │ ├── oauth_credentials_test.py │ ├── options.py │ ├── options_test.py │ └── sheets_config.py ├── requirements.txt ├── setup.py ├── sources │ ├── __init__.py │ ├── base_bounded_source.py │ ├── batches_from_executions.py │ ├── firestore_execution_source.py │ └── spreadsheet_execution_source.py └── uploaders │ ├── __init__.py │ ├── appsflyer │ ├── __init__.py │ └── appsflyer_s2s_uploader_async.py │ ├── big_query │ ├── __init__.py │ ├── transactional_events_results_writer.py │ └── transactional_events_results_writer_test.py │ ├── campaign_manager │ ├── __init__.py │ ├── campaign_manager_conversion_uploader.py │ └── campaign_manager_conversion_uploader_test.py │ ├── google_ads │ ├── __init__.py │ ├── conversions │ │ ├── __init__.py │ │ ├── google_ads_offline_conversions_uploader.py │ │ ├── google_ads_offline_conversions_uploader_test.py │ │ ├── google_ads_ssd_uploader.py │ │ └── google_ads_ssd_uploader_test.py │ └── customer_match │ │ ├── __init__.py │ │ ├── abstract_uploader.py │ │ ├── contact_info_uploader.py │ │ ├── mobile_uploader.py │ │ └── user_id_uploader.py │ ├── google_analytics │ ├── __init__.py │ ├── google_analytics_4_measurement_protocol.py │ ├── google_analytics_4_measurement_protocol_test.py │ ├── google_analytics_data_import_eraser.py │ ├── google_analytics_data_import_eraser_test.py │ ├── google_analytics_data_import_uploader.py │ ├── google_analytics_data_import_uploader_test.py │ ├── google_analytics_measurement_protocol.py │ ├── google_analytics_user_list_uploader.py │ └── google_analytics_user_list_uploader_test.py │ └── utils.py ├── mypy.ini ├── pyproject.toml ├── run_cloud.sh ├── run_tests.sh ├── terraform ├── external.tf ├── main.tf ├── scripts │ └── deploy_cloud.sh └── variables.tf └── terraform_deploy.sh /.github/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.github/PULL_REQUEST_TEMPLATE.md -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/workflows/codacy-analysis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.github/workflows/codacy-analysis.yml -------------------------------------------------------------------------------- /.github/workflows/python-app.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.github/workflows/python-app.yml -------------------------------------------------------------------------------- /.github/workflows/semantic-release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.github/workflows/semantic-release.yml -------------------------------------------------------------------------------- /.github/workflows/terraform.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.github/workflows/terraform.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/.gitignore -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/README.md -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/_config.yml -------------------------------------------------------------------------------- /cloud_config/buyers.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/cloud_config/buyers.json -------------------------------------------------------------------------------- /cloud_config/generate_megalist_token.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/cloud_config/generate_megalist_token.py -------------------------------------------------------------------------------- /cloud_config/scheduler_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/cloud_config/scheduler_sample.json -------------------------------------------------------------------------------- /documentation/Megalista - Technical User Guide - EXTERNAL.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/documentation/Megalista - Technical User Guide - EXTERNAL.pdf -------------------------------------------------------------------------------- /generate_megalist_token.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/generate_megalist_token.sh -------------------------------------------------------------------------------- /megalist_dataflow/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/main.py -------------------------------------------------------------------------------- /megalist_dataflow/mappers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/mappers/__init__.py -------------------------------------------------------------------------------- /megalist_dataflow/mappers/ads_ssd_hashing_mapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/mappers/ads_ssd_hashing_mapper.py -------------------------------------------------------------------------------- /megalist_dataflow/mappers/ads_user_list_pii_hashing_mapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/mappers/ads_user_list_pii_hashing_mapper.py -------------------------------------------------------------------------------- /megalist_dataflow/mappers/ads_user_list_pii_hashing_mapper_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/mappers/ads_user_list_pii_hashing_mapper_test.py -------------------------------------------------------------------------------- /megalist_dataflow/megalist_metadata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/megalist_metadata -------------------------------------------------------------------------------- /megalist_dataflow/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/models/__init__.py -------------------------------------------------------------------------------- /megalist_dataflow/models/execution.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/models/execution.py -------------------------------------------------------------------------------- /megalist_dataflow/models/oauth_credentials.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/models/oauth_credentials.py -------------------------------------------------------------------------------- /megalist_dataflow/models/oauth_credentials_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/models/oauth_credentials_test.py -------------------------------------------------------------------------------- /megalist_dataflow/models/options.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/models/options.py -------------------------------------------------------------------------------- /megalist_dataflow/models/options_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/models/options_test.py -------------------------------------------------------------------------------- /megalist_dataflow/models/sheets_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/models/sheets_config.py -------------------------------------------------------------------------------- /megalist_dataflow/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/requirements.txt -------------------------------------------------------------------------------- /megalist_dataflow/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/setup.py -------------------------------------------------------------------------------- /megalist_dataflow/sources/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /megalist_dataflow/sources/base_bounded_source.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/sources/base_bounded_source.py -------------------------------------------------------------------------------- /megalist_dataflow/sources/batches_from_executions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/sources/batches_from_executions.py -------------------------------------------------------------------------------- /megalist_dataflow/sources/firestore_execution_source.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/sources/firestore_execution_source.py -------------------------------------------------------------------------------- /megalist_dataflow/sources/spreadsheet_execution_source.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/sources/spreadsheet_execution_source.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/__init__.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/appsflyer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/appsflyer/appsflyer_s2s_uploader_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/appsflyer/appsflyer_s2s_uploader_async.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/big_query/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/big_query/transactional_events_results_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/big_query/transactional_events_results_writer.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/big_query/transactional_events_results_writer_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/big_query/transactional_events_results_writer_test.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/campaign_manager/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/campaign_manager/campaign_manager_conversion_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/campaign_manager/campaign_manager_conversion_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/campaign_manager/campaign_manager_conversion_uploader_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/campaign_manager/campaign_manager_conversion_uploader_test.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/conversions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader_test.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader_test.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/customer_match/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/customer_match/__init__.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/customer_match/abstract_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/customer_match/abstract_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/customer_match/contact_info_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/customer_match/contact_info_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/customer_match/mobile_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/customer_match/mobile_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_ads/customer_match/user_id_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_ads/customer_match/user_id_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_4_measurement_protocol.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_4_measurement_protocol.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_4_measurement_protocol_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_4_measurement_protocol_test.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser_test.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_data_import_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_data_import_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_data_import_uploader_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_data_import_uploader_test.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_measurement_protocol.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_measurement_protocol.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_user_list_uploader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_user_list_uploader.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/google_analytics/google_analytics_user_list_uploader_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/google_analytics/google_analytics_user_list_uploader_test.py -------------------------------------------------------------------------------- /megalist_dataflow/uploaders/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/megalist_dataflow/uploaders/utils.py -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/mypy.ini -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/pyproject.toml -------------------------------------------------------------------------------- /run_cloud.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/run_cloud.sh -------------------------------------------------------------------------------- /run_tests.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/run_tests.sh -------------------------------------------------------------------------------- /terraform/external.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/terraform/external.tf -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/terraform/main.tf -------------------------------------------------------------------------------- /terraform/scripts/deploy_cloud.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/terraform/scripts/deploy_cloud.sh -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/terraform/variables.tf -------------------------------------------------------------------------------- /terraform_deploy.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DP6/marketing-data-sync/HEAD/terraform_deploy.sh --------------------------------------------------------------------------------