├── .gitignore ├── .markdownlintrc ├── .scripts ├── README.md ├── e2e.sh ├── integration_test.sh ├── pr_check.sh ├── run_config.sh ├── run_playbooks.sh ├── run_test.sh └── templates │ ├── bigquery.yml.tmpl │ ├── redshift.yml.tmpl │ └── snowflake.yml.tmpl ├── .test ├── etc │ └── jupyter │ │ └── nbconfig │ │ └── notebook.d │ │ └── widgetsnbextension.json ├── great_expectations │ ├── config │ │ └── config_variables.yml │ ├── expectations │ │ ├── mobile │ │ │ └── v1 │ │ │ │ ├── mobile_base.json │ │ │ │ ├── mobile_base_redshift.json │ │ │ │ ├── mobile_metadata.json │ │ │ │ ├── mobile_screen_view_in_session_values.json │ │ │ │ ├── mobile_screen_views.json │ │ │ │ ├── mobile_sessions.json │ │ │ │ ├── mobile_staging_reconciliation.json │ │ │ │ └── mobile_users.json │ │ └── web │ │ │ └── v1 │ │ │ ├── base.json │ │ │ ├── base_redshift.json │ │ │ ├── integration_tests.json │ │ │ ├── metadata.json │ │ │ ├── page_view_in_session_values.json │ │ │ ├── page_views.json │ │ │ ├── sessions.json │ │ │ └── users.json │ ├── great_expectations.yml │ └── validation_configs │ │ ├── mobile │ │ └── v1 │ │ │ ├── bigquery │ │ │ ├── perm_tables.json │ │ │ └── temp_tables.json │ │ │ ├── redshift │ │ │ ├── perm_tables.json │ │ │ └── temp_tables.json │ │ │ └── snowflake │ │ │ ├── perm_tables.json │ │ │ └── temp_tables.json │ │ └── web │ │ └── v1 │ │ ├── bigquery │ │ ├── events_staged_integration_test_1.json │ │ ├── events_staged_integration_test_2.json │ │ ├── events_staged_integration_test_3.json │ │ ├── events_staged_integration_test_4.json │ │ ├── events_staged_integration_test_5.json │ │ ├── perm_integration_test_tables.json │ │ ├── perm_tables.json │ │ └── temp_tables.json │ │ ├── redshift │ │ ├── perm_tables.json │ │ └── temp_tables.json │ │ └── snowflake │ │ ├── perm_tables.json │ │ └── temp_tables.json ├── integration_tests │ └── web │ │ └── v1 │ │ └── bigquery_variables.yml.tmpl └── requirements.txt ├── CHANGELOG ├── LICENSE-2.0.txt ├── README.md ├── media ├── e2e.gif ├── mobile_full_model_structure.jpg ├── snowplow_architecture.png ├── snowplow_logo.png ├── web_full_model_structure.jpg └── web_model_module.jpg ├── mobile └── v1 │ ├── README.md │ ├── bigquery │ ├── CHANGELOG │ ├── README.md │ └── sql-runner │ │ ├── configs │ │ ├── datamodeling.json │ │ ├── datamodeling_custom_module.json │ │ ├── migrations.json │ │ ├── post_test.json │ │ ├── pre_test.json │ │ └── teardown_all.json │ │ ├── playbooks │ │ ├── custom │ │ │ ├── 04-session-goals │ │ │ │ ├── 01-session-goals-main.yml.tmpl │ │ │ │ └── 99-session-goals-complete.yml.tmpl │ │ │ └── README.md │ │ ├── standard │ │ │ ├── 00-setup │ │ │ │ ├── 00-setup-metadata.yml.tmpl │ │ │ │ ├── 99-metadata-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-metadata.yml.tmpl │ │ │ ├── 01-base │ │ │ │ ├── 01-base-main.yml.tmpl │ │ │ │ ├── 99-base-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-base.yml.tmpl │ │ │ ├── 02-screen-views │ │ │ │ ├── 01-screen-views-main.yml.tmpl │ │ │ │ ├── 99-screen-views-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-screen-views.yml.tmpl │ │ │ ├── 03-optional-modules │ │ │ │ └── 01-app-errors │ │ │ │ │ ├── 01-app-errors-main.yml.tmpl │ │ │ │ │ ├── 99-app-errors-complete.yml.tmpl │ │ │ │ │ ├── README.md │ │ │ │ │ └── XX-destroy-app-errors.yml.tmpl │ │ │ ├── 04-sessions │ │ │ │ ├── 01-sessions-main.yml.tmpl │ │ │ │ ├── 99-sessions-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-sessions.yml.tmpl │ │ │ ├── 05-users │ │ │ │ ├── 01-users-main.yml.tmpl │ │ │ │ ├── 99-users-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-users.yml.tmpl │ │ │ └── 99-migrations │ │ │ │ └── 1.1.0-migration.yml.tmpl │ │ └── tests │ │ │ └── 00-staging-reconciliation │ │ │ ├── 01-staging-reconciliation-main.yml.tmpl │ │ │ ├── 99-staging-reconciliation-complete.yml.tmpl │ │ │ └── README.md │ │ └── sql │ │ ├── custom │ │ ├── 04-session-goals │ │ │ ├── 01-session-goals-staged.sql │ │ │ ├── 02-session-goals-upsert.sql │ │ │ └── 99-session-goals-cleanup.sql │ │ └── README.md │ │ ├── standard │ │ ├── 00-setup │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-metadata.sql │ │ │ │ └── 01-functions-and-procedures.sql │ │ │ ├── 99-complete │ │ │ │ └── 01-cleanup-metadata.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-metadata.sql │ │ ├── 01-base │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-base.sql │ │ │ │ ├── 01-new-events-limits.sql │ │ │ │ ├── 02-run-manifest.sql │ │ │ │ ├── 03-sessions-to-process.sql │ │ │ │ ├── 04-sessions-to-include.sql │ │ │ │ ├── 05-batch-limits.sql │ │ │ │ ├── 06-events-this-run.sql │ │ │ │ ├── 07-base-metadata.sql │ │ │ │ └── 08-commit-base.sql │ │ │ ├── 99-complete │ │ │ │ ├── 98-base-manifest.sql │ │ │ │ └── 99-base-cleanup.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-base.sql │ │ ├── 02-screen-views │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-screen-views.sql │ │ │ │ ├── 01-screen-views.sql │ │ │ │ ├── 02-screen-views-metadata.sql │ │ │ │ └── 03-commit-screen-views.sql │ │ │ ├── 99-complete │ │ │ │ └── 99-screen-views-cleanup.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-screen-views.sql │ │ ├── 03-optional-modules │ │ │ └── 01-app-errors │ │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-app-errors.sql │ │ │ │ ├── 01-app-errors.sql │ │ │ │ ├── 02-app-errors-metadata.sql │ │ │ │ └── 03-commit-app-errors.sql │ │ │ │ ├── 99-complete │ │ │ │ └── 99-app-errors-cleanup.sql │ │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-app-errors.sql │ │ ├── 04-sessions │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-sessions.sql │ │ │ │ ├── 01-sessions-aggs.sql │ │ │ │ ├── 02-sessions-sv-details.sql │ │ │ │ ├── 03-sessions.sql │ │ │ │ ├── 04-sessions-metadata.sql │ │ │ │ ├── 05-sessions-prep-manifest.sql │ │ │ │ └── 06-commit-sessions.sql │ │ │ ├── 99-complete │ │ │ │ ├── 98-truncate-upstream-staged.sql │ │ │ │ └── 99-sessions-cleanup.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-sessions.sql │ │ ├── 05-users │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-users.sql │ │ │ │ ├── 01-userids-this-run.sql │ │ │ │ ├── 02-users-limits.sql │ │ │ │ ├── 03-users-sessions-this-run.sql │ │ │ │ ├── 04-users-aggs.sql │ │ │ │ ├── 05-users-lasts.sql │ │ │ │ ├── 06-users.sql │ │ │ │ ├── 07-users-metadata.sql │ │ │ │ └── 08-commit-users.sql │ │ │ ├── 99-complete │ │ │ │ ├── 98-manifest-and-truncate.sql │ │ │ │ └── 99-users-cleanup.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-users.sql │ │ └── 99-migrations │ │ │ └── 1.1.0-migration │ │ │ └── mobile-events-staged.sql │ │ └── tests │ │ └── 00-staging-reconciliation │ │ ├── 01-main │ │ └── 00-staging-reconciliation.sql │ │ └── 99-complete │ │ └── 99-staging-reconciliation-cleanup.sql │ ├── redshift │ ├── CHANGELOG │ ├── README.md │ └── sql-runner │ │ ├── configs │ │ ├── datamodeling.json │ │ ├── datamodeling_custom_module.json │ │ ├── post_test.json │ │ ├── pre_test.json │ │ └── teardown_all.json │ │ ├── playbooks │ │ ├── custom │ │ │ ├── 04-session-goals │ │ │ │ ├── 01-session-goals-main.yml.tmpl │ │ │ │ └── 99-session-goals-complete.yml.tmpl │ │ │ └── README.md │ │ ├── standard │ │ │ ├── 00-setup │ │ │ │ ├── 00-setup-metadata.yml.tmpl │ │ │ │ ├── 99-metadata-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-metadata.yml.tmpl │ │ │ ├── 01-base │ │ │ │ ├── 01-base-main.yml.tmpl │ │ │ │ ├── 99-base-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-base.yml.tmpl │ │ │ ├── 02-screen-views │ │ │ │ ├── 01-screen-views-main.yml.tmpl │ │ │ │ ├── 99-screen-views-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-screen-views.yml.tmpl │ │ │ ├── 03-optional-modules │ │ │ │ └── 01-app-errors │ │ │ │ │ ├── 01-app-errors-main.yml.tmpl │ │ │ │ │ ├── 99-app-errors-complete.yml.tmpl │ │ │ │ │ ├── README.md │ │ │ │ │ └── XX-destroy-app-errors.yml.tmpl │ │ │ ├── 04-sessions │ │ │ │ ├── 01-sessions-main.yml.tmpl │ │ │ │ ├── 99-sessions-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-sessions.yml.tmpl │ │ │ └── 05-users │ │ │ │ ├── 01-users-main.yml.tmpl │ │ │ │ ├── 99-users-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-users.yml.tmpl │ │ └── tests │ │ │ └── 00-staging-reconciliation │ │ │ ├── 01-staging-reconciliation-main.yml.tmpl │ │ │ ├── 99-staging-reconciliation-complete.yml.tmpl │ │ │ └── README.md │ │ └── sql │ │ ├── custom │ │ ├── 04-session-goals │ │ │ ├── 01-session-goals-setup.sql │ │ │ ├── 02-session-goals-staged.sql │ │ │ ├── 03-session-goals-upsert.sql │ │ │ └── 99-session-goals-cleanup.sql │ │ └── README.md │ │ ├── standard │ │ ├── 00-setup │ │ │ ├── 01-main │ │ │ │ └── 00-setup-metadata.sql │ │ │ ├── 99-complete │ │ │ │ └── 01-cleanup-metadata.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-metadata.sql │ │ ├── 01-base │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-base.sql │ │ │ │ ├── 01-new-events-limits.sql │ │ │ │ ├── 02-run-manifest.sql │ │ │ │ ├── 03-sessions-to-process.sql │ │ │ │ ├── 04-sessions-to-include.sql │ │ │ │ ├── 05-batch-limits.sql │ │ │ │ ├── 06-events-to-process.sql │ │ │ │ ├── 07-duplicates.sql │ │ │ │ ├── 08a-page-context.sql │ │ │ │ ├── 08b-mobile-context.sql │ │ │ │ ├── 08c-geolocation-context.sql │ │ │ │ ├── 08d-application-context.sql │ │ │ │ ├── 08e-screen-context.sql │ │ │ │ ├── 09-events-this-run.sql │ │ │ │ ├── 10-metadata.sql │ │ │ │ └── 11-commit-base.sql │ │ │ ├── 99-complete │ │ │ │ ├── 98-base-manifest.sql │ │ │ │ └── 99-base-cleanup.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-base.sql │ │ ├── 02-screen-views │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-screen-views.sql │ │ │ │ ├── 01-screen-view-limits.sql │ │ │ │ ├── 02-screen-view-ids.sql │ │ │ │ ├── 03-screen-view-events.sql │ │ │ │ ├── 04-screen-views.sql │ │ │ │ ├── 05-screen-views-metadata.sql │ │ │ │ └── 06-commit-screen-views.sql │ │ │ ├── 99-complete │ │ │ │ └── 99-screen-views-cleanup.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-screen-views.sql │ │ ├── 03-optional-modules │ │ │ └── 01-app-errors │ │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-app-errors.sql │ │ │ │ ├── 01-app-errors-limits.sql │ │ │ │ ├── 02-app-errors-events.sql │ │ │ │ ├── 03-app-errors.sql │ │ │ │ ├── 04-app-errors-metadata.sql │ │ │ │ └── 05-commit-app-errors.sql │ │ │ │ ├── 99-complete │ │ │ │ └── 99-app-errors-cleanup.sql │ │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-app-errors.sql │ │ ├── 04-sessions │ │ │ ├── 01-main │ │ │ │ ├── 00-setup-sessions.sql │ │ │ │ ├── 01-sessions-aggs.sql │ │ │ │ ├── 02-sessions-sv-details.sql │ │ │ │ ├── 03-sessions.sql │ │ │ │ ├── 04-sessions-metadata.sql │ │ │ │ ├── 05-sessions-prep-manifest.sql │ │ │ │ └── 06-commit-sessions.sql │ │ │ ├── 99-complete │ │ │ │ ├── 98-truncate-upstream-staged.sql │ │ │ │ └── 99-sessions-cleanup.sql │ │ │ └── XX-destroy │ │ │ │ └── XX-destroy-sessions.sql │ │ └── 05-users │ │ │ ├── 01-main │ │ │ ├── 00-setup-users.sql │ │ │ ├── 01-userids-this-run.sql │ │ │ ├── 02-users-limits.sql │ │ │ ├── 03-users-sessions-this-run.sql │ │ │ ├── 04-users-aggs.sql │ │ │ ├── 05-users-lasts.sql │ │ │ ├── 06-users.sql │ │ │ ├── 07-users-metadata.sql │ │ │ └── 08-commit-users.sql │ │ │ ├── 99-complete │ │ │ ├── 98-manifest-and-truncate.sql │ │ │ └── 99-users-cleanup.sql │ │ │ └── XX-destroy │ │ │ └── XX-destroy-users.sql │ │ └── tests │ │ └── 00-staging-reconciliation │ │ ├── 01-main │ │ └── 00-staging-reconciliation.sql │ │ ├── 99-complete │ │ └── 99-staging-reconciliation-cleanup.sql │ │ └── XX-destroy │ │ └── XX-destroy-staging-reconciliation.sql │ └── snowflake │ ├── CHANGELOG │ ├── README.md │ └── sql-runner │ ├── configs │ ├── datamodeling.json │ ├── datamodeling_custom_module.json │ ├── migrations.json │ ├── post_test.json │ ├── pre_test.json │ └── teardown_all.json │ ├── playbooks │ ├── custom │ │ └── 04-session-goals │ │ │ ├── 01-session-goals-main.yml.tmpl │ │ │ ├── 99-session-goals-complete.yml.tmpl │ │ │ └── README.md │ ├── standard │ │ ├── 00-setup │ │ │ ├── 00-setup-metadata.yml.tmpl │ │ │ ├── 99-metadata-complete.yml.tmpl │ │ │ ├── README.md │ │ │ └── XX-destroy-metadata.yml.tmpl │ │ ├── 01-base │ │ │ ├── 01-base-main.yml.tmpl │ │ │ ├── 99-base-complete.yml.tmpl │ │ │ ├── README.md │ │ │ └── XX-destroy-base.yml.tmpl │ │ ├── 02-screen-views │ │ │ ├── 01-screen-views-main.yml.tmpl │ │ │ ├── 99-screen-views-complete.yml.tmpl │ │ │ ├── README.md │ │ │ └── XX-destroy-screen-views.yml.tmpl │ │ ├── 03-optional-modules │ │ │ └── 01-app-errors │ │ │ │ ├── 01-app-errors-main.yml.tmpl │ │ │ │ ├── 99-app-errors-complete.yml.tmpl │ │ │ │ ├── README.md │ │ │ │ └── XX-destroy-app-errors.yml.tmpl │ │ ├── 04-sessions │ │ │ ├── 01-sessions-main.yml.tmpl │ │ │ ├── 99-sessions-complete.yml.tmpl │ │ │ ├── README.md │ │ │ └── XX-destroy-sessions.yml.tmpl │ │ ├── 05-users │ │ │ ├── 01-users-main.yml.tmpl │ │ │ ├── 99-users-complete.yml.tmpl │ │ │ ├── README.md │ │ │ └── XX-destroy-users.yml.tmpl │ │ └── 99-migrations │ │ │ ├── 1.1.0-migration.yml.tmpl │ │ │ └── 1.1.1-migration.yml.tmpl │ └── tests │ │ └── 00-staging-reconciliation │ │ ├── 01-staging-reconciliation-main.yml.tmpl │ │ ├── 99-staging-reconciliation-complete.yml.tmpl │ │ └── README.md │ └── sql │ ├── custom │ ├── 04-session-goals │ │ ├── 01-session-goals-staged.sql │ │ ├── 02-session-goals-upsert.sql │ │ └── 99-session-goals-cleanup.sql │ └── README.md │ ├── standard │ ├── 00-setup │ │ ├── 01-main │ │ │ ├── 00-setup-metadata.sql │ │ │ └── 01-stored-procedures.sql │ │ ├── 99-complete │ │ │ └── 99-metadata-cleanup.sql │ │ ├── StoredProcedures.md │ │ └── XX-destroy │ │ │ └── XX-destroy-metadata.sql │ ├── 01-base │ │ ├── 01-main │ │ │ ├── 00-setup-base.sql │ │ │ ├── 01-new-events-limits.sql │ │ │ ├── 02-run-manifest.sql │ │ │ ├── 03-sessions-to-process.sql │ │ │ ├── 04-sessions-to-include.sql │ │ │ ├── 05-batch-limits.sql │ │ │ ├── 06-events-this-run.sql │ │ │ ├── 07-metadata.sql │ │ │ └── 08-commit-base.sql │ │ ├── 99-complete │ │ │ ├── 98-base-manifest.sql │ │ │ └── 99-base-cleanup.sql │ │ └── XX-destroy │ │ │ └── XX-destroy-base.sql │ ├── 02-screen-views │ │ ├── 01-main │ │ │ ├── 00-setup-screen-views.sql │ │ │ ├── 01-screen-views.sql │ │ │ ├── 02-screen-views-metadata.sql │ │ │ └── 03-commit-screen-views.sql │ │ ├── 99-complete │ │ │ └── 99-screen-views-cleanup.sql │ │ └── XX-destroy │ │ │ └── XX-destroy-screen-views.sql │ ├── 03-optional-modules │ │ └── 01-app-errors │ │ │ ├── 01-main │ │ │ ├── 00-setup-app-errors.sql │ │ │ ├── 01-app-errors.sql │ │ │ ├── 02-app-errors-metadata.sql │ │ │ └── 03-commit-app-errors.sql │ │ │ ├── 99-complete │ │ │ └── 99-app-errors-cleanup.sql │ │ │ └── XX-destroy │ │ │ └── XX-destroy-app-errors.sql │ ├── 04-sessions │ │ ├── 01-main │ │ │ ├── 00-setup-sessions.sql │ │ │ ├── 01-sessions-aggs.sql │ │ │ ├── 02-sessions-sv-details.sql │ │ │ ├── 03-sessions.sql │ │ │ ├── 04-sessions-metadata.sql │ │ │ ├── 05-sessions-prep-manifest.sql │ │ │ └── 06-commit-sessions.sql │ │ ├── 99-complete │ │ │ ├── 98-truncate-upstream-staged.sql │ │ │ └── 99-sessions-cleanup.sql │ │ └── XX-destroy │ │ │ └── XX-destroy-sessions.sql │ ├── 05-users │ │ ├── 01-main │ │ │ ├── 00-setup-users.sql │ │ │ ├── 01-userids-this-run.sql │ │ │ ├── 02-users-limits.sql │ │ │ ├── 03-users-sessions-this-run.sql │ │ │ ├── 04-users-aggs.sql │ │ │ ├── 05-users-lasts.sql │ │ │ ├── 06-users.sql │ │ │ ├── 07-users-metadata.sql │ │ │ └── 08-commit-users.sql │ │ ├── 99-complete │ │ │ ├── 98-manifest-and-truncate.sql │ │ │ └── 99-users-cleanup.sql │ │ └── XX-destroy │ │ │ └── XX-destroy-users.sql │ └── 99-migrations │ │ ├── 1.1.0-migration │ │ └── mobile-events-staged.sql │ │ └── 1.1.1-migration │ │ └── alter-tables.sql │ └── tests │ └── 00-staging-reconciliation │ ├── 01-main │ └── 00-staging-reconciliation.sql │ └── 99-complete │ └── 99-staging-reconciliation-cleanup.sql └── web └── v1 ├── README.md ├── bigquery ├── CHANGELOG ├── README.md └── sql-runner │ ├── configs │ ├── datamodeling.json │ ├── example_with_custom.json │ ├── post_test.json │ ├── pre_test.json │ └── teardown_all.json │ ├── playbooks │ ├── custom │ │ ├── 02-custom-example │ │ │ └── 01-custom-example.yml.tmpl │ │ └── README.md │ └── standard │ │ ├── 00-setup │ │ ├── 00-setup-metadata.yml.tmpl │ │ ├── 99-metadata-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-metadata.yml.tmpl │ │ ├── 01-base │ │ ├── 01-base-main.yml.tmpl │ │ ├── 99-base-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-base.yml.tmpl │ │ ├── 02-page-views │ │ ├── 01-page-views-main.yml.tmpl │ │ ├── 99-page-views-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-page-views.yml.tmpl │ │ ├── 03-sessions │ │ ├── 01-sessions-main.yml.tmpl │ │ ├── 99-sessions-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-sessions.yml.tmpl │ │ └── 04-users │ │ ├── 01-users-main.yml.tmpl │ │ ├── 99-users-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-users.yml.tmpl │ └── sql │ ├── custom │ ├── 02-custom-example │ │ ├── 01-link-clicks.sql │ │ ├── 02-channel-engagement.sql │ │ ├── 03-commit.sql │ │ └── 99-custom-example-cleanup.sql │ └── README.md │ └── standard │ ├── 00-setup │ ├── 01-main │ │ ├── 00-setup-metadata.sql │ │ └── 01-functions-and-procedures.sql │ ├── 99-complete │ │ └── 01-cleanup-metadata.sql │ └── XX-destroy │ │ └── XX-destroy-metadata.sql │ ├── 01-base │ ├── 01-main │ │ ├── 00-setup-base.sql │ │ ├── 01-new-events-limits.sql │ │ ├── 02-run-manifest.sql │ │ ├── 03-sessions-to-process.sql │ │ ├── 04-sessions-to-include.sql │ │ ├── 05-batch-limits.sql │ │ ├── 06-events-this-run.sql │ │ ├── 07-base-metadata.sql │ │ └── 08-commit-base.sql │ ├── 99-complete │ │ ├── 98-base-manifest.sql │ │ └── 99-base-cleanup.sql │ └── XX-destroy │ │ └── XX-destroy-base.sql │ ├── 02-page-views │ ├── 01-main │ │ ├── 00-setup-page-views.sql │ │ ├── 01-page-view-events.sql │ │ ├── 02-engaged-time.sql │ │ ├── 03-scroll-depth.sql │ │ ├── 04-contexts.sql │ │ ├── 05-page-views.sql │ │ ├── 06-page-views-metadata.sql │ │ └── 07-commit-page-views.sql │ ├── 99-complete │ │ ├── 98-truncate-base-staged.sql │ │ └── 99-page-views-cleanup.sql │ └── XX-destroy │ │ └── XX-destroy-page-views.sql │ ├── 03-sessions │ ├── 01-main │ │ ├── 00-setup-sessions.sql │ │ ├── 01-sessions-aggs.sql │ │ ├── 02-sessions-lasts.sql │ │ ├── 03-sessions.sql │ │ ├── 04-sessions-metadata.sql │ │ ├── 05-sessions-prep-manifest.sql │ │ └── 06-commit-sessions.sql │ ├── 99-complete │ │ ├── 98-truncate-page-views-staged.sql │ │ └── 99-sessions-cleanup.sql │ └── XX-destroy │ │ └── XX-destroy-sessions.sql │ └── 04-users │ ├── 01-main │ ├── 00-setup-users.sql │ ├── 01-userids-this-run.sql │ ├── 02-users-limits.sql │ ├── 03-users-sessions-this-run.sql │ ├── 04-users-aggs.sql │ ├── 05-users-lasts.sql │ ├── 06-users.sql │ ├── 07-users-metadata.sql │ └── 08-commit-users.sql │ ├── 99-complete │ ├── 98-manifest-and-truncate.sql │ └── 99-users-cleanup.sql │ └── XX-destroy │ └── XX-destroy-users.sql ├── redshift ├── CHANGELOG ├── README.md └── sql-runner │ ├── configs │ ├── datamodeling.json │ ├── example_with_custom.json │ ├── migrations.json │ ├── post_test.json │ ├── pre_test.json │ └── teardown_all.json │ ├── playbooks │ ├── custom │ │ ├── 02-page-views-join │ │ │ ├── 01-page-views-join.yml.tmpl │ │ │ └── 99-page-views-join-complete.yml.tmpl │ │ └── README.md │ └── standard │ │ ├── 00-setup │ │ ├── 00-setup-metadata.yml.tmpl │ │ ├── 99-metadata-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-metadata.yml.tmpl │ │ ├── 01-base │ │ ├── 01-base-main.yml.tmpl │ │ ├── 99-base-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-base.yml.tmpl │ │ ├── 02-page-views │ │ ├── 01-page-views-main.yml.tmpl │ │ ├── 99-page-views-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-page-views.yml.tmpl │ │ ├── 03-sessions │ │ ├── 01-sessions-main.yml.tmpl │ │ ├── 99-sessions-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-sessions.yml.tmpl │ │ ├── 04-users │ │ ├── 01-users-main.yml.tmpl │ │ ├── 99-users-complete.yml.tmpl │ │ ├── README.md │ │ └── XX-destroy-users.yml.tmpl │ │ └── 99-migrations │ │ ├── 1.2.0-migration.yml.tmpl │ │ └── 1.3.0-migration.yml.tmpl │ └── sql │ ├── custom │ ├── 02-page-views-join │ │ ├── 01-page-views-join-setup.sql │ │ ├── 02-page-views-join-staged.sql │ │ ├── 03-page-views-join-upsert.sql │ │ └── 99-page-views-join-cleanup.sql │ └── README.md │ └── standard │ ├── 00-setup │ ├── 01-main │ │ └── 00-setup-metadata.sql │ ├── 99-complete │ │ └── 01-cleanup-metadata.sql │ └── XX-destroy │ │ └── XX-destroy-metadata.sql │ ├── 01-base │ ├── 01-main │ │ ├── 00-setup-base.sql │ │ ├── 01-new-events-limits.sql │ │ ├── 02-run-manifest.sql │ │ ├── 03-sessions-to-process.sql │ │ ├── 04-sessions-to-include.sql │ │ ├── 05-batch-limits.sql │ │ ├── 06-events-this-run.sql │ │ ├── 07-duplicates.sql │ │ ├── 08-add-page-view-id.sql │ │ ├── 09-metadata.sql │ │ └── 10-commit-base.sql │ ├── 99-complete │ │ ├── 98-base-manifest.sql │ │ └── 99-base-cleanup.sql │ └── XX-destroy │ │ └── XX-destroy-base.sql │ ├── 02-page-views │ ├── 01-main │ │ ├── 00-setup-page-views.sql │ │ ├── 01-page-view-limits.sql │ │ ├── 02-page-view-events.sql │ │ ├── 03-page-view-duplicates.sql │ │ ├── 04-engaged-time.sql │ │ ├── 05-scroll-depth.sql │ │ ├── 06-a-iab.sql │ │ ├── 06-b-ua-parser.sql │ │ ├── 06-c-yauaa.sql │ │ ├── 07-page-views.sql │ │ ├── 08-page-views-metadata.sql │ │ └── 09-commit-page-views.sql │ ├── 99-complete │ │ ├── 98-drop-base-staged.sql │ │ └── 99-page-views-cleanup.sql │ └── XX-destroy │ │ └── XX-destroy-page-views.sql │ ├── 03-sessions │ ├── 01-main │ │ ├── 00-setup-sessions.sql │ │ ├── 01-sessions-aggs.sql │ │ ├── 02-sessions-lasts.sql │ │ ├── 03-sessions.sql │ │ ├── 04-sessions-metadata.sql │ │ ├── 05-sessions-prep-manifest.sql │ │ └── 06-commit-sessions.sql │ ├── 99-complete │ │ ├── 98-truncate-page-views-staged.sql │ │ └── 99-sessions-cleanup.sql │ └── XX-destroy │ │ └── XX-destroy-sessions.sql │ ├── 04-users │ ├── 01-main │ │ ├── 00-setup-users.sql │ │ ├── 01-userids-this-run.sql │ │ ├── 02-users-limits.sql │ │ ├── 03-users-sessions-this-run.sql │ │ ├── 04-users-aggs.sql │ │ ├── 05-users-lasts.sql │ │ ├── 06-users.sql │ │ ├── 07-users-metadata.sql │ │ └── 08-commit-users.sql │ ├── 99-complete │ │ ├── 98-manifest-and-truncate.sql │ │ └── 99-users-cleanup.sql │ └── XX-destroy │ │ └── XX-destroy-users.sql │ └── 99-migrations │ ├── 1.2.0-migration │ └── events-staged.sql │ └── 1.3.0-migration │ ├── base-session-id-manifest.sql │ ├── sessions-userid-manifest.sql │ ├── sortkey-encoding.sql │ └── users-manifest.sql └── snowflake ├── CHANGELOG ├── README.md └── sql-runner ├── configs ├── datamodeling.json ├── example_with_custom.json ├── migrations.json ├── post_test.json ├── pre_test.json └── teardown_all.json ├── playbooks ├── custom │ ├── 02-page-views-join │ │ ├── 01-page-views-join.yml.tmpl │ │ ├── 99-page-views-join-complete.yml.tmpl │ │ └── XX-destroy-page-views-join.yml.tmpl │ └── README.md └── standard │ ├── 00-setup │ ├── 00-setup-metadata.yml.tmpl │ ├── 99-metadata-complete.yml.tmpl │ ├── README.md │ └── XX-destroy-metadata.yml.tmpl │ ├── 01-base │ ├── 01-base-main.yml.tmpl │ ├── 99-base-complete.yml.tmpl │ ├── README.md │ └── XX-destroy-base.yml.tmpl │ ├── 02-page-views │ ├── 01-page-views-main.yml.tmpl │ ├── 99-page-views-complete.yml.tmpl │ ├── README.md │ └── XX-destroy-page-views.yml.tmpl │ ├── 03-sessions │ ├── 01-sessions-main.yml.tmpl │ ├── 99-sessions-complete.yml.tmpl │ ├── README.md │ └── XX-destroy-sessions.yml.tmpl │ ├── 04-users │ ├── 01-users-main.yml.tmpl │ ├── 99-users-complete.yml.tmpl │ ├── README.md │ └── XX-destroy-users.yml.tmpl │ └── 99-migrations │ ├── 1.0.1-migration.yml.tmpl │ └── 1.0.3-migration.yml.tmpl └── sql ├── custom ├── 02-page-views-join │ ├── 01-main │ │ ├── 00-setup-page-views-join.sql │ │ ├── 01-page-views-join-staged.sql │ │ └── 02-page-views-join-upsert.sql │ ├── 99-complete │ │ └── 99-page-views-join-cleanup.sql │ └── XX-destroy │ │ └── XX-destroy-page-views-join.sql └── README.md └── standard ├── 00-setup ├── 01-main │ ├── 00-setup-metadata.sql │ └── 01-stored-procedures.sql ├── 99-complete │ └── 99-metadata-cleanup.sql ├── StoredProcedures.md └── XX-destroy │ └── XX-destroy-metadata.sql ├── 01-base ├── 01-main │ ├── 00-setup-base.sql │ ├── 01-new-events-limits.sql │ ├── 02-run-manifest.sql │ ├── 03-sessions-to-process.sql │ ├── 04-sessions-to-include.sql │ ├── 05-batch-limits.sql │ ├── 06-events-this-run.sql │ ├── 07-duplicates.sql │ ├── 08-metadata.sql │ └── 09-commit-base.sql ├── 99-complete │ ├── 98-base-manifest.sql │ └── 99-base-cleanup.sql └── XX-destroy │ └── XX-destroy-base.sql ├── 02-page-views ├── 01-main │ ├── 00-setup-page-views.sql │ ├── 01-page-view-events.sql │ ├── 02-page-view-duplicates.sql │ ├── 03-engaged-time.sql │ ├── 04-scroll-depth.sql │ ├── 05-page-views.sql │ ├── 06-page-views-metadata.sql │ └── 07-commit-page-views.sql ├── 99-complete │ ├── 98-truncate-base-staged.sql │ └── 99-page-views-cleanup.sql └── XX-destroy │ └── XX-destroy-page-views.sql ├── 03-sessions ├── 01-main │ ├── 00-setup-sessions.sql │ ├── 01-sessions-aggs.sql │ ├── 02-sessions-lasts.sql │ ├── 03-sessions.sql │ ├── 04-sessions-metadata.sql │ ├── 05-sessions-prep-manifest.sql │ └── 06-commit-sessions.sql ├── 99-complete │ ├── 98-truncate-page-views-staged.sql │ └── 99-sessions-cleanup.sql └── XX-destroy │ └── XX-destroy-sessions.sql ├── 04-users ├── 01-main │ ├── 00-setup-users.sql │ ├── 01-userids-this-run.sql │ ├── 02-users-limits.sql │ ├── 03-users-sessions-this-run.sql │ ├── 04-users-aggs.sql │ ├── 05-users-lasts.sql │ ├── 06-users.sql │ ├── 07-users-metadata.sql │ └── 08-commit-users.sql ├── 99-complete │ ├── 98-manifest-and-truncate.sql │ └── 99-users-cleanup.sql └── XX-destroy │ └── XX-destroy-users.sql └── 99-migrations ├── 1.0.1-migration ├── base-session-id-manifest.sql ├── events-staged.sql ├── page-views-staged.sql ├── page-views.sql ├── sessions.sql └── users-manifest.sql └── 1.0.3-migration └── alter-tables.sql /.gitignore: -------------------------------------------------------------------------------- 1 | # temp dir: 2 | tmp/ 3 | 4 | # script templates: 5 | .scripts/templates/ 6 | 7 | # virtualenv related: 8 | .test/bin/ 9 | .test/lib/ 10 | .test/include/ 11 | .test/.Python 12 | .test/pip-selfcheck.json 13 | .test/share 14 | 15 | # great expectations related: 16 | .test/great_expectations/results 17 | .test/great_expectations/config 18 | -------------------------------------------------------------------------------- /.markdownlintrc: -------------------------------------------------------------------------------- 1 | { 2 | "default": true, 3 | "MD013": false, 4 | "MD041": false 5 | } 6 | -------------------------------------------------------------------------------- /.scripts/pr_check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Expected input: 4 | # -b (binary) path to sql-runner binary 5 | # -d (database) target database for expectations 6 | # -a (auth) optional credentials for database target 7 | # -m (model) target model to run i.e. web or mobile 8 | 9 | while getopts 'b:d:a:m:' v 10 | do 11 | case $v in 12 | b) SQL_RUNNER_PATH=$OPTARG ;; 13 | d) DATABASE=$OPTARG ;; 14 | a) CREDENTIALS=$OPTARG ;; 15 | m) MODEL=$OPTARG ;; 16 | esac 17 | done 18 | 19 | script_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) 20 | 21 | # Set credentials via env vars 22 | export BIGQUERY_CREDS=${BIGQUERY_CREDS:-$CREDENTIALS} 23 | export REDSHIFT_PASSWORD=${REDSHIFT_PASSWORD:-$CREDENTIALS} 24 | export SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD:-$CREDENTIALS} 25 | 26 | echo "pr_check: Starting 10 e2e iterations" 27 | 28 | for i in {1..10}; do 29 | echo "pr_check: Starting e2e run $i"; 30 | 31 | bash $script_path/e2e.sh -b $SQL_RUNNER_PATH -m $MODEL -d $DATABASE || exit 1; 32 | 33 | echo "pr_check: e2e run $i Done"; 34 | 35 | done || exit 1 36 | 37 | echo "pr_check: Done" 38 | -------------------------------------------------------------------------------- /.scripts/templates/bigquery.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | -------------------------------------------------------------------------------- /.scripts/templates/redshift.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: PASSWORD_PLACEHOLDER 9 | :ssl: 10 | :variables: 11 | -------------------------------------------------------------------------------- /.scripts/templates/snowflake.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: PASSWORD_PLACEHOLDER 9 | :variables: 10 | -------------------------------------------------------------------------------- /.test/etc/jupyter/nbconfig/notebook.d/widgetsnbextension.json: -------------------------------------------------------------------------------- 1 | { 2 | "load_extensions": { 3 | "jupyter-js-widgets/extension": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /.test/great_expectations/expectations/mobile/v1/mobile_screen_view_in_session_values.json: -------------------------------------------------------------------------------- 1 | { 2 | "data_asset_type": "Dataset", 3 | "expectation_suite_name": "mobile_screen_view_in_session_values", 4 | "expectations": [ 5 | { 6 | "expectation_type": "expect_column_values_to_be_in_set", 7 | "kwargs": { 8 | "column": "dist_svis_values", 9 | "value_set": [1] 10 | } 11 | }, 12 | { 13 | "expectation_type": "expect_column_values_to_be_in_set", 14 | "kwargs": { 15 | "column": "all_minus_dist_svisi", 16 | "value_set": [0] 17 | } 18 | }, 19 | { 20 | "expectation_type": "expect_column_values_to_be_in_set", 21 | "kwargs": { 22 | "column": "all_minus_dist_svids", 23 | "value_set": [0] 24 | } 25 | } 26 | ], 27 | "meta": { 28 | "versions": { 29 | "test_suite_version": "1.0.4", 30 | "redshift_model_version": "1.1.0", 31 | "bigquery_model_version": "1.1.0", 32 | "snowflake_model_version": "1.1.0" 33 | }, 34 | "great_expectations.__version__": "0.12.0" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /.test/great_expectations/expectations/web/v1/page_view_in_session_values.json: -------------------------------------------------------------------------------- 1 | { 2 | "data_asset_type": "Dataset", 3 | "expectation_suite_name": "page_view_in_session_values", 4 | "expectations": [ 5 | { 6 | "expectation_type": "expect_column_values_to_be_in_set", 7 | "kwargs": { 8 | "column": "dist_pvis_values", 9 | "value_set": [1] 10 | } 11 | }, 12 | { 13 | "expectation_type": "expect_column_values_to_be_in_set", 14 | "kwargs": { 15 | "column": "all_minus_dist_pvisi", 16 | "value_set": [0] 17 | } 18 | }, 19 | { 20 | "expectation_type": "expect_column_values_to_be_in_set", 21 | "kwargs": { 22 | "column": "all_minus_dist_pvids", 23 | "value_set": [0] 24 | } 25 | } 26 | ], 27 | "meta": { 28 | "versions": { 29 | "test_suite_version": "1.1.1", 30 | "redshift_model_version": "1.3.1", 31 | "bigquery_model_version": "1.0.4", 32 | "snowflake_model_version": "1.0.2" 33 | }, 34 | "great_expectations.__version__": "0.12.0" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /.test/integration_tests/web/v1/bigquery_variables.yml.tmpl: -------------------------------------------------------------------------------- 1 | :variables: 2 | :input_schema: dv_test_data 3 | :scratch_schema: scratch_dev1 4 | :output_schema: derived_dev1 5 | :entropy: "" 6 | :start_date: 2021-03-01 7 | :update_cadence_days: 1 8 | :step: 9 | -------------------------------------------------------------------------------- /media/e2e.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowplow/data-models/dde99f4b1da6c8c4e9f4f0ad3cf60c5496a6e0ba/media/e2e.gif -------------------------------------------------------------------------------- /media/mobile_full_model_structure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowplow/data-models/dde99f4b1da6c8c4e9f4f0ad3cf60c5496a6e0ba/media/mobile_full_model_structure.jpg -------------------------------------------------------------------------------- /media/snowplow_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowplow/data-models/dde99f4b1da6c8c4e9f4f0ad3cf60c5496a6e0ba/media/snowplow_architecture.png -------------------------------------------------------------------------------- /media/snowplow_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowplow/data-models/dde99f4b1da6c8c4e9f4f0ad3cf60c5496a6e0ba/media/snowplow_logo.png -------------------------------------------------------------------------------- /media/web_full_model_structure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowplow/data-models/dde99f4b1da6c8c4e9f4f0ad3cf60c5496a6e0ba/media/web_full_model_structure.jpg -------------------------------------------------------------------------------- /media/web_model_module.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowplow/data-models/dde99f4b1da6c8c4e9f4f0ad3cf60c5496a6e0ba/media/web_model_module.jpg -------------------------------------------------------------------------------- /mobile/v1/bigquery/CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 1.1.0 (2021-06-15) 2 | -------------------------- 3 | BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (Close #100) 4 | BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #99) 5 | 6 | Version 1.0.0 (2021-04-26) 7 | -------------------------- 8 | Mobile: Add BigQuery mobile model v1 (close #73) 9 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/configs/migrations.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Bigquery", 6 | "playbooks": [ 7 | { 8 | "playbook": "standard/99-migrations/1.1.0-migration", 9 | "dependsOn": [] 10 | } 11 | ], 12 | "lockType": "hard", 13 | "owners": [ 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/configs/post_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "BigQuery", 6 | "sqlRunner": "0.9.3", 7 | "playbooks": [ 8 | { 9 | "playbook": "standard/01-base/99-base-complete", 10 | "dependsOn": [] 11 | }, 12 | { 13 | "playbook": "standard/02-screen-views/99-screen-views-complete", 14 | "dependsOn": [] 15 | }, 16 | { 17 | "playbook": "standard/03-optional-modules/01-app-errors/99-app-errors-complete", 18 | "dependsOn": [] 19 | }, 20 | { 21 | "playbook": "standard/04-sessions/99-sessions-complete", 22 | "dependsOn": [] 23 | }, 24 | { 25 | "playbook": "standard/05-users/99-users-complete", 26 | "dependsOn": [] 27 | }, 28 | { 29 | "playbook": "tests/00-staging-reconciliation/99-staging-reconciliation-complete", 30 | "dependsOn": [] 31 | } 32 | ], 33 | "lockType": "hard", 34 | "owners": [ 35 | ] 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/configs/teardown_all.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "BigQuery", 6 | "sqlRunner": "0.9.3", 7 | "playbooks": [ 8 | { 9 | "playbook": "standard/01-base/XX-destroy-base", 10 | "dependsOn": [] 11 | }, 12 | { 13 | "playbook": "standard/02-screen-views/XX-destroy-screen-views", 14 | "dependsOn": [] 15 | }, 16 | { 17 | "playbook": "standard/03-optional-modules/01-app-errors/XX-destroy-app-errors", 18 | "dependsOn": [] 19 | }, 20 | { 21 | "playbook": "standard/04-sessions/XX-destroy-sessions", 22 | "dependsOn": [] 23 | }, 24 | { 25 | "playbook": "standard/05-users/XX-destroy-users", 26 | "dependsOn": [] 27 | } 28 | ], 29 | "lockType": "hard", 30 | "owners": [ 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/custom/04-session-goals/01-session-goals-main.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :entropy: "" 10 | :steps: 11 | - :name: 01-session-goals-staged 12 | :queries: 13 | - :name: 01-session-goals-staged 14 | :file: custom/04-session-goals/01-session-goals-staged.sql 15 | :template: true 16 | - :name: 02-session-goals-upsert 17 | :queries: 18 | - :name: 02-session-goals-upsert 19 | :file: custom/04-session-goals/02-session-goals-upsert.sql 20 | :template: true 21 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/custom/04-session-goals/99-session-goals-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :entropy: "" 10 | :steps: 11 | - :name: 99-session-goals-cleanup 12 | :queries: 13 | - :name: 99-session-goals-cleanup 14 | :file: custom/04-session-goals/99-session-goals-cleanup.sql 15 | :template: true 16 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/custom/README.md: -------------------------------------------------------------------------------- 1 | # Custom playbooks 2 | 3 | This directory contains playbooks for the example custom modules [the custom sql directory](../../sql/custom). 4 | 5 | The readme in that directory also contains a guide to adding custom modules. 6 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/mobile/1.1.0 8 | :model: mobile 9 | :scratch_schema: scratch 10 | :output_schema: derived 11 | :entropy: "" 12 | :steps: 13 | - :name: 00-setup-metadata 14 | :queries: 15 | - :name: 00-setup-metadata 16 | :file: standard/00-setup/01-main/00-setup-metadata.sql 17 | :template: true 18 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/mobile/1.1.0 8 | :model: mobile 9 | :scratch_schema: scratch 10 | :output_schema: derived 11 | :entropy: "" 12 | :steps: 13 | - :name: 01-cleanup-metadata 14 | :queries: 15 | - :name: 01-cleanup-metadata 16 | :file: standard/00-setup/99-complete/01-cleanup-metadata.sql 17 | :template: true 18 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/README.md: -------------------------------------------------------------------------------- 1 | # Setup step playbooks 2 | 3 | These playbooks exist to accommodate scheduling jobs which don't conform to the usual flow of the standard model - they create and destroy the necessary tables to log metadata, with an ID which persists across modules. 4 | 5 | If running the standard model alone, or the standard model alongside custom steps, it is not necessary to run these steps. Instead, we can configure the `:ends_run:` variable to `true` in the `complete` playbook for the last module run in the standard model. 6 | 7 | In a scenario where we don't run the standard module, or we run portions of it on differing schedules, we can run the `00-setup-metadata.ymp.tmpl` playbook as the first step - to set up metadata and create the temporary run ID, and the `99-complete-metadata.yml.tmpl` playbook as the last step - to destroy the temporary run ID. 8 | 9 | If we would like to destroy the metadata tables for a full rebuild of the model, we may run the `XX-destroy-metadata.yml.tmpl` playbook to do so. It is advisable to rename the metadata table instead, however, in case there is some unforeseen need to see that data. 10 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/mobile/1.1.0 8 | :model: mobile 9 | :scratch_schema: scratch 10 | :output_schema: derived 11 | :entropy: "" 12 | :steps: 13 | - :name: 01-cleanup-metadata 14 | :queries: 15 | - :name: 01-cleanup-metadata 16 | :file: standard/00-setup/99-complete/01-cleanup-metadata.sql 17 | :template: true 18 | - :name: XX-destroy-metadata 19 | :queries: 20 | - :name: XX-destroy-metadata 21 | :file: standard/00-setup/XX-destroy/XX-destroy-metadata.sql 22 | :template: true 23 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/mobile/1.1.0 8 | :model: mobile 9 | :scratch_schema: scratch 10 | :output_schema: derived 11 | :entropy: "" 12 | :cleanup_mode: all 13 | :ends_run: false 14 | :steps: 15 | - :name: 98-base-manifest 16 | :queries: 17 | - :name: 98-base-manifest 18 | :file: standard/01-base/99-complete/98-base-manifest.sql 19 | :template: true 20 | - :name: 99-base-cleanup 21 | :queries: 22 | - :name: 99-base-cleanup 23 | :file: standard/01-base/99-complete/99-base-cleanup.sql 24 | :template: true 25 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/mobile/1.1.0 8 | :model: mobile 9 | :scratch_schema: scratch 10 | :output_schema: derived 11 | :entropy: "" 12 | :cleanup_mode: all 13 | :ends_run: true 14 | :steps: 15 | - :name: 99-base-cleanup 16 | :queries: 17 | - :name: 99-base-cleanup 18 | :file: standard/01-base/99-complete/99-base-cleanup.sql 19 | :template: true 20 | - :name: XX-destroy-base 21 | :queries: 22 | - :name: XX-destroy-base 23 | :file: standard/01-base/XX-destroy/XX-destroy-base.sql 24 | :template: true 25 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :model_version: bigquery/mobile/1.1.0 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: false 13 | :steps: 14 | - :name: 99-screen-views-cleanup 15 | :queries: 16 | - :name: 99-screen-views-cleanup 17 | :file: standard/02-screen-views/99-complete/99-screen-views-cleanup.sql 18 | :template: true 19 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :model_version: bigquery/mobile/1.1.0 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 99-screen-views-cleanup 15 | :queries: 16 | - :name: 99-screen-views-cleanup 17 | :file: standard/02-screen-views/99-complete/99-screen-views-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-screen-views 20 | :queries: 21 | - :name: XX-destroy-screen-views 22 | :file: standard/02-screen-views/XX-destroy/XX-destroy-screen-views.sql 23 | :template: true 24 | 25 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :model_version: bigquery/mobile/1.1.0 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: false 13 | :steps: 14 | - :name: 99-app-errors-cleanup 15 | :queries: 16 | - :name: 99-app-errors-cleanup 17 | :file: standard/03-optional-modules/01-app-errors/99-complete/99-app-errors-cleanup.sql 18 | :template: true 19 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :model_version: bigquery/mobile/1.1.0 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 99-app-errors-cleanup 15 | :queries: 16 | - :name: 99-app-errors-cleanup 17 | :file: standard/03-optional-modules/01-app-errors/99-complete/99-app-errors-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-app-errors 20 | :queries: 21 | - :name: XX-destroy-app-errors 22 | :file: standard/03-optional-modules/01-app-errors/XX-destroy/XX-destroy-app-errors.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :model_version: bigquery/mobile/1.1.0 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: false 13 | :steps: 14 | - :name: 98-truncate-upstream-staged 15 | :queries: 16 | - :name: 98-truncate-upstream-staged 17 | :file: standard/04-sessions/99-complete/98-truncate-upstream-staged.sql 18 | :template: true 19 | - :name: 99-sessions-cleanup 20 | :queries: 21 | - :name: 99-sessions-cleanup 22 | :file: standard/04-sessions/99-complete/99-sessions-cleanup.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :model_version: bigquery/mobile/1.1.0 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 99-sessions-cleanup 15 | :queries: 16 | - :name: 99-sessions-cleanup 17 | :file: standard/04-sessions/99-complete/99-sessions-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-sessions 20 | :queries: 21 | - :name: XX-destroy-sessions 22 | :file: standard/04-sessions/XX-destroy/XX-destroy-sessions.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :model_version: bigquery/mobile/1.1.0 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 98-manifest-and-truncate 15 | :queries: 16 | - :name: 98-manifest-and-truncate 17 | :file: standard/05-users/99-complete/98-manifest-and-truncate.sql 18 | :template: true 19 | - :name: 99-users-cleanup 20 | :queries: 21 | - :name: 99-users-cleanup 22 | :file: standard/05-users/99-complete/99-users-cleanup.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :model_version: bigquery/mobile/1.1.0 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 99-users-cleanup 15 | :queries: 16 | - :name: 99-users-cleanup 17 | :file: standard/05-users/99-complete/99-users-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-users 20 | :queries: 21 | - :name: XX-destroy-users 22 | :file: standard/05-users/XX-destroy/XX-destroy-users.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/standard/99-migrations/1.1.0-migration.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :entropy: "" 10 | :steps: 11 | - :name: 1.1.0-migration 12 | :queries: 13 | - :name: mobile-events-staged 14 | :file: standard/99-migrations/1.1.0-migration/mobile-events-staged.sql 15 | :template: true -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/mobile/1.1.0 8 | :model: mobile 9 | :scratch_schema: scratch 10 | :entropy: "" 11 | :app_errors: false 12 | :steps: 13 | - :name: 00-staging-reconciliation 14 | :queries: 15 | - :name: 00-staging-reconciliation 16 | :file: tests/00-staging-reconciliation/01-main/00-staging-reconciliation.sql 17 | :template: true 18 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/mobile/1.1.0 8 | :model: mobile 9 | :scratch_schema: scratch 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :steps: 13 | - :name: 99-staging-reconciliation-cleanup 14 | :queries: 15 | - :name: 99-staging-reconciliation-cleanup 16 | :file: tests/00-staging-reconciliation/99-complete/99-staging-reconciliation-cleanup.sql 17 | :template: true 18 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/custom/04-session-goals/02-session-goals-upsert.sql: -------------------------------------------------------------------------------- 1 | 2 | -- 2. Commit table procedure handles committing to prod, including table creation, and creation of new columns if 'automigrate' is set to TRUE 3 | 4 | CALL {{.output_schema}}.commit_table('{{.scratch_schema}}', -- sourceDataset 5 | 'session_goals_staged{{.entropy}}', -- sourceTable 6 | '{{.output_schema}}', -- targetDataset 7 | 'session_goals{{.entropy}}', -- targetTable 8 | 'session_id', -- joinKey 9 | 'start_tstamp', -- partitionKey 10 | TRUE); -- automigrate 11 | 12 | -- If we like, we can manually create and update our production table instead. 13 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/custom/04-session-goals/99-session-goals-cleanup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS {{.scratch_schema}}.session_goals_staged{{.entropy}}; 2 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/standard/00-setup/99-complete/01-cleanup-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.scratch_schema}}.{{.model}}_metadata_run_id{{.entropy}}; 18 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/standard/00-setup/XX-destroy/XX-destroy-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.datamodel_metadata{{.entropy}}; 18 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/standard/01-base/01-main/05-batch-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a new limit based on this data 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.{{.model}}_base_run_limits{{.entropy}} AS( 19 | SELECT 20 | MIN(min_tstamp) AS lower_limit, 21 | (SELECT upper_limit FROM {{.scratch_schema}}.{{.model}}_base_new_events_limits{{.entropy}}) AS upper_limit 22 | 23 | FROM 24 | {{.scratch_schema}}.{{.model}}_base_sessions_to_include{{.entropy}} 25 | ); 26 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/standard/04-sessions/01-main/05-sessions-prep-manifest.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Prep manifest data for users step 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.mobile_sessions_userid_manifest_this_run{{.entropy}} 19 | AS( 20 | SELECT 21 | device_user_id, 22 | MIN(start_tstamp) AS start_tstamp 23 | 24 | FROM 25 | {{.scratch_schema}}.mobile_sessions_this_run{{.entropy}} 26 | 27 | GROUP BY 1 28 | ); 29 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/standard/04-sessions/99-complete/98-truncate-upstream-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | TRUNCATE TABLE {{.scratch_schema}}.mobile_events_staged{{.entropy}}; 18 | TRUNCATE TABLE {{.scratch_schema}}.mobile_screen_views_staged{{.entropy}}; 19 | TRUNCATE TABLE {{.scratch_schema}}.mobile_app_errors_staged{{.entropy}}; 20 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/standard/05-users/01-main/02-users-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a limit for this run 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.mobile_users_limits{{.entropy}} AS( 19 | SELECT 20 | MIN(start_tstamp) AS lower_limit, 21 | MAX(start_tstamp) AS upper_limit 22 | 23 | FROM 24 | {{.scratch_schema}}.mobile_users_userids_this_run{{.entropy}} 25 | ); 26 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/standard/99-migrations/1.1.0-migration/mobile-events-staged.sql: -------------------------------------------------------------------------------- 1 | -- Create Procedure to drop mobile_events_staged if it is empty 2 | CREATE OR REPLACE PROCEDURE {{.scratch_schema}}.drop_mobile_events_staged_if_empty () 3 | BEGIN 4 | DECLARE rows_in_table INT64; 5 | SET rows_in_table = (SELECT count(*) FROM {{.scratch_schema}}.mobile_events_staged{{.entropy}}); 6 | 7 | IF rows_in_table > 0 THEN 8 | RAISE USING MESSAGE = 'Ensure that all data has been processed from events_staged and it is empty.'; 9 | ELSE 10 | DROP TABLE {{.scratch_schema}}.mobile_events_staged{{.entropy}}; 11 | END IF; 12 | END; 13 | 14 | -- Run the procedure 15 | CALL {{.scratch_schema}}.drop_mobile_events_staged_if_empty(); 16 | 17 | -- Remove the procedure 18 | DROP PROCEDURE {{.scratch_schema}}.drop_mobile_events_staged_if_empty; 19 | -------------------------------------------------------------------------------- /mobile/v1/bigquery/sql-runner/sql/tests/00-staging-reconciliation/99-complete/99-staging-reconciliation-cleanup.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | {{if eq .cleanup_mode "debug" "trace"}} SELECT 1; {{else}} 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_staging_reconciliation{{.entropy}}; 19 | {{end}} 20 | 21 | -------------------------------------------------------------------------------- /mobile/v1/redshift/CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 1.1.0 (2021-05-17) 2 | -------------------------- 3 | Redshift Mobile: Increase session_id character limit in manifest (Close #93) 4 | Redshift Mobile: Add simplified configs (Close #89) 5 | Redshift Mobile: Set app errors module to disabled by default (Close #88) 6 | Redshift Mobile: Add model_tstamp to derived tables (Close #82) 7 | Redshift Mobile: Move app errors columns to end of table (Close #83) 8 | Redshift Mobile: Remove CTE from user-aggs (Close #80) 9 | 10 | Version 1.0.0 (2021-03-25) 11 | -------------------------- 12 | Update scripts for mobile model (#66) 13 | Add Redshift mobile model v1 (#65) 14 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/configs/post_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "playbooks": [ 7 | { 8 | "playbook": "standard/01-base/99-base-complete", 9 | "dependsOn": [] 10 | }, 11 | { 12 | "playbook": "standard/02-screen-views/99-screen-views-complete", 13 | "dependsOn": [] 14 | }, 15 | { 16 | "playbook": "standard/03-optional-modules/01-app-errors/99-app-errors-complete", 17 | "dependsOn": [] 18 | }, 19 | { 20 | "playbook": "standard/04-sessions/99-sessions-complete", 21 | "dependsOn": [] 22 | }, 23 | { 24 | "playbook": "standard/05-users/99-users-complete", 25 | "dependsOn": [] 26 | }, 27 | { 28 | "playbook": "tests/00-staging-reconciliation/99-staging-reconciliation-complete", 29 | "dependsOn": [] 30 | } 31 | ], 32 | "lockType": "hard", 33 | "owners": [ 34 | ] 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/configs/teardown_all.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "playbooks": [ 7 | { 8 | "playbook": "standard/01-base/XX-destroy-base", 9 | "dependsOn": [] 10 | }, 11 | { 12 | "playbook": "standard/02-screen-views/XX-destroy-screen-views", 13 | "dependsOn": [] 14 | }, 15 | { 16 | "playbook": "standard/03-optional-modules/01-app-errors/XX-destroy-app-errors", 17 | "dependsOn": [] 18 | }, 19 | { 20 | "playbook": "standard/04-sessions/XX-destroy-sessions", 21 | "dependsOn": [] 22 | }, 23 | { 24 | "playbook": "standard/05-users/XX-destroy-users", 25 | "dependsOn": [] 26 | } 27 | ], 28 | "lockType": "hard", 29 | "owners": [ 30 | ] 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/custom/04-session-goals/01-session-goals-main.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 01-session-goals-setup 16 | :queries: 17 | - :name: 01-session-goals-setup 18 | :file: custom/04-session-goals/01-session-goals-setup.sql 19 | :template: true 20 | - :name: 02-session-goals-staged 21 | :queries: 22 | - :name: 02-session-goals-staged 23 | :file: custom/04-session-goals/02-session-goals-staged.sql 24 | :template: true 25 | - :name: 03-session-goals-upsert 26 | :queries: 27 | - :name: 03-session-goals-upsert 28 | :file: custom/04-session-goals/03-session-goals-upsert.sql 29 | :template: true 30 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/custom/04-session-goals/99-session-goals-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 99-session-goals-cleanup 16 | :queries: 17 | - :name: 99-session-goals-cleanup 18 | :file: custom/04-session-goals/99-session-goals-cleanup.sql 19 | :template: true 20 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/custom/README.md: -------------------------------------------------------------------------------- 1 | # Custom playbooks 2 | 3 | This directory contains playbooks for the example custom modules [the custom sql directory](../../sql/custom). 4 | 5 | The readme in that directory also contains a guide to adding custom modules. 6 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :model: mobile 13 | :scratch_schema: scratch 14 | :output_schema: derived 15 | :entropy: "" 16 | :steps: 17 | - :name: 00-setup-metadata 18 | :queries: 19 | - :name: 00-setup-metadata 20 | :file: standard/00-setup/01-main/00-setup-metadata.sql 21 | :template: true 22 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :model: mobile 13 | :scratch_schema: scratch 14 | :output_schema: derived 15 | :entropy: "" 16 | :steps: 17 | - :name: 01-cleanup-metadata 18 | :queries: 19 | - :name: 01-cleanup-metadata 20 | :file: standard/00-setup/99-complete/01-cleanup-metadata.sql 21 | :template: true 22 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/README.md: -------------------------------------------------------------------------------- 1 | # Setup step playbooks 2 | 3 | These playbooks exist to accommodate scheduling jobs which don't conform to the usual flow of the standard model - they create and destroy the necessary tables to log metadata, with an ID which persists across modules. 4 | 5 | If running the standard model alone, or the standard model alongside custom steps, it is not necessary to run these steps. Instead, we can configure the `:ends_run:` variable to `true` in the `complete` playbook for the last module run in the standard model. 6 | 7 | In a scenario where we don't run the standard module, or we run portions of it on differing schedules, we can run the `00-setup-metadata.ymp.tmpl` playbook as the first step - to set up metadata and create the temporary run ID, and the `99-complete-metadata.yml.tmpl` playbook as the last step - to destroy the temporary run ID. 8 | 9 | If we would like to destroy the metadata tables for a full rebuild of the model, we may run the `XX-destroy-metadata.yml.tmpl` playbook to do so. It is advisable to rename the metadata table instead, however, in case there is some unforeseen need to see that data. 10 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :model: mobile 13 | :scratch_schema: scratch 14 | :output_schema: derived 15 | :entropy: "" 16 | :steps: 17 | - :name: 01-cleanup-metadata 18 | :queries: 19 | - :name: 01-cleanup-metadata 20 | :file: standard/00-setup/99-complete/01-cleanup-metadata.sql 21 | :template: true 22 | - :name: XX-destroy-metadata 23 | :queries: 24 | - :name: XX-destroy-metadata 25 | :file: standard/00-setup/XX-destroy/XX-destroy-metadata.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :model: mobile 13 | :scratch_schema: scratch 14 | :output_schema: derived 15 | :entropy: "" 16 | :cleanup_mode: all 17 | :ends_run: false 18 | :steps: 19 | - :name: 98-base-manifest 20 | :queries: 21 | - :name: 98-base-manifest 22 | :file: standard/01-base/99-complete/98-base-manifest.sql 23 | :template: true 24 | - :name: 99-base-cleanup 25 | :queries: 26 | - :name: 99-base-cleanup 27 | :file: standard/01-base/99-complete/99-base-cleanup.sql 28 | :template: true 29 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :model: mobile 13 | :scratch_schema: scratch 14 | :output_schema: derived 15 | :entropy: "" 16 | :cleanup_mode: all 17 | :ends_run: true 18 | :steps: 19 | - :name: 99-base-cleanup 20 | :queries: 21 | - :name: 99-base-cleanup 22 | :file: standard/01-base/99-complete/99-base-cleanup.sql 23 | :template: true 24 | - :name: XX-destroy-base 25 | :queries: 26 | - :name: XX-destroy-base 27 | :file: standard/01-base/XX-destroy/XX-destroy-base.sql 28 | :template: true 29 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: false 17 | :steps: 18 | - :name: 99-screen-views-cleanup 19 | :queries: 20 | - :name: 99-screen-views-cleanup 21 | :file: standard/02-screen-views/99-complete/99-screen-views-cleanup.sql 22 | :template: true 23 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-screen-views-cleanup 19 | :queries: 20 | - :name: 99-screen-views-cleanup 21 | :file: standard/02-screen-views/99-complete/99-screen-views-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-screen-views 24 | :queries: 25 | - :name: XX-destroy-screen-views 26 | :file: standard/02-screen-views/XX-destroy/XX-destroy-screen-views.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: false 17 | :steps: 18 | - :name: 99-app-errors-cleanup 19 | :queries: 20 | - :name: 99-app-errors-cleanup 21 | :file: standard/03-optional-modules/01-app-errors/99-complete/99-app-errors-cleanup.sql 22 | :template: true 23 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-app-errors-cleanup 19 | :queries: 20 | - :name: 99-app-errors-cleanup 21 | :file: standard/03-optional-modules/01-app-errors/99-complete/99-app-errors-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-app-errors 24 | :queries: 25 | - :name: XX-destroy-app-errors 26 | :file: standard/03-optional-modules/01-app-errors/XX-destroy/XX-destroy-app-errors.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: false 17 | :steps: 18 | - :name: 98-truncate-upstream-staged 19 | :queries: 20 | - :name: 98-truncate-upstream-staged 21 | :file: standard/04-sessions/99-complete/98-truncate-upstream-staged.sql 22 | :template: true 23 | - :name: 99-sessions-cleanup 24 | :queries: 25 | - :name: 99-sessions-cleanup 26 | :file: standard/04-sessions/99-complete/99-sessions-cleanup.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-sessions-cleanup 19 | :queries: 20 | - :name: 99-sessions-cleanup 21 | :file: standard/04-sessions/99-complete/99-sessions-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-sessions 24 | :queries: 25 | - :name: XX-destroy-sessions 26 | :file: standard/04-sessions/XX-destroy/XX-destroy-sessions.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 98-manifest-and-truncate 19 | :queries: 20 | - :name: 98-manifest-and-truncate 21 | :file: standard/05-users/99-complete/98-manifest-and-truncate.sql 22 | :template: true 23 | - :name: 99-users-cleanup 24 | :queries: 25 | - :name: 99-users-cleanup 26 | :file: standard/05-users/99-complete/99-users-cleanup.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-users-cleanup 19 | :queries: 20 | - :name: 99-users-cleanup 21 | :file: standard/05-users/99-complete/99-users-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-users 24 | :queries: 25 | - :name: XX-destroy-users 26 | :file: standard/05-users/XX-destroy/XX-destroy-users.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :model: mobile 13 | :scratch_schema: scratch 14 | :entropy: "" 15 | :app_errors: false 16 | :steps: 17 | - :name: 00-staging-reconciliation 18 | :queries: 19 | - :name: 00-staging-reconciliation 20 | :file: tests/00-staging-reconciliation/01-main/00-staging-reconciliation.sql 21 | :template: true 22 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/mobile/1.1.0 12 | :model: mobile 13 | :scratch_schema: scratch 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :steps: 17 | - :name: 99-staging-reconciliation-cleanup 18 | :queries: 19 | - :name: 99-staging-reconciliation-cleanup 20 | :file: tests/00-staging-reconciliation/99-complete/99-staging-reconciliation-cleanup.sql 21 | :template: true 22 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/custom/04-session-goals/01-session-goals-setup.sql: -------------------------------------------------------------------------------- 1 | -- 1. Create prod table 2 | 3 | CREATE TABLE IF NOT EXISTS {{.output_schema}}.session_goals{{.entropy}} ( 4 | session_id CHAR(36), 5 | has_started_registration BOOLEAN, 6 | has_completed_registration BOOLEAN, 7 | has_used_search BOOLEAN, 8 | has_viewed_products BOOLEAN, 9 | has_completed_goals BOOLEAN 10 | ); 11 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/custom/04-session-goals/02-session-goals-staged.sql: -------------------------------------------------------------------------------- 1 | -- 2. Aggregate with a drop and recompute logic 2 | 3 | DROP TABLE IF EXISTS {{.scratch_schema}}.session_goals_staged{{.entropy}}; 4 | 5 | CREATE TABLE {{.scratch_schema}}.session_goals_staged{{.entropy}} AS ( 6 | 7 | WITH goals AS ( 8 | 9 | SELECT 10 | sv.session_id, 11 | BOOL_OR(sv.screen_view_name = 'registration') AS has_started_registration, 12 | BOOL_OR(sv.screen_view_name = 'my_account') AS has_completed_registration, 13 | BOOL_OR(sv.screen_view_name = 'search_results') AS has_used_search, 14 | BOOL_OR(sv.screen_view_name = 'products') AS has_viewed_products 15 | 16 | FROM 17 | {{.scratch_schema}}.mobile_screen_views_staged{{.entropy}} sv 18 | 19 | GROUP BY 1 20 | 21 | ) 22 | 23 | SELECT 24 | g.session_id, 25 | g.has_started_registration, 26 | g.has_completed_registration, 27 | g.has_used_search, 28 | g.has_viewed_products, 29 | CASE 30 | WHEN g.has_started_registration AND g.has_completed_registration 31 | AND g.has_used_search AND g.has_viewed_products THEN TRUE 32 | ELSE FALSE END AS has_completed_goals 33 | 34 | FROM goals g 35 | 36 | ); 37 | 38 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/custom/04-session-goals/03-session-goals-upsert.sql: -------------------------------------------------------------------------------- 1 | -- 3. DELETE - INSERT to production (and optionally drop the temp table) 2 | 3 | BEGIN; --it is safest to use a transaction, in case of failure. 4 | 5 | DELETE FROM {{.output_schema}}.session_goals{{.entropy}} 6 | WHERE session_id IN (SELECT session_id FROM {{.scratch_schema}}.session_goals_staged{{.entropy}}); 7 | 8 | INSERT INTO {{.output_schema}}.session_goals{{.entropy}} 9 | (SELECT * FROM {{.scratch_schema}}.session_goals_staged{{.entropy}}); 10 | 11 | END; 12 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/custom/04-session-goals/99-session-goals-cleanup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS {{.scratch_schema}}.session_goals_staged{{.entropy}}; 2 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/00-setup/99-complete/01-cleanup-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.scratch_schema}}.{{.model}}_metadata_run_id{{.entropy}}; 18 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/00-setup/XX-destroy/XX-destroy-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.datamodel_metadata{{.entropy}}; 18 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/01-base/01-main/05-batch-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a new limit based on this data 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.{{.model}}_base_run_limits{{.entropy}}; 19 | 20 | CREATE TABLE {{.scratch_schema}}.{{.model}}_base_run_limits{{.entropy}} AS( 21 | SELECT 22 | MIN(min_tstamp) AS lower_limit, 23 | (SELECT upper_limit FROM {{.scratch_schema}}.{{.model}}_base_new_events_limits{{.entropy}}) AS upper_limit 24 | 25 | FROM 26 | {{.scratch_schema}}.{{.model}}_base_sessions_to_include{{.entropy}} 27 | ); 28 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/01-base/XX-destroy/XX-destroy-base.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.{{.model}}_base_event_id_manifest{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.output_schema}}.{{.model}}_base_session_id_manifest{{.entropy}}; 19 | DROP TABLE IF EXISTS {{.scratch_schema}}.{{.model}}_events_staged{{.entropy}}; 20 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/01-screen-view-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a limit for this run - single row table. 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_sv_run_limits{{.entropy}}; 19 | 20 | CREATE TABLE {{.scratch_schema}}.mobile_sv_run_limits{{.entropy}} AS ( 21 | SELECT 22 | MIN(collector_tstamp) AS lower_limit, 23 | MAX(collector_tstamp) AS upper_limit 24 | 25 | FROM 26 | {{.scratch_schema}}.mobile_events_staged{{.entropy}} 27 | WHERE event_name = 'screen_view' 28 | ); 29 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/XX-destroy/XX-destroy-screen-views.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.mobile_screen_views{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_screen_views_staged{{.entropy}}; 19 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/XX-destroy/XX-destroy-app-errors.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.mobile_app_errors{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_app_errors_staged{{.entropy}}; 19 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/05-sessions-prep-manifest.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Prep manifest data for users step 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_sessions_userid_manifest_this_run{{.entropy}}; 19 | 20 | CREATE TABLE {{.scratch_schema}}.mobile_sessions_userid_manifest_this_run{{.entropy}} 21 | DISTSTYLE KEY 22 | DISTKEY (device_user_id) 23 | SORTKEY (device_user_id) 24 | AS( 25 | SELECT 26 | device_user_id, 27 | MIN(start_tstamp) AS min_tstamp 28 | 29 | FROM 30 | {{.scratch_schema}}.mobile_sessions_this_run{{.entropy}} 31 | 32 | GROUP BY 1 33 | ) 34 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/04-sessions/99-complete/98-truncate-upstream-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | TRUNCATE {{.scratch_schema}}.mobile_events_staged{{.entropy}}; 18 | TRUNCATE {{.scratch_schema}}.mobile_screen_views_staged{{.entropy}}; 19 | TRUNCATE {{.scratch_schema}}.mobile_app_errors_staged{{.entropy}}; 20 | 21 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/04-sessions/XX-destroy/XX-destroy-sessions.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.mobile_sessions{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_sessions_userid_manifest_staged{{.entropy}}; 19 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/02-users-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a limit for this run - single value table. 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_users_limits{{.entropy}}; 19 | 20 | CREATE TABLE {{.scratch_schema}}.mobile_users_limits{{.entropy}} AS( 21 | SELECT 22 | MIN(start_tstamp) AS lower_limit, 23 | MAX(start_tstamp) AS upper_limit 24 | 25 | FROM 26 | {{.scratch_schema}}.mobile_users_userids_this_run{{.entropy}} 27 | ); 28 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/05-users/99-complete/98-manifest-and-truncate.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | BEGIN; 18 | -- Update manifest 19 | DELETE 20 | FROM {{.output_schema}}.mobile_users_manifest{{.entropy}} 21 | WHERE device_user_id IN (SELECT device_user_id FROM {{.scratch_schema}}.mobile_users_userids_this_run{{.entropy}}); 22 | 23 | INSERT INTO {{.output_schema}}.mobile_users_manifest{{.entropy}} (SELECT * FROM {{.scratch_schema}}.mobile_users_userids_this_run{{.entropy}}); 24 | 25 | -- Truncate input table just processed 26 | TRUNCATE {{.scratch_schema}}.mobile_sessions_userid_manifest_staged{{.entropy}}; 27 | 28 | END; 29 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/standard/05-users/XX-destroy/XX-destroy-users.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.mobile_users{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.output_schema}}.mobile_users_manifest{{.entropy}}; 19 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/tests/00-staging-reconciliation/99-complete/99-staging-reconciliation-cleanup.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | {{if eq .cleanup_mode "debug" "trace"}} SELECT 1; {{else}} 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_staging_reconciliation{{.entropy}}; 19 | {{end}} 20 | 21 | -------------------------------------------------------------------------------- /mobile/v1/redshift/sql-runner/sql/tests/00-staging-reconciliation/XX-destroy/XX-destroy-staging-reconciliation.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_staging_reconciliation{{.entropy}}; 18 | 19 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 1.1.1 (2025-01-08) 2 | ---------------------------------------- 3 | Snowflake Mobile: Remove length limits from varchar fields 4 | 5 | Version 1.1.0 (2021-06-16) 6 | -------------------------- 7 | Snowflake Mobile: Update update_manifest to throw error message (Close #104) 8 | Snowflake Mobile: Update commit_table procedure to throw error messages (Close #103) 9 | Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #102) 10 | 11 | Version 1.0.0 (2021-05-06) 12 | -------------------------- 13 | Add Snowflake mobile model v1 (Close #85) 14 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/configs/migrations.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Snowflake", 6 | "playbooks": [ 7 | { 8 | "playbook": "standard/99-migrations/1.1.0-migration", 9 | "dependsOn": [] 10 | } 11 | ], 12 | "lockType": "hard", 13 | "owners": [ 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/configs/post_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "sqlRunner": "0.9.3", 7 | "playbooks": [ 8 | { 9 | "playbook": "standard/01-base/99-base-complete", 10 | "dependsOn": [] 11 | }, 12 | { 13 | "playbook": "standard/02-screen-views/99-screen-views-complete", 14 | "dependsOn": [] 15 | }, 16 | { 17 | "playbook": "standard/03-optional-modules/01-app-errors/99-app-errors-complete", 18 | "dependsOn": [] 19 | }, 20 | { 21 | "playbook": "standard/04-sessions/99-sessions-complete", 22 | "dependsOn": [] 23 | }, 24 | { 25 | "playbook": "standard/05-users/99-users-complete", 26 | "dependsOn": [] 27 | }, 28 | { 29 | "playbook": "tests/00-staging-reconciliation/99-staging-reconciliation-complete", 30 | "dependsOn": [] 31 | } 32 | ], 33 | "lockType": "hard", 34 | "owners": [ 35 | ] 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/configs/teardown_all.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "sqlRunner": "0.9.3", 7 | "playbooks": [ 8 | { 9 | "playbook": "standard/01-base/XX-destroy-base", 10 | "dependsOn": [] 11 | }, 12 | { 13 | "playbook": "standard/02-screen-views/XX-destroy-screen-views", 14 | "dependsOn": [] 15 | }, 16 | { 17 | "playbook": "standard/03-optional-modules/01-app-errors/XX-destroy-app-errors", 18 | "dependsOn": [] 19 | }, 20 | { 21 | "playbook": "standard/04-sessions/XX-destroy-sessions", 22 | "dependsOn": [] 23 | }, 24 | { 25 | "playbook": "standard/05-users/XX-destroy-users", 26 | "dependsOn": [] 27 | } 28 | ], 29 | "lockType": "hard", 30 | "owners": [ 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/custom/04-session-goals/01-session-goals-main.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 01-session-goals-staged 16 | :queries: 17 | - :name: 01-session-goals-staged 18 | :file: custom/04-session-goals/01-session-goals-staged.sql 19 | :template: true 20 | - :name: 02-session-goals-upsert 21 | :queries: 22 | - :name: 02-session-goals-upsert 23 | :file: custom/04-session-goals/02-session-goals-upsert.sql 24 | :template: true 25 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/custom/04-session-goals/99-session-goals-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :entropy: "" 13 | :steps: 14 | - :name: 99-session-goals-cleanup 15 | :queries: 16 | - :name: 99-session-goals-cleanup 17 | :file: custom/04-session-goals/99-session-goals-cleanup.sql 18 | :template: true 19 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/custom/04-session-goals/README.md: -------------------------------------------------------------------------------- 1 | # Custom playbooks 2 | 3 | This directory contains playbooks for the example custom modules [the custom sql directory](../../sql/custom). 4 | 5 | The readme in that directory also contains a guide to adding custom modules. 6 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :model: mobile 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :steps: 16 | - :name: 00-setup-metadata 17 | :queries: 18 | - :name: 00-setup-metadata 19 | :file: standard/00-setup/01-main/00-setup-metadata.sql 20 | :template: true 21 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :model: mobile 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :steps: 16 | - :name: 99-metadata-cleanup 17 | :queries: 18 | - :name: 99-metadata-cleanup 19 | :file: standard/00-setup/99-complete/99-metadata-cleanup.sql 20 | :template: true 21 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/00-setup/README.md: -------------------------------------------------------------------------------- 1 | # Setup step playbooks 2 | 3 | These playbooks exist to accommodate scheduling jobs which don't conform to the usual flow of the standard model - they create and destroy the necessary tables to log metadata, with an ID which persists across modules. 4 | 5 | If running the standard model alone, or the standard model alongside custom steps, it is not necessary to run these steps. Instead, we can configure the `:ends_run:` variable to `true` in the `complete` playbook for the last module run in the standard model. 6 | 7 | In a scenario where we don't run the standard module, or we run portions of it on differing schedules, we can run the `00-setup-metadata.ymp.tmpl` playbook as the first step - to set up metadata and create the temporary run ID, and the `99-complete-metadata.yml.tmpl` playbook as the last step - to destroy the temporary run ID. 8 | 9 | If we would like to destroy the metadata tables for a full rebuild of the model, we may run the `XX-destroy-metadata.yml.tmpl` playbook to do so. It is advisable to rename the metadata table instead, however, in case there is some unforeseen need to see that data. 10 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :model: mobile 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :steps: 16 | - :name: 99-metadata-cleanup 17 | :queries: 18 | - :name: 99-metadata-cleanup 19 | :file: standard/00-setup/99-complete/99-metadata-cleanup.sql 20 | :template: true 21 | - :name: XX-destroy-metadata 22 | :queries: 23 | - :name: XX-destroy-metadata 24 | :file: standard/00-setup/XX-destroy/XX-destroy-metadata.sql 25 | :template: true 26 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :model: mobile 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: false 17 | :steps: 18 | - :name: 98-base-manifest 19 | :queries: 20 | - :name: 98-base-manifest 21 | :file: standard/01-base/99-complete/98-base-manifest.sql 22 | :template: true 23 | - :name: 99-base-cleanup 24 | :queries: 25 | - :name: 99-base-cleanup 26 | :file: standard/01-base/99-complete/99-base-cleanup.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :model: mobile 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-base-cleanup 19 | :queries: 20 | - :name: 99-base-cleanup 21 | :file: standard/01-base/99-complete/99-base-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-base 24 | :queries: 25 | - :name: XX-destroy-base 26 | :file: standard/01-base/XX-destroy/XX-destroy-base.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: false 16 | :steps: 17 | - :name: 99-screen-views-cleanup 18 | :queries: 19 | - :name: 99-screen-views-cleanup 20 | :file: standard/02-screen-views/99-complete/99-screen-views-cleanup.sql 21 | :template: true 22 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 99-screen-views-cleanup 18 | :queries: 19 | - :name: 99-screen-views-cleanup 20 | :file: standard/02-screen-views/99-complete/99-screen-views-cleanup.sql 21 | :template: true 22 | - :name: XX-destroy-screen-views 23 | :queries: 24 | - :name: XX-destroy-screen-views 25 | :file: standard/02-screen-views/XX-destroy/XX-destroy-screen-views.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: false 16 | :steps: 17 | - :name: 99-app-errors-cleanup 18 | :queries: 19 | - :name: 99-app-errors-cleanup 20 | :file: standard/03-optional-modules/01-app-errors/99-complete/99-app-errors-cleanup.sql 21 | :template: true 22 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 99-app-errors-cleanup 18 | :queries: 19 | - :name: 99-app-errors-cleanup 20 | :file: standard/03-optional-modules/01-app-errors/99-complete/99-app-errors-cleanup.sql 21 | :template: true 22 | - :name: XX-destroy-app-errors 23 | :queries: 24 | - :name: XX-destroy-app-errors 25 | :file: standard/03-optional-modules/01-app-errors/XX-destroy/XX-destroy-app-errors.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: false 16 | :steps: 17 | - :name: 98-truncate-upstream-staged 18 | :queries: 19 | - :name: 98-truncate-upstream-staged 20 | :file: standard/04-sessions/99-complete/98-truncate-upstream-staged.sql 21 | :template: true 22 | - :name: 99-sessions-cleanup 23 | :queries: 24 | - :name: 99-sessions-cleanup 25 | :file: standard/04-sessions/99-complete/99-sessions-cleanup.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 99-sessions-cleanup 18 | :queries: 19 | - :name: 99-sessions-cleanup 20 | :file: standard/04-sessions/99-complete/99-sessions-cleanup.sql 21 | :template: true 22 | - :name: XX-destroy-sessions 23 | :queries: 24 | - :name: XX-destroy-sessions 25 | :file: standard/04-sessions/XX-destroy/XX-destroy-sessions.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 98-manifest-and-truncate 18 | :queries: 19 | - :name: 98-manifest-and-truncate 20 | :file: standard/05-users/99-complete/98-manifest-and-truncate.sql 21 | :template: true 22 | - :name: 99-users-cleanup 23 | :queries: 24 | - :name: 99-users-cleanup 25 | :file: standard/05-users/99-complete/99-users-cleanup.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 99-users-cleanup 18 | :queries: 19 | - :name: 99-users-cleanup 20 | :file: standard/05-users/99-complete/99-users-cleanup.sql 21 | :template: true 22 | - :name: XX-destroy-users 23 | :queries: 24 | - :name: XX-destroy-users 25 | :file: standard/05-users/XX-destroy/XX-destroy-users.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/99-migrations/1.1.0-migration.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :scratch_schema: scratch 11 | :output_schema: derived 12 | :entropy: "" 13 | :steps: 14 | - :name: 1.1.0-migration 15 | :queries: 16 | - :name: mobile-events-staged 17 | :file: standard/99-migrations/1.1.0-migration/mobile-events-staged.sql 18 | :template: true 19 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/standard/99-migrations/1.1.1-migration.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :scratch_schema: scratch 11 | :output_schema: derived 12 | :entropy: "" 13 | :steps: 14 | - :name: 1.1.1-migration 15 | :queries: 16 | - :name: alter-tables 17 | :file: standard/99-migrations/1.1.1-migration/alter-tables.sql 18 | :template: true 19 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :model: mobile 12 | :scratch_schema: scratch 13 | :entropy: "" 14 | :app_errors: false 15 | :steps: 16 | - :name: 00-staging-reconciliation 17 | :queries: 18 | - :name: 00-staging-reconciliation 19 | :file: tests/00-staging-reconciliation/01-main/00-staging-reconciliation.sql 20 | :template: true 21 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/mobile/1.1.1 11 | :model: mobile 12 | :scratch_schema: scratch 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :steps: 16 | - :name: 99-staging-reconciliation-cleanup 17 | :queries: 18 | - :name: 99-staging-reconciliation-cleanup 19 | :file: tests/00-staging-reconciliation/99-complete/99-staging-reconciliation-cleanup.sql 20 | :template: true 21 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/custom/04-session-goals/99-session-goals-cleanup.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.scratch_schema}}.session_goals_staged{{.entropy}}; 18 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/00-setup/99-complete/99-metadata-cleanup.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.scratch_schema}}.{{.model}}_metadata_run_id{{.entropy}}; 18 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/00-setup/XX-destroy/XX-destroy-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.datamodel_metadata{{.entropy}}; 18 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/01-base/01-main/01-new-events-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | -- Create a limit for this run - single row table. 19 | CREATE OR REPLACE TABLE {{.scratch_schema}}.{{.model}}_base_new_events_limits{{.entropy}} 20 | AS ( 21 | SELECT 22 | TIMEADD(HOUR, -{{or .lookback_window_hours 6}}, MAX(collector_tstamp)) AS lower_limit, 23 | TIMEADD(DAY, {{or .update_cadence_days 7}}, MAX(collector_tstamp)) AS upper_limit, 24 | TIMEADD(DAY, -{{or .session_lookback_days 365}}, MAX(collector_tstamp)) AS session_limit 25 | 26 | FROM 27 | {{.output_schema}}.{{.model}}_base_event_id_manifest{{.entropy}} 28 | ); 29 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/01-base/01-main/05-batch-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | -- Create a new limit based on this data 19 | CREATE OR REPLACE TABLE {{.scratch_schema}}.{{.model}}_base_run_limits{{.entropy}} 20 | AS ( 21 | SELECT 22 | MIN(min_tstamp) AS lower_limit, 23 | (SELECT upper_limit FROM {{.scratch_schema}}.{{.model}}_base_new_events_limits{{.entropy}}) AS upper_limit 24 | 25 | FROM 26 | {{.scratch_schema}}.{{.model}}_base_sessions_to_include{{.entropy}} 27 | ); 28 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/01-base/99-complete/98-base-manifest.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CALL {{.output_schema}}.update_manifest('base'); 19 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/04-sessions/01-main/05-sessions-prep-manifest.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | -- Prep manifest data for users step 19 | CREATE OR REPLACE TABLE {{.scratch_schema}}.mobile_sessions_userid_manifest_this_run{{.entropy}} 20 | AS ( 21 | SELECT 22 | device_user_id, 23 | MIN(start_tstamp) AS start_tstamp 24 | 25 | FROM 26 | {{.scratch_schema}}.mobile_sessions_this_run{{.entropy}} 27 | 28 | GROUP BY 1 29 | ); 30 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/04-sessions/99-complete/98-truncate-upstream-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | TRUNCATE TABLE {{.scratch_schema}}.mobile_events_staged{{.entropy}}; 19 | TRUNCATE TABLE {{.scratch_schema}}.mobile_screen_views_staged{{.entropy}}; 20 | TRUNCATE TABLE {{.scratch_schema}}.mobile_app_errors_staged{{.entropy}}; 21 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/05-users/01-main/01-userids-this-run.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.mobile_users_userids_this_run{{.entropy}} 19 | AS ( 20 | SELECT 21 | a.device_user_id, 22 | -- LEAST produces NULL if any input value is null 23 | LEAST(a.start_tstamp, COALESCE(b.start_tstamp, a.start_tstamp)) AS start_tstamp 24 | 25 | FROM 26 | {{.scratch_schema}}.mobile_sessions_userid_manifest_staged{{.entropy}} a 27 | LEFT JOIN 28 | {{.output_schema}}.mobile_users_manifest{{.entropy}} b 29 | ON a.device_user_id = b.device_user_id 30 | ); 31 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/05-users/01-main/02-users-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.mobile_users_limits{{.entropy}} 19 | AS ( 20 | SELECT 21 | MIN(start_tstamp) AS lower_limit, 22 | MAX(start_tstamp) AS upper_limit 23 | 24 | FROM 25 | {{.scratch_schema}}.mobile_users_userids_this_run{{.entropy}} 26 | ); 27 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/standard/05-users/99-complete/98-manifest-and-truncate.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | -- Update manifest and truncate input table just processed 19 | CALL {{.output_schema}}.update_manifest('users'); 20 | -------------------------------------------------------------------------------- /mobile/v1/snowflake/sql-runner/sql/tests/00-staging-reconciliation/99-complete/99-staging-reconciliation-cleanup.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Snowplow Analytics Ltd. All rights reserved. 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | {{if eq .cleanup_mode "debug" "trace"}} SELECT 1; {{else}} 15 | DROP TABLE IF EXISTS {{.scratch_schema}}.mobile_staging_reconciliation{{.entropy}}; 16 | {{end}} 17 | -------------------------------------------------------------------------------- /web/v1/bigquery/CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 1.0.4 (2022-12-13) 2 | -------------------------- 3 | BigQuery: Fix upsert limit in commit_table procedure (Close #75) 4 | 5 | Version 1.0.3 (2021-03-24) 6 | -------------------------- 7 | BigQuery: Change session lookback to use days rather than hours (#60) 8 | BigQuery: Fix typo in stored procedure (#61) 9 | 10 | Version 1.0.2 (2021-03-11) 11 | -------------------------- 12 | BigQuery: Handle schema evolution in core contexts (close #52) 13 | 14 | Version 1.0.1 (2021-02-24) 15 | ------------------------- 16 | BigQuery: Revert default schemas to use `atomic`, `scratch`, `derived` (#41) 17 | BigQuery: Add backwards compatibility for derived_tstamp partitioned tables (#38) 18 | BigQuery: Fix typo in `XX-destroy` steps (#40) 19 | BigQuery: ensure non-null page_view_id in page_views (#37) 20 | BigQuery: Fix error for yauaa context (#33) 21 | BigQuery: Replace example customisation (#46) 22 | 23 | Version 1.0.0 (2021-01-20) 24 | --------------------------- 25 | Add and update READMEs for BigQuery v1 (#30) 26 | Extend and update great expectations for BigQuery v1 (#28) 27 | Add BigQuery web model v1 (#26) 28 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/configs/post_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "BigQuery", 6 | "sqlRunner": "0.9.2", 7 | "playbooks": [ 8 | { 9 | "playbook": "standard/01-base/99-base-complete", 10 | "dependsOn": [] 11 | }, 12 | { 13 | "playbook": "standard/02-page-views/99-page-views-complete", 14 | "dependsOn": [] 15 | }, 16 | { 17 | "playbook": "standard/03-sessions/99-sessions-complete", 18 | "dependsOn": [] 19 | }, 20 | { 21 | "playbook": "standard/04-users/99-users-complete", 22 | "dependsOn": [] 23 | } 24 | ], 25 | "lockType": "hard", 26 | "owners": [ 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/configs/pre_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "BigQuery", 6 | "sqlRunner": "0.9.2", 7 | "playbooks": [ 8 | { 9 | "playbook": "standard/01-base/01-base-main", 10 | "dependsOn": [] 11 | }, 12 | { 13 | "playbook": "standard/02-page-views/01-page-views-main", 14 | "dependsOn": ["standard/01-base/01-base-main"] 15 | }, 16 | { 17 | "playbook": "standard/03-sessions/01-sessions-main", 18 | "dependsOn": ["standard/02-page-views/01-page-views-main"] 19 | }, 20 | { 21 | "playbook": "standard/04-users/01-users-main", 22 | "dependsOn": ["standard/03-sessions/01-sessions-main"] 23 | } 24 | ], 25 | "lockType": "hard", 26 | "owners": [ 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/configs/teardown_all.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "BigQuery", 6 | "sqlRunner": "0.9.2", 7 | "playbooks": [ 8 | { 9 | "playbook": "standard/01-base/XX-destroy-base", 10 | "dependsOn": [] 11 | }, 12 | { 13 | "playbook": "standard/02-page-views/XX-destroy-page-views", 14 | "dependsOn": [] 15 | }, 16 | { 17 | "playbook": "standard/03-sessions/XX-destroy-sessions", 18 | "dependsOn": [] 19 | }, 20 | { 21 | "playbook": "standard/04-users/XX-destroy-users", 22 | "dependsOn": [] 23 | } 24 | ], 25 | "lockType": "hard", 26 | "owners": [ 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/custom/02-custom-example/01-custom-example.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :scratch_schema: scratch 8 | :output_schema: derived 9 | :entropy: "" 10 | :steps: 11 | - :name: 01-link-clicks 12 | :queries: 13 | - :name: 01-link-clicks 14 | :file: custom/02-custom-example/01-link-clicks.sql 15 | :template: true 16 | - :name: 02-channel-engagement 17 | :queries: 18 | - :name: 02-channel-engagement 19 | :file: custom/02-custom-example/02-channel-engagement.sql 20 | :template: true 21 | - :name: 03-commit 22 | :queries: 23 | - :name: 03-commit 24 | :file: custom/02-custom-example/03-commit.sql 25 | :template: true 26 | - :name: 99-custom-example-cleanup 27 | :queries: 28 | - :name: 99-custom-example-cleanup 29 | :file: custom/02-custom-example/99-custom-example-cleanup.sql 30 | :template: true 31 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/custom/README.md: -------------------------------------------------------------------------------- 1 | # Custom playbooks 2 | 3 | This directory contains playbooks for the example custom modules [the custom sql directory](../../sql/custom). 4 | 5 | The readme in that directory also contains a guide to adding custom modules. 6 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :steps: 12 | - :name: 00-setup-metadata 13 | :queries: 14 | - :name: 00-setup-metadata 15 | :file: standard/00-setup/01-main/00-setup-metadata.sql 16 | :template: true 17 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :steps: 12 | - :name: 01-cleanup-metadata 13 | :queries: 14 | - :name: 01-cleanup-metadata 15 | :file: standard/00-setup/99-complete/01-cleanup-metadata.sql 16 | :template: true 17 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/00-setup/README.md: -------------------------------------------------------------------------------- 1 | # Setup step playbooks 2 | 3 | These playbooks exist to accommodate scheduling jobs which don't conform to the usual flow of the standard model - they create and destroy the necessary tables to log metadata, with an ID which persists across modules. 4 | 5 | If running the standard model alone, or the standard model alongside custom steps, it is not necessary to run these steps. Instead, we can configure the `:ends_run:` variable to `true` in the `complete` playbook for the last module run in the standard model. 6 | 7 | In a scenario where we don't run the standard module, or we run portions of it on differing schedules, we can run the `00-setup-metadata.ymp.tmpl` playbook as the first step - to set up metadata and create the temporary run ID, and the `99-complete-metadata.yml.tmpl` playbook as the last step - to destroy the temporary run ID. 8 | 9 | If we would like to destroy the metadata tables for a full rebuild of the model, we may run the `XX-destroy-metadata.yml.tmpl` playbook to do so. It is advisable to rename the metadata table instead, however, in case there is some unforeseen need to see that data. 10 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :steps: 12 | - :name: 01-cleanup-metadata 13 | :queries: 14 | - :name: 01-cleanup-metadata 15 | :file: standard/00-setup/99-complete/01-cleanup-metadata.sql 16 | :template: true 17 | - :name: XX-destroy-metadata 18 | :queries: 19 | - :name: XX-destroy-metadata 20 | :file: standard/00-setup/XX-destroy/XX-destroy-metadata.sql 21 | :template: true 22 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 98-base-manifest 15 | :queries: 16 | - :name: 98-base-manifest 17 | :file: standard/01-base/99-complete/98-base-manifest.sql 18 | :template: true 19 | - :name: 99-base-cleanup 20 | :queries: 21 | - :name: 99-base-cleanup 22 | :file: standard/01-base/99-complete/99-base-cleanup.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 99-base-cleanup 15 | :queries: 16 | - :name: 99-base-cleanup 17 | :file: standard/01-base/99-complete/99-base-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-base 20 | :queries: 21 | - :name: XX-destroy-base 22 | :file: standard/01-base/XX-destroy/XX-destroy-base.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: false 13 | :steps: 14 | - :name: 98-truncate-base-staged 15 | :queries: 16 | - :name: 98-truncate-base-staged 17 | :file: standard/02-page-views/99-complete/98-truncate-base-staged.sql 18 | :template: true 19 | - :name: 99-page-views-cleanup 20 | :queries: 21 | - :name: 99-page-views-cleanup 22 | :file: standard/02-page-views/99-complete/99-page-views-cleanup.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 99-page-views-cleanup 15 | :queries: 16 | - :name: 99-page-views-cleanup 17 | :file: standard/02-page-views/99-complete/99-page-views-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-page-views 20 | :queries: 21 | - :name: XX-destroy-page-views 22 | :file: standard/02-page-views/XX-destroy/XX-destroy-page-views.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: false 13 | :steps: 14 | - :name: 98-truncate-page-views-staged 15 | :queries: 16 | - :name: 98-truncate-page-views-staged 17 | :file: standard/03-sessions/99-complete/98-truncate-page-views-staged.sql 18 | :template: true 19 | - :name: 99-sessions-cleanup 20 | :queries: 21 | - :name: 99-sessions-cleanup 22 | :file: standard/03-sessions/99-complete/99-sessions-cleanup.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 99-sessions-cleanup 15 | :queries: 16 | - :name: 99-sessions-cleanup 17 | :file: standard/03-sessions/99-complete/99-sessions-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-sessions 20 | :queries: 21 | - :name: XX-destroy-sessions 22 | :file: standard/03-sessions/XX-destroy/XX-destroy-sessions.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 98-manifest-and-truncate 15 | :queries: 16 | - :name: 98-manifest-and-truncate 17 | :file: standard/04-users/99-complete/98-manifest-and-truncate.sql 18 | :template: true 19 | - :name: 99-users-cleanup 20 | :queries: 21 | - :name: 99-users-cleanup 22 | :file: standard/04-users/99-complete/99-users-cleanup.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: bigquery 4 | :project: 5 | :region: 6 | :variables: 7 | :model_version: bigquery/web/1.0.4 8 | :scratch_schema: scratch 9 | :output_schema: derived 10 | :entropy: "" 11 | :cleanup_mode: all 12 | :ends_run: true 13 | :steps: 14 | - :name: 99-users-cleanup 15 | :queries: 16 | - :name: 99-users-cleanup 17 | :file: standard/04-users/99-complete/99-users-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-users 20 | :queries: 21 | - :name: XX-destroy-users 22 | :file: standard/04-users/XX-destroy/XX-destroy-users.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/custom/02-custom-example/03-commit.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Commit table procedure handles committing to prod, including table creation, and creation of new columns if 'automigrate' is set to TRUE 3 | 4 | CALL {{.output_schema}}.commit_table('{{.scratch_schema}}', -- sourceDataset 5 | 'channel_engagement_staged{{.entropy}}', -- sourceTable 6 | '{{.output_schema}}', -- targetDataset 7 | 'channel_engagement{{.entropy}}', -- targetTable 8 | 'page_view_id', -- joinKey 9 | 'start_tstamp', -- partitionKey 10 | TRUE); -- automigrate 11 | 12 | -- If we like, we can manually create and update our production table instead. 13 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/custom/02-custom-example/99-custom-example-cleanup.sql: -------------------------------------------------------------------------------- 1 | -- We can run this cleanup straight after the commit step if we like, as long as no subsequent logic depends on it. 2 | 3 | DROP TABLE IF EXISTS {{.scratch_schema}}.link_clicks{{.entropy}}; 4 | DROP TABLE IF EXISTS {{.scratch_schema}}.channel_engagement_staged{{.entropy}}; 5 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/00-setup/99-complete/01-cleanup-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.scratch_schema}}.metadata_run_id{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/00-setup/XX-destroy/XX-destroy-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.datamodel_metadata{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/01-base/01-main/05-batch-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a new limit based on this data 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.base_run_limits{{.entropy}} AS( 19 | SELECT 20 | MIN(min_tstamp) AS lower_limit, 21 | (SELECT upper_limit FROM {{.scratch_schema}}.base_new_events_limits{{.entropy}}) AS upper_limit 22 | 23 | FROM 24 | {{.scratch_schema}}.base_sessions_to_include{{.entropy}} 25 | ); 26 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/01-base/XX-destroy/XX-destroy-base.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.base_event_id_manifest{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.output_schema}}.base_session_id_manifest{{.entropy}}; 19 | DROP TABLE IF EXISTS {{.scratch_schema}}.events_staged{{.entropy}}; 20 | DROP FUNCTION IF EXISTS {{.output_schema}}.columnCheckQuery; 21 | DROP PROCEDURE IF EXISTS {{.output_schema}}.commit_table; 22 | DROP PROCEDURE IF EXISTS {{.output_schema}}.combine_context_versions; 23 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/02-page-views/99-complete/98-truncate-base-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | TRUNCATE TABLE {{.scratch_schema}}.events_staged{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/02-page-views/XX-destroy/XX-destroy-page-views.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.page_views{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.page_views_staged{{.entropy}}; 19 | DROP FUNCTION IF EXISTS {{.output_schema}}.columnCheckQuery; 20 | DROP PROCEDURE IF EXISTS {{.output_schema}}.commit_table; 21 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/03-sessions/01-main/01-sessions-aggs.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | CREATE OR REPLACE TABLE {{.scratch_schema}}.sessions_aggregates{{.entropy}} 18 | AS( 19 | SELECT 20 | domain_sessionid, 21 | -- time 22 | MIN(start_tstamp) AS start_tstamp, 23 | MAX(end_tstamp) AS end_tstamp, 24 | 25 | -- engagement 26 | COUNT(DISTINCT page_view_id) AS page_views, 27 | SUM(engaged_time_in_s) AS engaged_time_in_s 28 | 29 | FROM 30 | {{.scratch_schema}}.page_views_staged{{.entropy}} 31 | 32 | GROUP BY 1 33 | ); 34 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/03-sessions/01-main/05-sessions-prep-manifest.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Prep manifest data for users step 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.sessions_userid_manifest_this_run{{.entropy}} 19 | AS( 20 | SELECT 21 | domain_userid, 22 | MIN(start_tstamp) AS min_tstamp 23 | 24 | FROM 25 | {{.scratch_schema}}.sessions_this_run{{.entropy}} 26 | 27 | GROUP BY 1 28 | ); 29 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/03-sessions/99-complete/98-truncate-page-views-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | TRUNCATE TABLE {{.scratch_schema}}.page_views_staged{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/03-sessions/XX-destroy/XX-destroy-sessions.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.sessions{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}}; 19 | DROP FUNCTION IF EXISTS {{.output_schema}}.columnCheckQuery; 20 | DROP PROCEDURE IF EXISTS {{.output_schema}}.commit_table; 21 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/04-users/01-main/02-users-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a limit for this run 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.users_limits{{.entropy}} AS( 19 | SELECT 20 | MIN(start_tstamp) AS lower_limit, 21 | MAX(start_tstamp) AS upper_limit 22 | 23 | FROM 24 | {{.scratch_schema}}.users_userids_this_run{{.entropy}} 25 | ); 26 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/04-users/01-main/04-users-aggs.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | CREATE OR REPLACE TABLE {{.scratch_schema}}.users_aggregates{{.entropy}} 18 | AS( 19 | SELECT 20 | domain_userid, 21 | 22 | -- time 23 | MIN(start_tstamp) AS start_tstamp, 24 | MAX(end_tstamp) AS end_tstamp, 25 | 26 | -- engagement 27 | SUM(page_views) AS page_views, 28 | COUNT(DISTINCT domain_sessionid) AS sessions, 29 | SUM(engaged_time_in_s) AS engaged_time_in_s 30 | 31 | FROM {{.scratch_schema}}.users_sessions_this_run{{.entropy}} 32 | 33 | GROUP BY 1 34 | ); 35 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/04-users/99-complete/98-manifest-and-truncate.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | BEGIN 18 | 19 | -- Update manifest 20 | DELETE 21 | FROM {{.output_schema}}.users_manifest{{.entropy}} 22 | WHERE domain_userid IN (SELECT domain_userid FROM {{.scratch_schema}}.users_userids_this_run{{.entropy}}); 23 | 24 | INSERT INTO {{.output_schema}}.users_manifest{{.entropy}} (SELECT * FROM {{.scratch_schema}}.users_userids_this_run{{.entropy}}); 25 | 26 | -- Truncate input table just processed 27 | TRUNCATE TABLE {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}}; 28 | 29 | END; 30 | -------------------------------------------------------------------------------- /web/v1/bigquery/sql-runner/sql/standard/04-users/XX-destroy/XX-destroy-users.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.users{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.output_schema}}.users_manifest{{.entropy}}; 19 | DROP FUNCTION IF EXISTS {{.output_schema}}.columnCheckQuery; 20 | DROP PROCEDURE IF EXISTS {{.output_schema}}.commit_table; 21 | -------------------------------------------------------------------------------- /web/v1/redshift/CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 1.3.1 (2022-09-02) 2 | -------------------------- 3 | Redshift Web: load_tstamp missing from table definition (#135) 4 | 5 | Version 1.3.0 (2022-06-07) 6 | -------------------------- 7 | Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!) 8 | Redshift Web: Fix column lengths in manifest tables (#131) 9 | 10 | Version 1.2.0 (2021-01-25) 11 | --------------------------- 12 | Redshift: Add config (#12) 13 | Redshift: Update test on page_view_in_session_index (#13) 14 | Redshift: Rename 02-page-views/setup.sql and /limits.sql to include module name (#8) 15 | Redshift: Rename metadata tables and columns (#21) 16 | Redshift: Rename variables to include unit of measurement (#23) 17 | Redshift: Remove scan limit from engagement time step (#5) 18 | Redshift: Remove NOT NULL constraint on page_view_id in base (#20) 19 | Redshift: Fix table definitions (#10) 20 | Redshift: Remove or replace second example customisation (#14) 21 | Redshift: Update copyright notices (#19) 22 | Add CHANGELOG (#6) 23 | 24 | Version 1.1.0 (2020-11-05) 25 | --------------------------- 26 | Add Redshift web model v1 (#4) 27 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/configs/migrations.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "playbooks": [ 7 | { 8 | "playbook": "standard/99-migrations/1.2.0-migration", 9 | "dependsOn": [] 10 | }, 11 | { 12 | "playbook": "standard/99-migrations/1.3.0-migration", 13 | "dependsOn": [] 14 | } 15 | ], 16 | "lockType": "hard", 17 | "owners": [ 18 | ] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/configs/post_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "playbooks": [ 7 | { 8 | "playbook": "standard/01-base/99-base-complete", 9 | "dependsOn": [] 10 | }, 11 | { 12 | "playbook": "standard/02-page-views/99-page-views-complete", 13 | "dependsOn": [] 14 | }, 15 | { 16 | "playbook": "standard/03-sessions/99-sessions-complete", 17 | "dependsOn": [] 18 | }, 19 | { 20 | "playbook": "standard/04-users/99-users-complete", 21 | "dependsOn": [] 22 | } 23 | ], 24 | "lockType": "hard", 25 | "owners": [ 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/configs/pre_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "playbooks": [ 7 | { 8 | "playbook": "standard/01-base/01-base-main", 9 | "dependsOn": [] 10 | }, 11 | { 12 | "playbook": "standard/02-page-views/01-page-views-main", 13 | "dependsOn": ["standard/01-base/01-base-main"] 14 | }, 15 | { 16 | "playbook": "standard/03-sessions/01-sessions-main", 17 | "dependsOn": ["standard/02-page-views/01-page-views-main"] 18 | }, 19 | { 20 | "playbook": "standard/04-users/01-users-main", 21 | "dependsOn": ["standard/03-sessions/01-sessions-main"] 22 | } 23 | ], 24 | "lockType": "hard", 25 | "owners": [ 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/configs/teardown_all.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "playbooks": [ 7 | { 8 | "playbook": "standard/01-base/XX-destroy-base", 9 | "dependsOn": [] 10 | }, 11 | { 12 | "playbook": "standard/02-page-views/XX-destroy-page-views", 13 | "dependsOn": [] 14 | }, 15 | { 16 | "playbook": "standard/03-sessions/XX-destroy-sessions", 17 | "dependsOn": [] 18 | }, 19 | { 20 | "playbook": "standard/04-users/XX-destroy-users", 21 | "dependsOn": [] 22 | } 23 | ], 24 | "lockType": "hard", 25 | "owners": [ 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/custom/02-page-views-join/01-page-views-join.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :input_schema: atomic 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :steps: 16 | - :name: 01-page-views-join-setup 17 | :queries: 18 | - :name: 01-page-views-join-setup 19 | :file: custom/02-page-views-join/01-page-views-join-setup.sql 20 | :template: true 21 | - :name: 02-page-views-join-staged 22 | :queries: 23 | - :name: 02-page-views-join-staged 24 | :file: custom/02-page-views-join/02-page-views-join-staged.sql 25 | :template: true 26 | - :name: 03-page-views-join-upsert 27 | :queries: 28 | - :name: 03-page-views-join-upsert 29 | :file: custom/02-page-views-join/03-page-views-join-upsert.sql 30 | :template: true 31 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/custom/02-page-views-join/99-page-views-join-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :input_schema: atomic 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :steps: 16 | - :name: 99-page-views-join-cleanup 17 | :queries: 18 | - :name: 99-page-views-join-cleanup 19 | :file: custom/02-page-views-join/99-page-views-join-cleanup.sql 20 | :template: true 21 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/custom/README.md: -------------------------------------------------------------------------------- 1 | # Custom playbooks 2 | 3 | This directory contains playbooks for the example custom modules [the custom sql directory](../../sql/custom). 4 | 5 | The readme in that directory also contains a guide to adding custom modules. 6 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :steps: 16 | - :name: 00-setup-metadata 17 | :queries: 18 | - :name: 00-setup-metadata 19 | :file: standard/00-setup/01-main/00-setup-metadata.sql 20 | :template: true 21 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :steps: 16 | - :name: 01-cleanup-metadata 17 | :queries: 18 | - :name: 01-cleanup-metadata 19 | :file: standard/00-setup/99-complete/01-cleanup-metadata.sql 20 | :template: true 21 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/00-setup/README.md: -------------------------------------------------------------------------------- 1 | # Setup step playbooks 2 | 3 | These playbooks exist to accommodate scheduling jobs which don't conform to the usual flow of the standard model - they create and destroy the necessary tables to log metadata, with an ID which persists across modules. 4 | 5 | If running the standard model alone, or the standard model alongside custom steps, it is not necessary to run these steps. Instead, we can configure the `:ends_run:` variable to `true` in the `complete` playbook for the last module run in the standard model. 6 | 7 | In a scenario where we don't run the standard module, or we run portions of it on differing schedules, we can run the `00-setup-metadata.ymp.tmpl` playbook as the first step - to set up metadata and create the temporary run ID, and the `99-complete-metadata.yml.tmpl` playbook as the last step - to destroy the temporary run ID. 8 | 9 | If we would like to destroy the metadata tables for a full rebuild of the model, we may run the `XX-destroy-metadata.yml.tmpl` playbook to do so. It is advisable to rename the metadata table instead, however, in case there is some unforeseen need to see that data. 10 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :steps: 16 | - :name: 01-cleanup-metadata 17 | :queries: 18 | - :name: 01-cleanup-metadata 19 | :file: standard/00-setup/99-complete/01-cleanup-metadata.sql 20 | :template: true 21 | - :name: XX-destroy-metadata 22 | :queries: 23 | - :name: XX-destroy-metadata 24 | :file: standard/00-setup/XX-destroy/XX-destroy-metadata.sql 25 | :template: true 26 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: false 17 | :steps: 18 | - :name: 98-base-manifest 19 | :queries: 20 | - :name: 98-base-manifest 21 | :file: standard/01-base/99-complete/98-base-manifest.sql 22 | :template: true 23 | - :name: 99-base-cleanup 24 | :queries: 25 | - :name: 99-base-cleanup 26 | :file: standard/01-base/99-complete/99-base-cleanup.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-base-cleanup 19 | :queries: 20 | - :name: 99-base-cleanup 21 | :file: standard/01-base/99-complete/99-base-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-base 24 | :queries: 25 | - :name: XX-destroy-base 26 | :file: standard/01-base/XX-destroy/XX-destroy-base.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: false 17 | :steps: 18 | - :name: 98-drop-base-staged 19 | :queries: 20 | - :name: 98-drop-base-staged 21 | :file: standard/02-page-views/99-complete/98-drop-base-staged.sql 22 | :template: true 23 | - :name: 99-page-views-cleanup 24 | :queries: 25 | - :name: 99-page-views-cleanup 26 | :file: standard/02-page-views/99-complete/99-page-views-cleanup.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-page-views-cleanup 19 | :queries: 20 | - :name: 99-page-views-cleanup 21 | :file: standard/02-page-views/99-complete/99-page-views-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-page-views 24 | :queries: 25 | - :name: XX-destroy-page-views 26 | :file: standard/02-page-views/XX-destroy/XX-destroy-page-views.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: false 17 | :steps: 18 | - :name: 98-truncate-page-views-staged 19 | :queries: 20 | - :name: 98-truncate-page-views-staged 21 | :file: standard/03-sessions/99-complete/98-truncate-page-views-staged.sql 22 | :template: true 23 | - :name: 99-sessions-cleanup 24 | :queries: 25 | - :name: 99-sessions-cleanup 26 | :file: standard/03-sessions/99-complete/99-sessions-cleanup.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-sessions-cleanup 19 | :queries: 20 | - :name: 99-sessions-cleanup 21 | :file: standard/03-sessions/99-complete/99-sessions-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-sessions 24 | :queries: 25 | - :name: XX-destroy-sessions 26 | :file: standard/03-sessions/XX-destroy/XX-destroy-sessions.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: false 17 | :steps: 18 | - :name: 98-manifest-and-truncate 19 | :queries: 20 | - :name: 98-manifest-and-truncate 21 | :file: standard/04-users/99-complete/98-manifest-and-truncate.sql 22 | :template: true 23 | - :name: 99-users-cleanup 24 | :queries: 25 | - :name: 99-users-cleanup 26 | :file: standard/04-users/99-complete/99-users-cleanup.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :model_version: redshift/web/1.3.1 12 | :scratch_schema: scratch 13 | :output_schema: derived 14 | :entropy: "" 15 | :cleanup_mode: all 16 | :ends_run: true 17 | :steps: 18 | - :name: 99-users-cleanup 19 | :queries: 20 | - :name: 99-users-cleanup 21 | :file: standard/04-users/99-complete/99-users-cleanup.sql 22 | :template: true 23 | - :name: XX-destroy-users 24 | :queries: 25 | - :name: XX-destroy-users 26 | :file: standard/04-users/XX-destroy/XX-destroy-users.sql 27 | :template: true 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/99-migrations/1.2.0-migration.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 1.2.0-migration 16 | :queries: 17 | - :name: events-staged 18 | :file: standard/99-migrations/1.2.0-migration/events-staged.sql 19 | :template: true -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/playbooks/standard/99-migrations/1.3.0-migration.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: redshift 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 1.3.0-migration 16 | :queries: 17 | - :name: base-session-id-manifest 18 | :file: standard/99-migrations/1.3.0-migration/base-session-id-manifest.sql 19 | :template: true 20 | - :name: 1.3.0-migration 21 | :queries: 22 | - :name: sessions-userid-manifest 23 | :file: standard/99-migrations/1.3.0-migration/sessions-userid-manifest.sql 24 | :template: true 25 | - :name: 1.3.0-migration 26 | :queries: 27 | - :name: users-manifest 28 | :file: standard/99-migrations/1.3.0-migration/users-manifest.sql 29 | :template: true 30 | - :name: 1.3.0-migration 31 | :queries: 32 | - :name: sortkey-encoding 33 | :file: standard/99-migrations/1.3.0-migration/sortkey-encoding.sql 34 | :template: true -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/custom/02-page-views-join/01-page-views-join-setup.sql: -------------------------------------------------------------------------------- 1 | -- 1. Create prod table 2 | 3 | CREATE TABLE IF NOT EXISTS {{.output_schema}}.page_views_join{{.entropy}} ( 4 | page_view_id CHAR(36), 5 | link_clicks INT, 6 | first_link_target CHAR(2000), 7 | bounced_page_view BOOLEAN, 8 | engagement_score FLOAT, 9 | channel CHAR(255) 10 | ); 11 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/custom/02-page-views-join/03-page-views-join-upsert.sql: -------------------------------------------------------------------------------- 1 | -- 3. DELETE - INSERT to production (and optionally drop the temp table) 2 | 3 | BEGIN; --it is safest to use a transaction, in case of failure. 4 | 5 | DELETE FROM {{.output_schema}}.page_views_join{{.entropy}} 6 | WHERE page_view_id IN (SELECT page_view_id FROM {{.scratch_schema}}.page_views_join_staged{{.entropy}}); 7 | 8 | INSERT INTO {{.output_schema}}.page_views_join{{.entropy}} 9 | (SELECT * FROM {{.scratch_schema}}.page_views_join_staged{{.entropy}}); 10 | 11 | END; 12 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/custom/02-page-views-join/99-page-views-join-cleanup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS {{.scratch_schema}}.page_views_join_staged{{.entropy}}; 2 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/00-setup/99-complete/01-cleanup-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.scratch_schema}}.metadata_run_id{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/00-setup/XX-destroy/XX-destroy-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.datamodel_metadata{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/01-base/01-main/05-batch-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a new limit based on this data 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.base_run_limits{{.entropy}}; 19 | 20 | CREATE TABLE {{.scratch_schema}}.base_run_limits{{.entropy}} AS( 21 | SELECT 22 | MIN(min_tstamp) AS lower_limit, 23 | (SELECT upper_limit FROM {{.scratch_schema}}.base_new_events_limits{{.entropy}}) AS upper_limit 24 | 25 | FROM 26 | {{.scratch_schema}}.base_sessions_to_include{{.entropy}} 27 | ); 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/01-base/XX-destroy/XX-destroy-base.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.base_event_id_manifest{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.output_schema}}.base_session_id_manifest{{.entropy}}; 19 | DROP TABLE IF EXISTS {{.scratch_schema}}.events_staged{{.entropy}}; 20 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/02-page-views/01-main/01-page-view-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a limit for this run - single row table. 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.pv_run_limits{{.entropy}}; 19 | 20 | CREATE TABLE {{.scratch_schema}}.pv_run_limits{{.entropy}} AS( 21 | SELECT 22 | MIN(collector_tstamp) AS lower_limit, 23 | MAX(collector_tstamp) AS upper_limit 24 | 25 | FROM 26 | {{.scratch_schema}}.events_staged{{.entropy}} 27 | WHERE page_view_id IS NOT NULL 28 | ); 29 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/02-page-views/99-complete/98-drop-base-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE {{.scratch_schema}}.events_staged{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/02-page-views/XX-destroy/XX-destroy-page-views.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.page_views{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.page_views_staged{{.entropy}}; 19 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/03-sessions/01-main/05-sessions-prep-manifest.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Prep manifest data for users step 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.sessions_userid_manifest_this_run{{.entropy}}; 19 | 20 | CREATE TABLE {{.scratch_schema}}.sessions_userid_manifest_this_run{{.entropy}} 21 | DISTSTYLE KEY 22 | DISTKEY (domain_userid) 23 | SORTKEY (domain_userid) 24 | AS( 25 | SELECT 26 | domain_userid, 27 | MIN(start_tstamp) AS min_tstamp 28 | 29 | FROM 30 | {{.scratch_schema}}.sessions_this_run{{.entropy}} 31 | 32 | GROUP BY 1 33 | ) 34 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/03-sessions/99-complete/98-truncate-page-views-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | TRUNCATE {{.scratch_schema}}.page_views_staged{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/03-sessions/XX-destroy/XX-destroy-sessions.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.sessions{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}}; 19 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/04-users/01-main/02-users-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -- Create a limit for this run - single value table. 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.users_limits{{.entropy}}; 19 | 20 | CREATE TABLE {{.scratch_schema}}.users_limits{{.entropy}} AS( 21 | SELECT 22 | MIN(start_tstamp) AS lower_limit, 23 | MAX(start_tstamp) AS upper_limit 24 | 25 | FROM 26 | {{.scratch_schema}}.users_userids_this_run{{.entropy}} 27 | ); 28 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/04-users/99-complete/98-manifest-and-truncate.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | BEGIN; 18 | -- Update manifest 19 | DELETE 20 | FROM {{.output_schema}}.users_manifest{{.entropy}} 21 | WHERE domain_userid IN (SELECT domain_userid FROM {{.scratch_schema}}.users_userids_this_run{{.entropy}}); 22 | 23 | INSERT INTO {{.output_schema}}.users_manifest{{.entropy}} (SELECT * FROM {{.scratch_schema}}.users_userids_this_run{{.entropy}}); 24 | 25 | -- Truncate input table just processed 26 | TRUNCATE {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}}; 27 | 28 | END; 29 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/04-users/XX-destroy/XX-destroy-users.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020-2021 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.users{{.entropy}}; 18 | DROP TABLE IF EXISTS {{.output_schema}}.users_manifest{{.entropy}}; 19 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/99-migrations/1.2.0-migration/events-staged.sql: -------------------------------------------------------------------------------- 1 | -- Define stored procedure so we can do a safety check 2 | CREATE OR REPLACE PROCEDURE drop_events_staged_if_empty() 3 | AS $$ 4 | DECLARE 5 | rows_in_table int; 6 | BEGIN 7 | SELECT INTO rows_in_table COUNT(*) FROM {{.scratch_schema}}.events_staged{{.entropy}}; 8 | IF rows_in_table > 0 THEN 9 | RAISE EXCEPTION 'Ensure that all data has been processed from events_staged and it is empty.'; 10 | ELSE 11 | DROP TABLE {{.scratch_schema}}.events_staged{{.entropy}}; 12 | END IF; 13 | END; 14 | $$ LANGUAGE plpgsql; 15 | 16 | -- Call the procedure 17 | CALL drop_events_staged_if_empty(); 18 | 19 | -- Drop the procedure. 20 | DROP PROCEDURE drop_events_staged_if_empty(); -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/base-session-id-manifest.sql: -------------------------------------------------------------------------------- 1 | -- Extend column lengths to match their correct values 2 | -- Since ALTER TABLE ALTER COLUMN cannot run inside a multiple command statement, we need a file each. 3 | ALTER TABLE {{.output_schema}}.base_session_id_manifest{{.entropy}} ALTER COLUMN session_id type VARCHAR(128); 4 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/sessions-userid-manifest.sql: -------------------------------------------------------------------------------- 1 | -- Extend column lengths to match their correct values 2 | -- Since ALTER TABLE ALTER COLUMN cannot run inside a multiple command statement, we need a file each. 3 | ALTER TABLE {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}} ALTER COLUMN domain_userid type VARCHAR(128); 4 | -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/sortkey-encoding.sql: -------------------------------------------------------------------------------- 1 | -- Change sortkey encodings to RAW 2 | ALTER TABLE {{.output_schema}}.page_views{{.entropy}} ALTER COLUMN start_tstamp ENCODE RAW; 3 | 4 | ALTER TABLE {{.scratch_schema}}.page_views_staged{{.entropy}} ALTER COLUMN start_tstamp ENCODE RAW; 5 | 6 | ALTER TABLE {{.output_schema}}.sessions{{.entropy}} ALTER COLUMN start_tstamp ENCODE RAW; 7 | 8 | ALTER TABLE {{.output_schema}}.users{{.entropy}} ALTER COLUMN start_tstamp ENCODE RAW; -------------------------------------------------------------------------------- /web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/users-manifest.sql: -------------------------------------------------------------------------------- 1 | -- Extend column lengths to match their correct values 2 | -- Since ALTER TABLE ALTER COLUMN cannot run inside a multiple command statement, we need a file each. 3 | ALTER TABLE {{.output_schema}}.users_manifest{{.entropy}} ALTER COLUMN domain_userid type VARCHAR(128); -------------------------------------------------------------------------------- /web/v1/snowflake/CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 1.0.3 (2025-01-08) 2 | ---------------------------------------- 3 | Snowflake Web: Remove length limits from varchar fields 4 | 5 | Version 1.0.2 (2022-10-13) 6 | -------------------------- 7 | Snowflake web: Optimise partition pruning (Close #140) 8 | 9 | Version 1.0.1 (2022-02-24) 10 | -------------------------- 11 | Snowflake web: Update column check stored procedure (Close #125) 12 | Snowflake web: Fix varchar length for pseudonymized fields (Close #122) 13 | Snowflake web: Remove start_date variable from users module (Close #123) 14 | Snowflake web: Update copyright notices (Close #124) 15 | Snowflake web: Fix logic in users_sessions_this_run to account for sparse data (Close #120) 16 | Snowflake web: Fix varchar length for yauaa columns (Close #97) 17 | Snowflake web: Fix se_label column length in events_staged (Close #109) 18 | 19 | Version 1.0.0 (2021-02-11) 20 | -------------------------- 21 | Add Snowflake web model v1 (#31) 22 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/configs/migrations.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "dagName": "standard-webmodel", 5 | "enabled": true, 6 | "storage": "Default", 7 | "sqlRunner": "0.9.3", 8 | "lockType": "hard", 9 | "playbooks": [ 10 | { 11 | "playbook": "standard/99-migrations/1.0.1-migration", 12 | "dependsOn": [] 13 | } 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/configs/post_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "sqlRunner": "0.9.3", 7 | "lockType": "hard", 8 | "playbooks": [ 9 | { 10 | "playbook": "standard/01-base/99-base-complete", 11 | "dependsOn": [] 12 | }, 13 | { 14 | "playbook": "standard/02-page-views/99-page-views-complete", 15 | "dependsOn": [ 16 | "standard/01-base/99-base-complete" 17 | ] 18 | }, 19 | { 20 | "playbook": "standard/03-sessions/99-sessions-complete", 21 | "dependsOn": [ 22 | "standard/02-page-views/99-page-views-complete" 23 | ] 24 | }, 25 | { 26 | "playbook": "standard/04-users/99-users-complete", 27 | "dependsOn": [ 28 | "standard/03-sessions/99-sessions-complete" 29 | ] 30 | } 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/configs/pre_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "enabled": true, 5 | "storage": "Default", 6 | "sqlRunner": "0.9.3", 7 | "lockType": "hard", 8 | "playbooks": [ 9 | { 10 | "playbook": "standard/01-base/01-base-main", 11 | "dependsOn": [] 12 | }, 13 | { 14 | "playbook": "standard/02-page-views/01-page-views-main", 15 | "dependsOn": [ 16 | "standard/01-base/01-base-main" 17 | ] 18 | }, 19 | { 20 | "playbook": "standard/03-sessions/01-sessions-main", 21 | "dependsOn": [ 22 | "standard/02-page-views/01-page-views-main" 23 | ] 24 | }, 25 | { 26 | "playbook": "standard/04-users/01-users-main", 27 | "dependsOn": [ 28 | "standard/03-sessions/01-sessions-main" 29 | ] 30 | } 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/configs/teardown_all.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", 3 | "data": { 4 | "dagName": "XX-destroy-standard-webmodel", 5 | "enabled": false, 6 | "storage": "Default", 7 | "sqlRunner": "0.9.3", 8 | "lockType": "hard", 9 | "playbooks": [ 10 | { 11 | "playbook": "standard/01-base/XX-destroy-base", 12 | "dependsOn": [] 13 | }, 14 | { 15 | "playbook": "standard/02-page-views/XX-destroy-page-views", 16 | "dependsOn": [ 17 | "standard/01-base/XX-destroy-base" 18 | ] 19 | }, 20 | { 21 | "playbook": "standard/03-sessions/XX-destroy-sessions", 22 | "dependsOn": [ 23 | "standard/02-page-views/XX-destroy-page-views" 24 | ] 25 | }, 26 | { 27 | "playbook": "standard/04-users/XX-destroy-users", 28 | "dependsOn": [ 29 | "standard/03-sessions/XX-destroy-sessions" 30 | ] 31 | } 32 | ] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/custom/02-page-views-join/01-page-views-join.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :scratch_schema: scratch 11 | :output_schema: derived 12 | :entropy: "" 13 | :steps: 14 | - :name: 00-setup-page-views-join 15 | :queries: 16 | - :name: 00-setup-page-views-join 17 | :file: custom/02-page-views-join/01-main/00-setup-page-views-join.sql 18 | :template: true 19 | - :name: 01-page-views-join-staged 20 | :queries: 21 | - :name: 01-page-views-join-staged 22 | :file: custom/02-page-views-join/01-main/01-page-views-join-staged.sql 23 | :template: true 24 | - :name: 02-page-views-join-upsert 25 | :queries: 26 | - :name: 02-page-views-join-upsert 27 | :file: custom/02-page-views-join/01-main/02-page-views-join-upsert.sql 28 | :template: true 29 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/custom/02-page-views-join/99-page-views-join-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :scratch_schema: scratch 11 | :entropy: "" 12 | :steps: 13 | - :name: 99-page-views-join-cleanup 14 | :queries: 15 | - :name: 99-page-views-join-cleanup 16 | :file: custom/02-page-views-join/99-complete/99-page-views-join-cleanup.sql 17 | :template: true 18 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/custom/02-page-views-join/XX-destroy-page-views-join.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :scratch_schema: scratch 11 | :output_schema: derived 12 | :entropy: "" 13 | :steps: 14 | - :name: 99-page-views-join-cleanup 15 | :queries: 16 | - :name: 99-page-views-join-cleanup 17 | :file: custom/02-page-views-join/99-complete/99-page-views-join-cleanup.sql 18 | :template: true 19 | - :name: XX-destroy-page-views-join 20 | :queries: 21 | - :name: XX-destroy-page-views-join 22 | :file: custom/02-page-views-join/XX-destroy/XX-destroy-page-views-join.sql 23 | :template: true 24 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/custom/README.md: -------------------------------------------------------------------------------- 1 | # Custom playbooks 2 | 3 | This directory contains playbooks for the example custom modules of [the custom sql directory](../../sql/custom). 4 | 5 | The [README](../../sql/custom/README.md) in that directory also contains a guide to adding custom modules. 6 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 00-setup-metadata 16 | :queries: 17 | - :name: 00-setup-metadata 18 | :file: standard/00-setup/01-main/00-setup-metadata.sql 19 | :template: true 20 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 99-metadata-cleanup 16 | :queries: 17 | - :name: 99-metadata-cleanup 18 | :file: standard/00-setup/99-complete/99-metadata-cleanup.sql 19 | :template: true 20 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/00-setup/README.md: -------------------------------------------------------------------------------- 1 | # Setup step playbooks 2 | 3 | These playbooks exist to accommodate scheduling jobs which don't conform to the usual flow of the standard model - they create and destroy the necessary tables to log metadata, with an ID which persists across modules. 4 | 5 | If running the standard model alone, or the standard model alongside custom steps, it is not necessary to run these steps. Instead, we can configure the `:ends_run:` variable to `true` in the `complete` playbook for the last module run in the standard model. 6 | 7 | In a scenario where we don't run the standard module, or we run portions of it on differing schedules, we can run the `00-setup-metadata.ymp.tmpl` playbook as the first step - to set up metadata and create the temporary run ID, and the `99-complete-metadata.yml.tmpl` playbook as the last step - to destroy the temporary run ID. 8 | 9 | If we would like to destroy the metadata tables for a full rebuild of the model, we may run the `XX-destroy-metadata.yml.tmpl` playbook to do so. It is advisable to rename the metadata table instead, however, in case there is some unforeseen need to see that data. 10 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 99-metadata-cleanup 16 | :queries: 17 | - :name: 99-metadata-cleanup 18 | :file: standard/00-setup/99-complete/99-metadata-cleanup.sql 19 | :template: true 20 | - :name: XX-destroy-metadata 21 | :queries: 22 | - :name: XX-destroy-metadata 23 | :file: standard/00-setup/XX-destroy/XX-destroy-metadata.sql 24 | :template: true 25 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: false 16 | :steps: 17 | - :name: 98-base-manifest 18 | :queries: 19 | - :name: 98-base-manifest 20 | :file: standard/01-base/99-complete/98-base-manifest.sql 21 | :template: true 22 | - :name: 99-base-cleanup 23 | :queries: 24 | - :name: 99-base-cleanup 25 | :file: standard/01-base/99-complete/99-base-cleanup.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 99-base-cleanup 18 | :queries: 19 | - :name: 99-base-cleanup 20 | :file: standard/01-base/99-complete/99-base-cleanup.sql 21 | :template: true 22 | - :name: XX-destroy-base 23 | :queries: 24 | - :name: XX-destroy-base 25 | :file: standard/01-base/XX-destroy/XX-destroy-base.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: false 16 | :steps: 17 | - :name: 98-truncate-base-staged 18 | :queries: 19 | - :name: 98-truncate-base-staged 20 | :file: standard/02-page-views/99-complete/98-truncate-base-staged.sql 21 | :template: true 22 | - :name: 99-page-views-cleanup 23 | :queries: 24 | - :name: 99-page-views-cleanup 25 | :file: standard/02-page-views/99-complete/99-page-views-cleanup.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: false 16 | :steps: 17 | - :name: 99-page-views-cleanup 18 | :queries: 19 | - :name: 99-page-views-cleanup 20 | :file: standard/02-page-views/99-complete/99-page-views-cleanup.sql 21 | :template: true 22 | - :name: XX-destroy-page-views 23 | :queries: 24 | - :name: XX-destroy-page-views 25 | :file: standard/02-page-views/XX-destroy/XX-destroy-page-views.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: false 16 | :steps: 17 | - :name: 98-truncate-page-views-staged 18 | :queries: 19 | - :name: 98-truncate-page-views-staged 20 | :file: standard/03-sessions/99-complete/98-truncate-page-views-staged.sql 21 | :template: true 22 | - :name: 99-sessions-cleanup 23 | :queries: 24 | - :name: 99-sessions-cleanup 25 | :file: standard/03-sessions/99-complete/99-sessions-cleanup.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 99-sessions-cleanup 18 | :queries: 19 | - :name: 99-sessions-cleanup 20 | :file: standard/03-sessions/99-complete/99-sessions-cleanup.sql 21 | :template: true 22 | - :name: XX-destroy-sessions 23 | :queries: 24 | - :name: XX-destroy-sessions 25 | :file: standard/03-sessions/XX-destroy/XX-destroy-sessions.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 98-manifest-and-truncate 18 | :queries: 19 | - :name: 98-manifest-and-truncate 20 | :file: standard/04-users/99-complete/98-manifest-and-truncate.sql 21 | :template: true 22 | - :name: 99-users-cleanup 23 | :queries: 24 | - :name: 99-users-cleanup 25 | :file: standard/04-users/99-complete/99-users-cleanup.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :account: 5 | :database: 6 | :warehouse: 7 | :username: 8 | :password: 9 | :variables: 10 | :model_version: snowflake/web/1.0.3 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :cleanup_mode: all 15 | :ends_run: true 16 | :steps: 17 | - :name: 99-users-cleanup 18 | :queries: 19 | - :name: 99-users-cleanup 20 | :file: standard/04-users/99-complete/99-users-cleanup.sql 21 | :template: true 22 | - :name: XX-destroy-users 23 | :queries: 24 | - :name: XX-destroy-users 25 | :file: standard/04-users/XX-destroy/XX-destroy-users.sql 26 | :template: true 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/playbooks/standard/99-migrations/1.0.3-migration.yml.tmpl: -------------------------------------------------------------------------------- 1 | :targets: 2 | - :name: 3 | :type: snowflake 4 | :host: 5 | :database: 6 | :port: 7 | :username: 8 | :password: 9 | :ssl: 10 | :variables: 11 | :scratch_schema: scratch 12 | :output_schema: derived 13 | :entropy: "" 14 | :steps: 15 | - :name: 1.0.3-migration 16 | :queries: 17 | - :name: alter-tables 18 | :file: standard/99-migrations/1.0.3-migration/alter-tables.sql 19 | :template: true 20 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/custom/02-page-views-join/01-main/00-setup-page-views-join.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CREATE TABLE IF NOT EXISTS {{.output_schema}}.page_views_join{{.entropy}} ( 19 | 20 | page_view_id VARCHAR NOT NULL, 21 | start_tstamp TIMESTAMP_NTZ, 22 | link_clicks INTEGER, 23 | first_link_target VARCHAR, 24 | bounced_page_view BOOLEAN, 25 | engagement_score DOUBLE PRECISION, 26 | channel VARCHAR 27 | ); 28 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/custom/02-page-views-join/01-main/02-page-views-join-upsert.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CALL {{.output_schema}}.mk_transaction( 19 | ' 20 | DELETE FROM {{.output_schema}}.page_views_join{{.entropy}} 21 | WHERE 22 | page_view_id IN (SELECT page_view_id FROM {{.scratch_schema}}.page_views_join_staged{{.entropy}}); 23 | 24 | INSERT INTO {{.output_schema}}.page_views_join{{.entropy}} 25 | SELECT * FROM {{.scratch_schema}}.page_views_join_staged{{.entropy}}; 26 | ' 27 | ); 28 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/custom/02-page-views-join/99-complete/99-page-views-join-cleanup.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | DROP TABLE IF EXISTS {{.scratch_schema}}.page_views_join_staged{{.entropy}}; 19 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/custom/02-page-views-join/XX-destroy/XX-destroy-page-views-join.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | DROP TABLE IF EXISTS {{.output_schema}}.page_views_join{{.entropy}}; 19 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/00-setup/99-complete/99-metadata-cleanup.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.scratch_schema}}.metadata_run_id{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/00-setup/XX-destroy/XX-destroy-metadata.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | DROP TABLE IF EXISTS {{.output_schema}}.datamodel_metadata{{.entropy}}; 18 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/01-base/01-main/01-new-events-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | -- Create a limit for this run - single row table. 19 | CREATE OR REPLACE TABLE {{.scratch_schema}}.base_new_events_limits{{.entropy}} 20 | AS ( 21 | SELECT 22 | TIMEADD(HOUR, -{{or .lookback_window_hours 6}}, MAX(collector_tstamp)) AS lower_limit, 23 | TIMEADD(DAY, {{or .update_cadence_days 7}}, MAX(collector_tstamp)) AS upper_limit, 24 | TIMEADD(DAY, -{{or .session_lookback_days 365}}, MAX(collector_tstamp)) AS session_limit 25 | 26 | FROM 27 | {{.output_schema}}.base_event_id_manifest{{.entropy}} 28 | ); 29 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/01-base/01-main/05-batch-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | -- Create a new limit based on this data 19 | CREATE OR REPLACE TABLE {{.scratch_schema}}.base_run_limits{{.entropy}} 20 | AS ( 21 | SELECT 22 | MIN(min_tstamp) AS lower_limit, 23 | (SELECT upper_limit FROM {{.scratch_schema}}.base_new_events_limits{{.entropy}}) AS upper_limit 24 | 25 | FROM 26 | {{.scratch_schema}}.base_sessions_to_include{{.entropy}} 27 | ); 28 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/02-page-views/99-complete/98-truncate-base-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | TRUNCATE TABLE {{.scratch_schema}}.events_staged{{.entropy}}; 19 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/03-sessions/01-main/01-sessions-aggs.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.sessions_aggregates{{.entropy}} 19 | AS ( 20 | SELECT 21 | domain_sessionid, 22 | 23 | -- time 24 | MIN(start_tstamp) AS start_tstamp, 25 | MAX(end_tstamp) AS end_tstamp, 26 | 27 | -- engagement 28 | COUNT(DISTINCT page_view_id) AS page_views, 29 | SUM(engaged_time_in_s) AS engaged_time_in_s 30 | 31 | FROM 32 | {{.scratch_schema}}.page_views_staged{{.entropy}} 33 | 34 | GROUP BY 1 35 | ); 36 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/03-sessions/01-main/05-sessions-prep-manifest.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | -- Prep manifest data for users step 19 | CREATE OR REPLACE TABLE {{.scratch_schema}}.sessions_userid_manifest_this_run{{.entropy}} 20 | AS ( 21 | SELECT 22 | domain_userid, 23 | MIN(start_tstamp) AS start_tstamp 24 | 25 | FROM 26 | {{.scratch_schema}}.sessions_this_run{{.entropy}} 27 | 28 | GROUP BY 1 29 | ); 30 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/03-sessions/99-complete/98-truncate-page-views-staged.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | TRUNCATE TABLE {{.scratch_schema}}.page_views_staged{{.entropy}}; 19 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/04-users/01-main/01-userids-this-run.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.users_userids_this_run{{.entropy}} 19 | AS ( 20 | SELECT 21 | a.domain_userid, 22 | -- LEAST produces NULL if any input value is null 23 | LEAST(a.start_tstamp, COALESCE(b.start_tstamp, a.start_tstamp)) AS start_tstamp 24 | 25 | FROM 26 | {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}} AS a 27 | 28 | LEFT JOIN {{.output_schema}}.users_manifest{{.entropy}} AS b 29 | ON a.domain_userid = b.domain_userid 30 | ); 31 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/04-users/01-main/02-users-limits.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.users_limits{{.entropy}} 19 | AS ( 20 | SELECT 21 | MIN(start_tstamp) AS lower_limit, 22 | MAX(start_tstamp) AS upper_limit 23 | 24 | FROM 25 | {{.scratch_schema}}.users_userids_this_run{{.entropy}} 26 | ); 27 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/04-users/01-main/03-users-sessions-this-run.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.users_sessions_this_run{{.entropy}} 19 | AS ( 20 | SELECT 21 | a.* 22 | 23 | FROM 24 | {{.output_schema}}.sessions{{.entropy}} AS a 25 | 26 | INNER JOIN {{.scratch_schema}}.users_userids_this_run{{.entropy}} AS b 27 | ON a.domain_userid = b.domain_userid 28 | 29 | WHERE a.start_tstamp >= (SELECT lower_limit FROM {{.scratch_schema}}.users_limits{{.entropy}}) 30 | ); 31 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/04-users/01-main/04-users-aggs.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021-2022 Snowplow Analytics Ltd. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | CREATE OR REPLACE TABLE {{.scratch_schema}}.users_aggregates{{.entropy}} 19 | AS ( 20 | SELECT 21 | domain_userid, 22 | 23 | -- time 24 | MIN(start_tstamp) AS start_tstamp, 25 | MAX(end_tstamp) AS end_tstamp, 26 | 27 | -- engagement 28 | SUM(page_views) AS page_views, 29 | COUNT(DISTINCT domain_sessionid) AS sessions, 30 | SUM(engaged_time_in_s) AS engaged_time_in_s 31 | 32 | FROM 33 | {{.scratch_schema}}.users_sessions_this_run{{.entropy}} 34 | 35 | GROUP BY 1 36 | ); 37 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/99-migrations/1.0.1-migration/base-session-id-manifest.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE IF EXISTS {{.output_schema}}.base_session_id_manifest{{.entropy}} 2 | ALTER COLUMN session_id SET DATA TYPE VARCHAR(128); 3 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/99-migrations/1.0.1-migration/events-staged.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE IF EXISTS {{.scratch_schema}}.events_staged{{.entropy}} 2 | ALTER COLUMN se_label SET DATA TYPE VARCHAR(4096); 3 | -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/99-migrations/1.0.1-migration/page-views-staged.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE IF EXISTS {{.scratch_schema}}.page_views_staged{{.entropy}} 2 | ALTER COLUMN agent_name SET DATA TYPE VARCHAR, 3 | agent_name_version SET DATA TYPE VARCHAR, 4 | agent_name_version_major SET DATA TYPE VARCHAR, 5 | agent_version SET DATA TYPE VARCHAR, 6 | agent_version_major SET DATA TYPE VARCHAR, 7 | device_brand SET DATA TYPE VARCHAR, 8 | device_name SET DATA TYPE VARCHAR, 9 | device_version SET DATA TYPE VARCHAR, 10 | layout_engine_name SET DATA TYPE VARCHAR, 11 | layout_engine_name_version SET DATA TYPE VARCHAR, 12 | layout_engine_name_version_major SET DATA TYPE VARCHAR, 13 | layout_engine_version SET DATA TYPE VARCHAR, 14 | layout_engine_version_major SET DATA TYPE VARCHAR, 15 | operating_system_name SET DATA TYPE VARCHAR, 16 | operating_system_name_version SET DATA TYPE VARCHAR, 17 | operating_system_version SET DATA TYPE VARCHAR; -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/99-migrations/1.0.1-migration/page-views.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE IF EXISTS {{.output_schema}}.page_views{{.entropy}} 2 | ALTER COLUMN agent_name SET DATA TYPE VARCHAR, 3 | agent_name_version SET DATA TYPE VARCHAR, 4 | agent_name_version_major SET DATA TYPE VARCHAR, 5 | agent_version SET DATA TYPE VARCHAR, 6 | agent_version_major SET DATA TYPE VARCHAR, 7 | device_brand SET DATA TYPE VARCHAR, 8 | device_name SET DATA TYPE VARCHAR, 9 | device_version SET DATA TYPE VARCHAR, 10 | layout_engine_name SET DATA TYPE VARCHAR, 11 | layout_engine_name_version SET DATA TYPE VARCHAR, 12 | layout_engine_name_version_major SET DATA TYPE VARCHAR, 13 | layout_engine_version SET DATA TYPE VARCHAR, 14 | layout_engine_version_major SET DATA TYPE VARCHAR, 15 | operating_system_name SET DATA TYPE VARCHAR, 16 | operating_system_name_version SET DATA TYPE VARCHAR, 17 | operating_system_version SET DATA TYPE VARCHAR; -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/99-migrations/1.0.1-migration/sessions.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE IF EXISTS {{.output_schema}}.sessions{{.entropy}} 2 | ALTER COLUMN agent_name SET DATA TYPE VARCHAR, 3 | agent_name_version SET DATA TYPE VARCHAR, 4 | agent_name_version_major SET DATA TYPE VARCHAR, 5 | agent_version SET DATA TYPE VARCHAR, 6 | agent_version_major SET DATA TYPE VARCHAR, 7 | device_brand SET DATA TYPE VARCHAR, 8 | device_name SET DATA TYPE VARCHAR, 9 | device_version SET DATA TYPE VARCHAR, 10 | layout_engine_name SET DATA TYPE VARCHAR, 11 | layout_engine_name_version SET DATA TYPE VARCHAR, 12 | layout_engine_name_version_major SET DATA TYPE VARCHAR, 13 | layout_engine_version SET DATA TYPE VARCHAR, 14 | layout_engine_version_major SET DATA TYPE VARCHAR, 15 | operating_system_name SET DATA TYPE VARCHAR, 16 | operating_system_name_version SET DATA TYPE VARCHAR, 17 | operating_system_version SET DATA TYPE VARCHAR; -------------------------------------------------------------------------------- /web/v1/snowflake/sql-runner/sql/standard/99-migrations/1.0.1-migration/users-manifest.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE IF EXISTS {{.output_schema}}.users_manifest{{.entropy}} 2 | ALTER COLUMN domain_userid SET DATA TYPE VARCHAR(128); 3 | --------------------------------------------------------------------------------