├── report_tables └── property_overview │ ├── function │ ├── requirements.txt │ └── main.py │ ├── workflow │ └── property_overview.yaml │ ├── deploy_query.sh │ ├── deploy_function.sh │ └── query │ └── property_overview.sql ├── settings_downloader_function ├── requirements.txt └── main.py ├── schemas ├── ga4_adsense_links_schema.json ├── ga4_firebase_links_schema.json ├── ga4_accounts_schema.json ├── ga4_channel_groups_schema.json ├── ua_accounts_schema.json ├── ga4_measurement_protocol_secrets_schema.json ├── ga4_custom_metrics_schema.json ├── ga4_custom_dimensions_schema.json ├── ga4_conversion_events_schema.json ├── ga4_expanded_data_sets_schema.json ├── ga4_account_summaries_schema.json ├── ga4_dv360_links_schema.json ├── ga4_sa360_links_schema.json ├── ua_segments_schema.json ├── ga4_google_ads_links_schema.json ├── ua_google_ads_links_schema.json ├── ga4_bigquery_links_schema.json ├── ga4_audiences_schema.json ├── ua_custom_dimensions_schema.json ├── ga4_enhanced_measurement_settings_schema.json ├── ga4_dv360_link_proposals_schema.json ├── ga4_event_create_rules_schema.json ├── ua_account_summaries_schema.json ├── ua_custom_metrics_schema.json ├── ga4_data_streams_schema.json ├── ua_filter_links_schema.json ├── ga4_properties_schema.json ├── ua_views_schema.json ├── ua_goals_schema.json ├── ua_audiences_schema.json ├── ga4_sk_ad_network_conversion_value_schemas_schema.json └── ua_filters_schema.json ├── docs ├── contributing.md └── code-of-conduct.md ├── deploy_function.sh ├── README.md ├── deploy_bq_tables.sh └── LICENSE /report_tables/property_overview/function/requirements.txt: -------------------------------------------------------------------------------- 1 | functions-framework==3.* 2 | google-cloud-bigquery 3 | -------------------------------------------------------------------------------- /settings_downloader_function/requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client 2 | oauth2client 3 | google-cloud-bigquery 4 | google-auth 5 | git+https://github.com/googleapis/python-analytics-admin@main 6 | google-auth-oauthlib 7 | pyhumps 8 | protobuf 9 | -------------------------------------------------------------------------------- /schemas/ga4_adsense_links_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "name", 4 | "type": "STRING", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "type": "STRING", 9 | "name": "property_display_name", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "type": "STRING", 14 | "name": "property", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "type": "STRING", 19 | "name": "ad_client_code", 20 | "mode": "NULLABLE" 21 | } 22 | ] -------------------------------------------------------------------------------- /schemas/ga4_firebase_links_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "property", 5 | "type": "STRING" 6 | }, 7 | { 8 | "description": "bq-datetime", 9 | "mode": "NULLABLE", 10 | "name": "create_time", 11 | "type": "TIMESTAMP" 12 | }, 13 | { 14 | "mode": "NULLABLE", 15 | "name": "project", 16 | "type": "STRING" 17 | }, 18 | { 19 | "mode": "NULLABLE", 20 | "name": "property_display_name", 21 | "type": "STRING" 22 | }, 23 | { 24 | "mode": "NULLABLE", 25 | "name": "name", 26 | "type": "STRING" 27 | } 28 | ] -------------------------------------------------------------------------------- /schemas/ga4_accounts_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "deleted", 5 | "type": "BOOLEAN" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "region_code", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "name", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "update_time", 20 | "type": "TIMESTAMP" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "create_time", 25 | "type": "TIMESTAMP" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "display_name", 30 | "type": "STRING" 31 | } 32 | ] -------------------------------------------------------------------------------- /schemas/ga4_channel_groups_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "name", 4 | "type": "STRING", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "type": "STRING", 9 | "name": "property_display_name", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "type": "STRING", 14 | "name": "property", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "type": "STRING", 19 | "name": "display_name", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "type": "STRING", 24 | "name": "grouping_rule", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "type": "BOOLEAN", 29 | "name": "system_defined", 30 | "mode": "NULLABLE" 31 | } 32 | ] -------------------------------------------------------------------------------- /schemas/ua_accounts_schema.json: -------------------------------------------------------------------------------- 1 | [{"type": "STRING", "name": "kind", "mode": "NULLABLE"}, {"fields": [{"type": "STRING", "name": "href", "mode": "NULLABLE"}, {"type": "STRING", "name": "type", "mode": "NULLABLE"}], "type": "RECORD", "name": "childLink", "mode": "NULLABLE"}, {"type": "DATETIME", "name": "updated", "mode": "NULLABLE"}, {"type": "STRING", "name": "name", "mode": "NULLABLE"}, {"type": "DATETIME", "name": "created", "mode": "NULLABLE"}, {"type": "BOOL", "name": "starred", "mode": "NULLABLE"}, {"type": "STRING", "name": "id", "mode": "NULLABLE"}, {"type": "STRING", "name": "selfLink", "mode": "NULLABLE"}, {"fields": [{"type": "RECORD", "name": "effective", "mode": "REPEATED"}], "type": "RECORD", "name": "permissions", "mode": "NULLABLE"}] -------------------------------------------------------------------------------- /schemas/ga4_measurement_protocol_secrets_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "name", 5 | "type": "STRING" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "display_name", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "secret_value", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "stream_name", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "type", 25 | "type": "STRING" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "property", 30 | "type": "STRING" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "property_display_name", 35 | "type": "STRING" 36 | } 37 | ] -------------------------------------------------------------------------------- /schemas/ga4_custom_metrics_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "measurement_unit", 5 | "type": "INTEGER" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "display_name", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "name", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "scope", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "parameter_name", 25 | "type": "STRING" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "property_display_name", 30 | "type": "STRING" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "description", 35 | "type": "STRING" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "property", 40 | "type": "STRING" 41 | } 42 | ] -------------------------------------------------------------------------------- /schemas/ga4_custom_dimensions_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "property", 5 | "type": "STRING" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "display_name", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "name", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "scope", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "parameter_name", 25 | "type": "STRING" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "disallow_ads_personalization", 30 | "type": "BOOLEAN" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "property_display_name", 35 | "type": "STRING" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "description", 40 | "type": "STRING" 41 | } 42 | ] -------------------------------------------------------------------------------- /schemas/ga4_conversion_events_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "property", 5 | "type": "STRING" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "custom", 10 | "type": "BOOLEAN" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "deletable", 15 | "type": "BOOLEAN" 16 | }, 17 | { 18 | "description": "bq-datetime", 19 | "mode": "NULLABLE", 20 | "name": "create_time", 21 | "type": "TIMESTAMP" 22 | }, 23 | { 24 | "mode": "NULLABLE", 25 | "name": "property_display_name", 26 | "type": "STRING" 27 | }, 28 | { 29 | "mode": "NULLABLE", 30 | "name": "event_name", 31 | "type": "STRING" 32 | }, 33 | { 34 | "mode": "NULLABLE", 35 | "name": "name", 36 | "type": "STRING" 37 | }, 38 | { 39 | "mode": "NULLABLE", 40 | "name": "counting_method", 41 | "type": "STRING" 42 | } 43 | ] -------------------------------------------------------------------------------- /schemas/ga4_expanded_data_sets_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "name", 5 | "type": "STRING" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "display_name", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "REPEATED", 14 | "name": "dimension_names", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "REPEATED", 19 | "name": "metric_names", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "dimension_filter_expression", 25 | "type": "STRING" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "data_collection_start_time", 30 | "type": "STRING" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "property", 35 | "type": "STRING" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "property_display_name", 40 | "type": "STRING" 41 | } 42 | ] -------------------------------------------------------------------------------- /schemas/ga4_account_summaries_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "fields": [ 4 | { 5 | "mode": "NULLABLE", 6 | "name": "display_name", 7 | "type": "STRING" 8 | }, 9 | { 10 | "mode": "NULLABLE", 11 | "name": "property", 12 | "type": "STRING" 13 | }, 14 | { 15 | "mode":"NULLABLE", 16 | "name":"parent", 17 | "type":"STRING" 18 | }, 19 | { 20 | "mode":"NULLABLE", 21 | "name":"property_type", 22 | "type":"STRING" 23 | } 24 | ], 25 | "mode": "REPEATED", 26 | "name": "property_summaries", 27 | "type": "RECORD" 28 | }, 29 | { 30 | "mode": "NULLABLE", 31 | "name": "account", 32 | "type": "STRING" 33 | }, 34 | { 35 | "mode": "NULLABLE", 36 | "name": "display_name", 37 | "type": "STRING" 38 | }, 39 | { 40 | "mode": "NULLABLE", 41 | "name": "name", 42 | "type": "STRING" 43 | } 44 | ] -------------------------------------------------------------------------------- /schemas/ga4_dv360_links_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"BOOL", 4 | "name":"campaign_data_sharing_enabled", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "type":"STRING", 9 | "name":"name", 10 | "mode":"NULLABLE" 11 | }, 12 | { 13 | "type":"BOOL", 14 | "name":"cost_data_sharing_enabled", 15 | "mode":"NULLABLE" 16 | }, 17 | { 18 | "type":"BOOL", 19 | "name":"ads_personalization_enabled", 20 | "mode":"NULLABLE" 21 | }, 22 | { 23 | "type":"STRING", 24 | "name":"advertiser_id", 25 | "mode":"NULLABLE" 26 | }, 27 | { 28 | "type":"STRING", 29 | "name":"advertiser_display_name", 30 | "mode":"NULLABLE" 31 | }, 32 | { 33 | "type":"STRING", 34 | "name":"property", 35 | "mode":"NULLABLE" 36 | }, 37 | { 38 | "type":"STRING", 39 | "name":"property_display_name", 40 | "mode":"NULLABLE" 41 | } 42 | ] -------------------------------------------------------------------------------- /schemas/ga4_sa360_links_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"BOOL", 4 | "name":"campaign_data_sharing_enabled", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "type":"STRING", 9 | "name":"name", 10 | "mode":"NULLABLE" 11 | }, 12 | { 13 | "type":"BOOL", 14 | "name":"cost_data_sharing_enabled", 15 | "mode":"NULLABLE" 16 | }, 17 | { 18 | "type":"BOOL", 19 | "name":"ads_personalization_enabled", 20 | "mode":"NULLABLE" 21 | }, 22 | { 23 | "type":"STRING", 24 | "name":"advertiser_id", 25 | "mode":"NULLABLE" 26 | }, 27 | { 28 | "type":"STRING", 29 | "name":"site_stats_sharing_enabled", 30 | "mode":"NULLABLE" 31 | }, 32 | { 33 | "type":"STRING", 34 | "name":"property", 35 | "mode":"NULLABLE" 36 | }, 37 | { 38 | "type":"STRING", 39 | "name":"property_display_name", 40 | "mode":"NULLABLE" 41 | } 42 | ] -------------------------------------------------------------------------------- /schemas/ua_segments_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"STRING", 4 | "name":"definition", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "type":"STRING", 9 | "name":"kind", 10 | "mode":"NULLABLE" 11 | }, 12 | { 13 | "type":"TIMESTAMP", 14 | "name":"updated", 15 | "mode":"NULLABLE" 16 | }, 17 | { 18 | "type":"STRING", 19 | "name":"segmentId", 20 | "mode":"NULLABLE" 21 | }, 22 | { 23 | "type":"TIMESTAMP", 24 | "name":"created", 25 | "mode":"NULLABLE" 26 | }, 27 | { 28 | "type":"STRING", 29 | "name":"type", 30 | "mode":"NULLABLE" 31 | }, 32 | { 33 | "type":"STRING", 34 | "name":"id", 35 | "mode":"NULLABLE" 36 | }, 37 | { 38 | "type":"STRING", 39 | "name":"selfLink", 40 | "mode":"NULLABLE" 41 | }, 42 | { 43 | "type":"STRING", 44 | "name":"name", 45 | "mode":"NULLABLE" 46 | } 47 | ] -------------------------------------------------------------------------------- /schemas/ga4_google_ads_links_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "update_time", 5 | "type": "TIMESTAMP" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "property", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "ads_personalization_enabled", 15 | "type": "BOOLEAN" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "name", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "property_display_name", 25 | "type": "STRING" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "can_manage_clients", 30 | "type": "BOOLEAN" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "creator_email_address", 35 | "type": "STRING" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "customer_id", 40 | "type": "STRING" 41 | }, 42 | { 43 | "mode": "NULLABLE", 44 | "name": "create_time", 45 | "type": "TIMESTAMP" 46 | } 47 | ] -------------------------------------------------------------------------------- /schemas/ua_google_ads_links_schema.json: -------------------------------------------------------------------------------- 1 | [{"type": "STRING", "name": "kind", "mode": "NULLABLE"}, {"type": "STRING", "name": "name", "mode": "NULLABLE"}, {"fields": [{"type": "STRING", "name": "kind", "mode": "NULLABLE"}, {"type": "STRING", "name": "customerId", "mode": "NULLABLE"}, {"type": "BOOL", "name": "autoTaggingEnabled", "mode": "NULLABLE"}], "type": "RECORD", "name": "adWordsAccounts", "mode": "REPEATED"}, {"fields": [{"fields": [{"type": "STRING", "name": "kind", "mode": "NULLABLE"}, {"type": "STRING", "name": "name", "mode": "NULLABLE"}, {"type": "STRING", "name": "internalWebPropertyId", "mode": "NULLABLE"}, {"type": "STRING", "name": "href", "mode": "NULLABLE"}, {"type": "STRING", "name": "id", "mode": "NULLABLE"}, {"type": "STRING", "name": "accountId", "mode": "NULLABLE"}], "type": "RECORD", "name": "webPropertyRef", "mode": "NULLABLE"}], "type": "RECORD", "name": "entity", "mode": "NULLABLE"}, {"type": "STRING", "name": "profileIds", "mode": "REPEATED"}, {"type": "STRING", "name": "id", "mode": "NULLABLE"}, {"type": "STRING", "name": "selfLink", "mode": "NULLABLE"}] -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). -------------------------------------------------------------------------------- /schemas/ga4_bigquery_links_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"BOOL", 4 | "name":"daily_export_enabled", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "type":"STRING", 9 | "name":"name", 10 | "mode":"NULLABLE" 11 | }, 12 | { 13 | "type":"STRING", 14 | "name":"create_time", 15 | "mode":"NULLABLE" 16 | }, 17 | { 18 | "type":"BOOL", 19 | "name":"streaming_export_enabled", 20 | "mode":"NULLABLE" 21 | }, 22 | { 23 | "type":"BOOL", 24 | "name":"fresh_daily_export_enabled", 25 | "mode":"NULLABLE" 26 | }, 27 | { 28 | "type":"BOOL", 29 | "name":"include_advertising_id", 30 | "mode":"NULLABLE" 31 | }, 32 | { 33 | "type":"STRING", 34 | "name":"export_streams", 35 | "mode":"REPEATED" 36 | }, 37 | { 38 | "type":"STRING", 39 | "name":"excluded_events", 40 | "mode":"REPEATED" 41 | }, 42 | { 43 | "type":"STRING", 44 | "name":"property", 45 | "mode":"NULLABLE" 46 | }, 47 | { 48 | "type":"STRING", 49 | "name":"property_display_name", 50 | "mode":"NULLABLE" 51 | }, 52 | { 53 | "type":"STRING", 54 | "name":"project", 55 | "mode":"NULLABLE" 56 | } 57 | ] -------------------------------------------------------------------------------- /schemas/ga4_audiences_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type": "STRING", 4 | "name": "name", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "type": "STRING", 9 | "name": "display_name", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "type": "STRING", 14 | "name": "description", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "type": "INT64", 19 | "name": "membership_duration_days", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "type": "BOOL", 24 | "name": "ads_personalization_enabled", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "type": "STRING", 29 | "name": "property_display_name", 30 | "mode": "NULLABLE" 31 | }, 32 | { 33 | "type": "STRING", 34 | "name": "property_name", 35 | "mode": "NULLABLE" 36 | }, 37 | { 38 | "type": "STRING", 39 | "name": "property", 40 | "mode": "NULLABLE" 41 | }, 42 | { 43 | "type": "RECORD", 44 | "name": "event_trigger", 45 | "mode": "NULLABLE", 46 | "fields": [ 47 | { 48 | "type": "STRING", 49 | "name": "event_name", 50 | "mode": "NULLABLE" 51 | }, 52 | { 53 | "type": "STRING", 54 | "name": "log_condition", 55 | "mode": "NULLABLE" 56 | } 57 | ] 58 | }, 59 | { 60 | "type": "STRING", 61 | "name": "filter_clauses", 62 | "mode": "NULLABLE" 63 | }, 64 | { 65 | "type": "STRING", 66 | "name": "exclusion_duration_mode", 67 | "mode": "NULLABLE" 68 | } 69 | ] -------------------------------------------------------------------------------- /report_tables/property_overview/workflow/property_overview.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | main: 15 | steps: 16 | - writeGoogleAnalyticsSettingsToBigQuery: 17 | call: http.post 18 | args: 19 | url: {downloader-function-url} 20 | auth: 21 | type: OIDC 22 | result: settingsStatus 23 | - checkStatus: 24 | switch: 25 | - condition: '${settingsStatus.body == "success"}' 26 | steps: 27 | - updatePropertyOverviewBigQueryTable: 28 | call: http.post 29 | args: 30 | url: {property-overview-function-url} 31 | auth: 32 | type: OIDC 33 | result: propertyOverviewStatus 34 | - complete: 35 | return: ${propertyOverviewStatus} 36 | 37 | -------------------------------------------------------------------------------- /schemas/ua_custom_dimensions_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"TIMESTAMP", 4 | "name":"updated", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "type":"STRING", 9 | "name":"webPropertyId", 10 | "mode":"NULLABLE" 11 | }, 12 | { 13 | "type":"BOOL", 14 | "name":"active", 15 | "mode":"NULLABLE" 16 | }, 17 | { 18 | "type":"STRING", 19 | "name":"id", 20 | "mode":"NULLABLE" 21 | }, 22 | { 23 | "type":"INT64", 24 | "name":"accountId", 25 | "mode":"NULLABLE" 26 | }, 27 | { 28 | "type":"INT64", 29 | "name":"index", 30 | "mode":"NULLABLE" 31 | }, 32 | { 33 | "type":"STRING", 34 | "name":"kind", 35 | "mode":"NULLABLE" 36 | }, 37 | { 38 | "type":"STRING", 39 | "name":"name", 40 | "mode":"NULLABLE" 41 | }, 42 | { 43 | "type":"TIMESTAMP", 44 | "name":"created", 45 | "mode":"NULLABLE" 46 | }, 47 | { 48 | "type":"STRING", 49 | "name":"scope", 50 | "mode":"NULLABLE" 51 | }, 52 | { 53 | "fields":[ 54 | { 55 | "type":"STRING", 56 | "name":"href", 57 | "mode":"NULLABLE" 58 | }, 59 | { 60 | "type":"STRING", 61 | "name":"type", 62 | "mode":"NULLABLE" 63 | } 64 | ], 65 | "type":"RECORD", 66 | "name":"parentLink", 67 | "mode":"NULLABLE" 68 | }, 69 | { 70 | "type":"STRING", 71 | "name":"selfLink", 72 | "mode":"NULLABLE" 73 | } 74 | ] -------------------------------------------------------------------------------- /report_tables/property_overview/deploy_query.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ########################################################################### 3 | # 4 | # Copyright 2023 Google Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | echo "******** Welcome ********** 20 | * 21 | * Google Analytics Property Overview Table Deployment 22 | * 23 | ***************************" 24 | 25 | read -p "Please enter your Google Cloud Project ID: " project_id 26 | 27 | bq mk -t --time_partitioning_type=DAY \ 28 | $project_id:analytics_settings_database.property_overview 29 | 30 | cd query 31 | 32 | sql=$(cat property_overview.sql) 33 | 34 | bq query \ 35 | --use_legacy_sql=false \ 36 | --destination_table=$project_id:analytics_settings_database.property_overview \ 37 | --display_name="Property Overview" \ 38 | --schedule="every day 23:30" \ 39 | --append_table=true \ 40 | "$sql" 41 | 42 | echo "*************************** 43 | * 44 | * Google Analytics Property Overview Table Deployment Complete 45 | * 46 | ***************************" 47 | 48 | -------------------------------------------------------------------------------- /schemas/ga4_enhanced_measurement_settings_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"STRING", 4 | "name":"name", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "type":"BOOL", 9 | "name":"stream_enabled", 10 | "mode":"NULLABLE" 11 | }, 12 | { 13 | "type":"BOOL", 14 | "name":"scrolls_enabled", 15 | "mode":"NULLABLE" 16 | }, 17 | { 18 | "type":"BOOL", 19 | "name":"outbound_clicks_enabled", 20 | "mode":"NULLABLE" 21 | }, 22 | { 23 | "type":"BOOL", 24 | "name":"site_search_enabled", 25 | "mode":"NULLABLE" 26 | }, 27 | { 28 | "type":"BOOL", 29 | "name":"video_engagement_enabled", 30 | "mode":"NULLABLE" 31 | }, 32 | { 33 | "type":"BOOL", 34 | "name":"file_downloads_enabled", 35 | "mode":"NULLABLE" 36 | }, 37 | { 38 | "type":"BOOL", 39 | "name":"page_changes_enabled", 40 | "mode":"NULLABLE" 41 | }, 42 | { 43 | "type":"BOOL", 44 | "name":"form_interactions_enabled", 45 | "mode":"NULLABLE" 46 | }, 47 | { 48 | "type":"STRING", 49 | "name":"search_query_parameter", 50 | "mode":"NULLABLE" 51 | }, 52 | { 53 | "type":"STRING", 54 | "name":"uri_query_parameter", 55 | "mode":"NULLABLE" 56 | }, 57 | { 58 | "type":"STRING", 59 | "name":"property", 60 | "mode":"NULLABLE" 61 | }, 62 | { 63 | "type":"STRING", 64 | "name":"property_display_name", 65 | "mode":"NULLABLE" 66 | }, 67 | { 68 | "mode": "NULLABLE", 69 | "name": "stream_name", 70 | "type": "STRING" 71 | } 72 | ] -------------------------------------------------------------------------------- /schemas/ga4_dv360_link_proposals_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"STRING", 4 | "name":"advertiser_id", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "fields":[ 9 | { 10 | "type":"STRING", 11 | "name":"link_proposal_state", 12 | "mode":"NULLABLE" 13 | }, 14 | { 15 | "type":"STRING", 16 | "name":"link_proposal_initiating_product", 17 | "mode":"NULLABLE" 18 | }, 19 | { 20 | "type":"STRING", 21 | "name":"requestor_email", 22 | "mode":"NULLABLE" 23 | } 24 | ], 25 | "type":"RECORD", 26 | "name":"link_proposal_status_details", 27 | "mode":"NULLABLE" 28 | }, 29 | { 30 | "type":"STRING", 31 | "name":"validation_email", 32 | "mode":"NULLABLE" 33 | }, 34 | { 35 | "type":"STRING", 36 | "name":"name", 37 | "mode":"NULLABLE" 38 | }, 39 | { 40 | "type":"STRING", 41 | "name":"property_display_name", 42 | "mode":"NULLABLE" 43 | }, 44 | { 45 | "type":"BOOL", 46 | "name":"ads_personalization_enabled", 47 | "mode":"NULLABLE" 48 | }, 49 | { 50 | "type":"STRING", 51 | "name":"property", 52 | "mode":"NULLABLE" 53 | }, 54 | { 55 | "type":"STRING", 56 | "name":"advertiser_display_name", 57 | "mode":"NULLABLE" 58 | }, 59 | { 60 | "type":"BOOL", 61 | "name":"cost_data_sharing_enabled", 62 | "mode":"NULLABLE" 63 | }, 64 | { 65 | "type":"BOOL", 66 | "name":"campaign_data_sharing_enabled", 67 | "mode":"NULLABLE" 68 | } 69 | ] -------------------------------------------------------------------------------- /schemas/ga4_event_create_rules_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "name", 4 | "type": "STRING", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "type": "STRING", 9 | "name": "property_display_name", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "type": "STRING", 14 | "name": "property", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "name": "type", 19 | "type": "STRING", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "name": "stream_name", 24 | "type": "STRING", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "name": "destination_event", 29 | "type": "STRING", 30 | "mode": "NULLABLE" 31 | }, 32 | { 33 | "name": "event_conditions", 34 | "type": "RECORD", 35 | "mode": "NULLABLE", 36 | "fields": [ 37 | { 38 | "name": "field", 39 | "type": "STRING", 40 | "mode": "NULLABLE" 41 | }, 42 | { 43 | "name": "comparison_type", 44 | "type": "STRING", 45 | "mode": "NULLABLE" 46 | }, 47 | { 48 | "name": "value", 49 | "type": "STRING", 50 | "mode": "NULLABLE" 51 | }, 52 | { 53 | "name": "negated", 54 | "type": "BOOLEAN", 55 | "mode": "NULLABLE" 56 | } 57 | ] 58 | }, 59 | { 60 | "name": "source_copy_parameters", 61 | "type": "BOOLEAN", 62 | "mode": "NULLABLE" 63 | }, 64 | { 65 | "name": "parameter_mutations", 66 | "type": "RECORD", 67 | "mode": "NULLABLE", 68 | "fields": [ 69 | { 70 | "name": "parameter", 71 | "type": "STRING", 72 | "mode": "NULLABLE" 73 | }, 74 | { 75 | "name": "parameter_value", 76 | "type": "STRING", 77 | "mode": "NULLABLE" 78 | } 79 | ] 80 | } 81 | ] -------------------------------------------------------------------------------- /schemas/ua_account_summaries_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "name", 5 | "type": "STRING" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "kind", 10 | "type": "STRING" 11 | }, 12 | { 13 | "fields": [ 14 | { 15 | "mode": "NULLABLE", 16 | "name": "level", 17 | "type": "STRING" 18 | }, 19 | { 20 | "mode": "NULLABLE", 21 | "name": "internalWebPropertyId", 22 | "type": "INT64" 23 | }, 24 | { 25 | "mode": "NULLABLE", 26 | "name": "name", 27 | "type": "STRING" 28 | }, 29 | { 30 | "mode": "NULLABLE", 31 | "name": "websiteUrl", 32 | "type": "STRING" 33 | }, 34 | { 35 | "mode": "NULLABLE", 36 | "name": "id", 37 | "type": "STRING" 38 | }, 39 | { 40 | "fields": [ 41 | { 42 | "mode": "NULLABLE", 43 | "name": "type", 44 | "type": "STRING" 45 | }, 46 | { 47 | "mode": "NULLABLE", 48 | "name": "name", 49 | "type": "STRING" 50 | }, 51 | { 52 | "mode": "NULLABLE", 53 | "name": "id", 54 | "type": "INT64" 55 | }, 56 | { 57 | "mode": "NULLABLE", 58 | "name": "kind", 59 | "type": "STRING" 60 | } 61 | ], 62 | "mode": "REPEATED", 63 | "name": "profiles", 64 | "type": "RECORD" 65 | }, 66 | { 67 | "mode": "NULLABLE", 68 | "name": "kind", 69 | "type": "STRING" 70 | } 71 | ], 72 | "mode": "REPEATED", 73 | "name": "webProperties", 74 | "type": "RECORD" 75 | }, 76 | { 77 | "mode": "NULLABLE", 78 | "name": "id", 79 | "type": "INT64" 80 | } 81 | ] -------------------------------------------------------------------------------- /schemas/ua_custom_metrics_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"TIMESTAMP", 4 | "name":"updated", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "type":"STRING", 9 | "name":"max_value", 10 | "mode":"NULLABLE" 11 | }, 12 | { 13 | "type":"INT64", 14 | "name":"min_value", 15 | "mode":"NULLABLE" 16 | }, 17 | { 18 | "type":"STRING", 19 | "name":"webPropertyId", 20 | "mode":"NULLABLE" 21 | }, 22 | { 23 | "type":"BOOL", 24 | "name":"active", 25 | "mode":"NULLABLE" 26 | }, 27 | { 28 | "type":"STRING", 29 | "name":"id", 30 | "mode":"NULLABLE" 31 | }, 32 | { 33 | "type":"INT64", 34 | "name":"accountId", 35 | "mode":"NULLABLE" 36 | }, 37 | { 38 | "type":"INT64", 39 | "name":"index", 40 | "mode":"NULLABLE" 41 | }, 42 | { 43 | "type":"STRING", 44 | "name":"kind", 45 | "mode":"NULLABLE" 46 | }, 47 | { 48 | "type":"STRING", 49 | "name":"name", 50 | "mode":"NULLABLE" 51 | }, 52 | { 53 | "type":"TIMESTAMP", 54 | "name":"created", 55 | "mode":"NULLABLE" 56 | }, 57 | { 58 | "type":"STRING", 59 | "name":"scope", 60 | "mode":"NULLABLE" 61 | }, 62 | { 63 | "fields":[ 64 | { 65 | "type":"STRING", 66 | "name":"href", 67 | "mode":"NULLABLE" 68 | }, 69 | { 70 | "type":"STRING", 71 | "name":"type", 72 | "mode":"NULLABLE" 73 | } 74 | ], 75 | "type":"RECORD", 76 | "name":"parentLink", 77 | "mode":"NULLABLE" 78 | }, 79 | { 80 | "type":"STRING", 81 | "name":"type", 82 | "mode":"NULLABLE" 83 | }, 84 | { 85 | "type":"STRING", 86 | "name":"selfLink", 87 | "mode":"NULLABLE" 88 | } 89 | ] -------------------------------------------------------------------------------- /deploy_function.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ########################################################################### 3 | # 4 | # Copyright 2023 Google Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | cat < Service Accounts ](https://console.cloud.google.com/iam-admin/serviceaccounts) and create a new service account. 19 | - Give the service account a name. This guide will use analytics-settings-database as an example. 20 | - Grant the service account the BigQuery Admin role. 21 | - Grant the service account the Cloud Funtions Invoker role. 22 | 3. Click on your newly created service account and navigate to "Keys". 23 | 4. Create a new JSON key. Make note of the file that was downloaded to your device. 24 | 5. Open cloud shell and enter the following to create the cloud function: 25 | 26 | ``` 27 | 28 | rm -rf analytics-settings-database && git clone https://github.com/google/analytics-settings-database.git && cd analytics-settings-database && bash deploy_function.sh 29 | 30 | ``` 31 | - Follow the steps outlined in the deploy script to create the HTTP function. 32 | - Once the function has been created, navigate to [Cloud Functions](https://console.cloud.google.com/functions/list) and click on the cloud function. 33 | - Edit the function and under "Runtime, build, connections and security settings", change the service account to the one you recently created. 34 | - Click next. 35 | - Click "+" to create a new file. Name this file credentials.json and add the contents of the key file you downloaded earlier afer you created your service account. 36 | - Click deploy. Your cloud function should now be operational. 37 | 6. If it is not already open, open cloud shell again and enter the following to create your BigQuery dataset and tables. This will automatically create a data set named "analytics\_settings\_database" and populate it with the required tables. 38 | ``` 39 | 40 | cd analytics-settings-database && bash deploy_bq_tables.sh 41 | 42 | ``` 43 | - Once the script has run, your new tables should be visible in BigQuery. 44 | 7. Navigate to [Cloud Scheduler](https://console.cloud.google.com/cloudscheduler). 45 | - Create a job. 46 | - Give the schedule a name. This guide will use analytics-settings-database as an example. 47 | - Enter a frequency. You can enter whatever frequency you would like, but this guide will use 0 22 * * * as an example to run the scheduler daily at 10 PM (or 22:00). 48 | - Enter your timezone. 49 | - Set the target type to HTTP. 50 | - Set the URL to your cloud function's URL. 51 | - Set the Auth header to "Add OIDC token" and select your service account. 52 | - Save the schedule. 53 | 8. Copy your service account email and grant it access to the GA4 accounts you want it to access. 54 | 55 | Upon completing the implementation process, the settings for your Google Analytics accounts that the API can access will be loaded into BigQuery daily at 10 PM. The frequency with which this happens can be adjusted by modifying the Cloud Scheduler Job created during the deployment process. 56 | 57 | #### Property Overview Table 58 | 59 | The property overview table serves as an example of how the various settings tables can be combined to create a useful report on the status of various settings for your properties. You can either create a scheduled query based on the property\_overview.sql query or create a cloud function workflow. 60 | 61 | ##### Scheduled Query Deployment 62 | 1. Open cloud shell and enter the following to create the cloud function: 63 | 64 | ``` 65 | 66 | cd analytics-settings-database/report_tables/property_overview && bash deploy_query.sh 67 | 68 | ``` 69 | 2. Upon completing the deploy script, the scheduled query should be complete. 70 | 3. Setup is now complete. Your new property overview table will be populated when your scheduled query runs. 71 | 72 | ##### Cloud Function Workflow 73 | 74 | 1. Open cloud shell and enter the following to create the cloud function: 75 | 76 | ``` 77 | 78 | cd analytics-settings-database/report_tables/property_overview && bash deploy_function.sh 79 | 80 | ``` 81 | When the script completes, the following should be created: 82 | - A property overview cloud function. 83 | - A property overview table. 84 | - A property overview workflow. 85 | 2. Navigate to your [workflows](https://console.cloud.google.com/workflows) and edit your new workflow. 86 | 3. Add a cloud scheduler trigger and save your workflow. 87 | 4. If you previously created a cloud scheduler to trigger your downloader function, navigate to to the [Cloud Scheduler](https://console.cloud.google.com/cloudscheduler) page and delete the cloud scheduler that was previously used with your cloud function. 88 | 5. Setup is now complete and your new property overview table should be populated whenever your worklow is scheduled to run. 89 | -------------------------------------------------------------------------------- /deploy_bq_tables.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ########################################################################### 3 | # 4 | # Copyright 2023 Google Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | cat < 0 THEN TRUE ELSE FALSE END) AS web_streams_created, 26 | (CASE WHEN SUM(DISTINCT CASE WHEN data_streams.type = 'ANDROID_APP_DATA_STREAM' THEN 1 ELSE 0 END) > 0 THEN TRUE ELSE FALSE END) AS android_streams_created, 27 | (CASE WHEN SUM(DISTINCT CASE WHEN data_streams.type = 'IOS_APP_DATA_STREAM' THEN 1 ELSE 0 END) > 0 THEN TRUE ELSE FALSE END) AS ios_streams_created, 28 | (CASE WHEN SUM(conversion_events.is_custom) > 0 THEN TRUE ELSE FALSE END) AS custom_conversions_created, 29 | SUM(conversion_events.is_custom) AS number_of_custom_conversion_events, 30 | (CASE WHEN COUNT(DISTINCT google_ads_links.customer_id) > 0 THEN TRUE ELSE FALSE END) AS google_ads_linked, 31 | COUNT(DISTINCT google_ads_links.customer_id) AS number_of_google_ads_links, 32 | (CASE WHEN COUNT(DISTINCT dv360_links.name) > 0 THEN TRUE ELSE FALSE END) AS dv360_linked, 33 | COUNT(DISTINCT dv360_links.name) AS number_of_dv360_links, 34 | (CASE WHEN COUNT(DISTINCT firebase_links.project) > 0 THEN TRUE ELSE FALSE END) AS firebase_linked, 35 | COUNT(DISTINCT firebase_links.project) AS number_of_firebase_links, 36 | (CASE WHEN COUNT(DISTINCT custom_dimensions.parameter_name) > 0 THEN TRUE ELSE FALSE END) AS custom_dimensions_created, 37 | COUNT(DISTINCT custom_dimensions.parameter_name) AS number_of_custom_dimensions, 38 | (CASE WHEN COUNT(DISTINCT custom_metrics.parameter_name) > 0 THEN TRUE ELSE FALSE END) AS custom_metrics_created, 39 | COUNT(DISTINCT custom_metrics.parameter_name) AS number_of_custom_metrics, 40 | (CASE WHEN COUNT(DISTINCT measurement_protocol_secrets.secret_value) > 0 THEN TRUE ELSE FALSE END) AS measurement_protocol_secret_created, 41 | COUNT(DISTINCT measurement_protocol_secrets.secret_value) AS number_of_measurement_protocol_secrets, 42 | (CASE WHEN SUM(audiences.is_custom) > 0 THEN TRUE ELSE FALSE END) AS custom_audiences_created, 43 | SUM(audiences.is_custom) AS number_of_custom_audiences 44 | FROM 45 | analytics_settings_database.ga4_account_summaries AS summaries, 46 | UNNEST(property_summaries) AS property_summaries 47 | LEFT JOIN ( 48 | SELECT 49 | name AS property_id, 50 | service_level, 51 | google_signals_settings.state AS google_signals_state 52 | FROM 53 | analytics_settings_database.ga4_properties 54 | WHERE 55 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS properties 56 | ON 57 | property_summaries.property = properties.property_id 58 | LEFT JOIN ( 59 | SELECT 60 | property AS property_id, 61 | type, 62 | name AS id 63 | FROM 64 | analytics_settings_database.ga4_data_streams 65 | WHERE 66 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS data_streams 67 | ON 68 | property_summaries.property = data_streams.property_id 69 | LEFT JOIN ( 70 | SELECT 71 | property AS property_id, 72 | name AS id, 73 | event_name, 74 | CASE 75 | WHEN REGEXP_CONTAINS('purchase|first_open|in_app_purchase|app_store_subscription_convert|app_store_subscription_renew', event_name ) THEN 0 76 | ELSE 77 | 1 78 | END 79 | AS is_custom 80 | FROM 81 | analytics_settings_database.ga4_conversion_events 82 | WHERE 83 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS conversion_events 84 | ON 85 | property_summaries.property = conversion_events.property_id 86 | LEFT JOIN ( 87 | SELECT 88 | property AS property_id, 89 | customer_id 90 | FROM 91 | analytics_settings_database.ga4_google_ads_links 92 | WHERE 93 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS google_ads_links 94 | ON 95 | property_summaries.property = google_ads_links.property_id 96 | LEFT JOIN ( 97 | SELECT 98 | property AS property_id, 99 | name 100 | FROM 101 | analytics_settings_database.ga4_dv360_links 102 | WHERE 103 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS dv360_links 104 | ON 105 | property_summaries.property = google_ads_links.property_id 106 | LEFT JOIN ( 107 | SELECT 108 | property AS property_id, 109 | project 110 | FROM 111 | analytics_settings_database.ga4_firebase_links 112 | WHERE 113 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS firebase_links 114 | ON 115 | property_summaries.property = firebase_links.property_id 116 | LEFT JOIN ( 117 | SELECT 118 | property AS property_id, 119 | parameter_name 120 | FROM 121 | analytics_settings_database.ga4_custom_dimensions 122 | WHERE 123 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS custom_dimensions 124 | ON 125 | property_summaries.property = custom_dimensions.property_id 126 | LEFT JOIN ( 127 | SELECT 128 | property AS property_id, 129 | parameter_name 130 | FROM 131 | analytics_settings_database.ga4_custom_metrics 132 | WHERE 133 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS custom_metrics ON 134 | property_summaries.property = custom_metrics.property_id 135 | LEFT JOIN ( 136 | SELECT 137 | property AS property_id, 138 | secret_value 139 | FROM 140 | analytics_settings_database.ga4_measurement_protocol_secrets 141 | WHERE 142 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS measurement_protocol_secrets 143 | ON 144 | property_summaries.property = measurement_protocol_secrets.property_id 145 | LEFT JOIN ( 146 | SELECT 147 | property AS property_id, 148 | name, 149 | CASE 150 | WHEN REGEXP_CONTAINS('Purchasers|All Users', display_name) THEN 0 151 | ELSE 152 | 1 153 | END as is_custom 154 | FROM 155 | analytics_settings_database.ga4_audiences 156 | WHERE 157 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS audiences ON 158 | property_summaries.property = audiences.property_id 159 | WHERE 160 | DATE(_PARTITIONTIME) = CURRENT_DATE() 161 | GROUP BY 162 | 1, 2, 3, 4, 5, 6 163 | -------------------------------------------------------------------------------- /schemas/ua_filters_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"TIMESTAMP", 4 | "name":"updated", 5 | "mode":"NULLABLE" 6 | }, 7 | { 8 | "fields":[ 9 | { 10 | "type":"STRING", 11 | "name":"field", 12 | "mode":"NULLABLE" 13 | }, 14 | { 15 | "type":"INT64", 16 | "name":"fieldIndex", 17 | "mode":"NULLABLE" 18 | } 19 | ], 20 | "type":"RECORD", 21 | "name":"uppercaseDetails", 22 | "mode":"NULLABLE" 23 | }, 24 | { 25 | "fields":[ 26 | { 27 | "type":"STRING", 28 | "name":"field", 29 | "mode":"NULLABLE" 30 | }, 31 | { 32 | "type":"INT64", 33 | "name":"fieldIndex", 34 | "mode":"NULLABLE" 35 | } 36 | ], 37 | "type":"RECORD", 38 | "name":"lowercaseDetails", 39 | "mode":"NULLABLE" 40 | }, 41 | { 42 | "fields":[ 43 | { 44 | "type":"STRING", 45 | "name":"kind", 46 | "mode":"NULLABLE" 47 | }, 48 | { 49 | "type":"STRING", 50 | "name":"expressionValue", 51 | "mode":"NULLABLE" 52 | }, 53 | { 54 | "type":"BOOL", 55 | "name":"caseSensitive", 56 | "mode":"NULLABLE" 57 | }, 58 | { 59 | "type":"STRING", 60 | "name":"field", 61 | "mode":"NULLABLE" 62 | }, 63 | { 64 | "type":"INT64", 65 | "name":"fieldIndex", 66 | "mode":"NULLABLE" 67 | }, 68 | { 69 | "type":"STRING", 70 | "name":"matchType", 71 | "mode":"NULLABLE" 72 | } 73 | ], 74 | "type":"RECORD", 75 | "name":"excludeDetails", 76 | "mode":"NULLABLE" 77 | }, 78 | { 79 | "fields":[ 80 | { 81 | "type":"STRING", 82 | "name":"kind", 83 | "mode":"NULLABLE" 84 | }, 85 | { 86 | "type":"STRING", 87 | "name":"expressionValue", 88 | "mode":"NULLABLE" 89 | }, 90 | { 91 | "type":"BOOL", 92 | "name":"caseSensitive", 93 | "mode":"NULLABLE" 94 | }, 95 | { 96 | "type":"STRING", 97 | "name":"field", 98 | "mode":"NULLABLE" 99 | }, 100 | { 101 | "type":"INT64", 102 | "name":"fieldIndex", 103 | "mode":"NULLABLE" 104 | }, 105 | { 106 | "type":"STRING", 107 | "name":"matchType", 108 | "mode":"NULLABLE" 109 | } 110 | ], 111 | "type":"RECORD", 112 | "name":"includeDetails", 113 | "mode":"NULLABLE" 114 | }, 115 | { 116 | "type":"INT64", 117 | "name":"id", 118 | "mode":"NULLABLE" 119 | }, 120 | { 121 | "type":"INT64", 122 | "name":"accountId", 123 | "mode":"NULLABLE" 124 | }, 125 | { 126 | "type":"STRING", 127 | "name":"kind", 128 | "mode":"NULLABLE" 129 | }, 130 | { 131 | "type":"STRING", 132 | "name":"name", 133 | "mode":"NULLABLE" 134 | }, 135 | { 136 | "type":"TIMESTAMP", 137 | "name":"created", 138 | "mode":"NULLABLE" 139 | }, 140 | { 141 | "fields":[ 142 | { 143 | "type":"INT64", 144 | "name":"outputToFieldIndex", 145 | "mode":"NULLABLE" 146 | }, 147 | { 148 | "type":"STRING", 149 | "name":"outputConstructor", 150 | "mode":"NULLABLE" 151 | }, 152 | { 153 | "type":"STRING", 154 | "name":"extractA", 155 | "mode":"NULLABLE" 156 | }, 157 | { 158 | "type":"INT64", 159 | "name":"fieldAIndex", 160 | "mode":"NULLABLE" 161 | }, 162 | { 163 | "type":"STRING", 164 | "name":"fieldB", 165 | "mode":"NULLABLE" 166 | }, 167 | { 168 | "type":"STRING", 169 | "name":"extractB", 170 | "mode":"NULLABLE" 171 | }, 172 | { 173 | "type":"STRING", 174 | "name":"outputToField", 175 | "mode":"NULLABLE" 176 | }, 177 | { 178 | "type":"INT64", 179 | "name":"fieldBIndex", 180 | "mode":"NULLABLE" 181 | }, 182 | { 183 | "type":"STRING", 184 | "name":"fieldA", 185 | "mode":"NULLABLE" 186 | }, 187 | { 188 | "type":"BOOL", 189 | "name":"overrideOutputField", 190 | "mode":"NULLABLE" 191 | }, 192 | { 193 | "type":"BOOL", 194 | "name":"caseSensitive", 195 | "mode":"NULLABLE" 196 | }, 197 | { 198 | "type":"BOOL", 199 | "name":"fieldARequired", 200 | "mode":"NULLABLE" 201 | }, 202 | { 203 | "type":"BOOL", 204 | "name":"fieldBRequired", 205 | "mode":"NULLABLE" 206 | } 207 | ], 208 | "type":"RECORD", 209 | "name":"advancedDetails", 210 | "mode":"NULLABLE" 211 | }, 212 | { 213 | "fields":[ 214 | { 215 | "type":"STRING", 216 | "name":"field", 217 | "mode":"NULLABLE" 218 | }, 219 | { 220 | "type":"INT64", 221 | "name":"fieldIndex", 222 | "mode":"NULLABLE" 223 | }, 224 | { 225 | "type":"STRING", 226 | "name":"searchString", 227 | "mode":"NULLABLE" 228 | }, 229 | { 230 | "type":"STRING", 231 | "name":"replaceString", 232 | "mode":"NULLABLE" 233 | }, 234 | { 235 | "type":"BOOL", 236 | "name":"caseSensitive", 237 | "mode":"NULLABLE" 238 | } 239 | ], 240 | "type":"RECORD", 241 | "name":"searchAndReplaceDetails", 242 | "mode":"NULLABLE" 243 | }, 244 | { 245 | "fields":[ 246 | { 247 | "type":"STRING", 248 | "name":"href", 249 | "mode":"NULLABLE" 250 | }, 251 | { 252 | "type":"STRING", 253 | "name":"type", 254 | "mode":"NULLABLE" 255 | } 256 | ], 257 | "type":"RECORD", 258 | "name":"parentLink", 259 | "mode":"NULLABLE" 260 | }, 261 | { 262 | "type":"STRING", 263 | "name":"type", 264 | "mode":"NULLABLE" 265 | }, 266 | { 267 | "type":"STRING", 268 | "name":"selfLink", 269 | "mode":"NULLABLE" 270 | } 271 | ] -------------------------------------------------------------------------------- /report_tables/property_overview/function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import functions_framework 17 | from google.cloud import bigquery 18 | from google.cloud.bigquery.table import TableReference 19 | 20 | PROJECT_ID = os.environ.get('PROJECT_ID') 21 | DATASET_ID = os.environ.get('DATASET_ID') 22 | TABLE_ID = os.environ.get('TABLE_ID') 23 | 24 | # Construct a BigQuery client object. 25 | client = bigquery.Client() 26 | 27 | @functions_framework.http 28 | def main(request): 29 | query = """ 30 | SELECT 31 | summaries.display_name AS account_name, 32 | summaries.account AS account_id, 33 | property_summaries.display_name AS property_name, 34 | property_summaries.property AS property_id, 35 | properties.service_level as standard_or_360, 36 | properties.google_signals_state as google_signals_state, 37 | SUM(DISTINCT CASE WHEN data_streams.type = 'WEB_DATA_STREAM' THEN 1 ELSE 0 END) as number_of_web_streams, 38 | SUM(DISTINCT CASE WHEN data_streams.type = 'ANDROID_APP_DATA_STREAM' THEN 1 ELSE 0 END) as number_of_android_app_streams, 39 | SUM(DISTINCT CASE WHEN data_streams.type = 'IOS_APP_DATA_STREAM' THEN 1 ELSE 0 END) as number_of_ios_app_streams, 40 | (CASE WHEN SUM(DISTINCT CASE WHEN data_streams.type = 'WEB_DATA_STREAM' THEN 1 ELSE 0 END) > 0 THEN TRUE ELSE FALSE END) AS web_streams_created, 41 | (CASE WHEN SUM(DISTINCT CASE WHEN data_streams.type = 'ANDROID_APP_DATA_STREAM' THEN 1 ELSE 0 END) > 0 THEN TRUE ELSE FALSE END) AS android_streams_created, 42 | (CASE WHEN SUM(DISTINCT CASE WHEN data_streams.type = 'IOS_APP_DATA_STREAM' THEN 1 ELSE 0 END) > 0 THEN TRUE ELSE FALSE END) AS ios_streams_created, 43 | (CASE WHEN SUM(conversion_events.is_custom) > 0 THEN TRUE ELSE FALSE END) AS custom_conversions_created, 44 | SUM(conversion_events.is_custom) AS number_of_custom_conversion_events, 45 | (CASE WHEN COUNT(DISTINCT google_ads_links.customer_id) > 0 THEN TRUE ELSE FALSE END) AS google_ads_linked, 46 | COUNT(DISTINCT google_ads_links.customer_id) AS number_of_google_ads_links, 47 | (CASE WHEN COUNT(DISTINCT dv360_links.name) > 0 THEN TRUE ELSE FALSE END) AS dv360_linked, 48 | COUNT(DISTINCT dv360_links.name) AS number_of_dv360_links, 49 | (CASE WHEN COUNT(DISTINCT firebase_links.project) > 0 THEN TRUE ELSE FALSE END) AS firebase_linked, 50 | COUNT(DISTINCT firebase_links.project) AS number_of_firebase_links, 51 | (CASE WHEN COUNT(DISTINCT custom_dimensions.parameter_name) > 0 THEN TRUE ELSE FALSE END) AS custom_dimensions_created, 52 | COUNT(DISTINCT custom_dimensions.parameter_name) AS number_of_custom_dimensions, 53 | (CASE WHEN COUNT(DISTINCT custom_metrics.parameter_name) > 0 THEN TRUE ELSE FALSE END) AS custom_metrics_created, 54 | COUNT(DISTINCT custom_metrics.parameter_name) AS number_of_custom_metrics, 55 | (CASE WHEN COUNT(DISTINCT measurement_protocol_secrets.secret_value) > 0 THEN TRUE ELSE FALSE END) AS measurement_protocol_secret_created, 56 | COUNT(DISTINCT measurement_protocol_secrets.secret_value) AS number_of_measurement_protocol_secrets, 57 | (CASE WHEN SUM(audiences.is_custom) > 0 THEN TRUE ELSE FALSE END) AS custom_audiences_created, 58 | SUM(audiences.is_custom) AS number_of_custom_audiences 59 | FROM 60 | analytics_settings_database.ga4_account_summaries AS summaries, 61 | UNNEST(property_summaries) AS property_summaries 62 | LEFT JOIN ( 63 | SELECT 64 | name AS property_id, 65 | service_level, 66 | google_signals_settings.state AS google_signals_state 67 | FROM 68 | analytics_settings_database.ga4_properties 69 | WHERE 70 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS properties 71 | ON 72 | property_summaries.property = properties.property_id 73 | LEFT JOIN ( 74 | SELECT 75 | property AS property_id, 76 | type, 77 | name AS id 78 | FROM 79 | analytics_settings_database.ga4_data_streams 80 | WHERE 81 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS data_streams 82 | ON 83 | property_summaries.property = data_streams.property_id 84 | LEFT JOIN ( 85 | SELECT 86 | property AS property_id, 87 | name AS id, 88 | event_name, 89 | CASE 90 | WHEN REGEXP_CONTAINS('purchase|first_open|in_app_purchase|app_store_subscription_convert|app_store_subscription_renew', event_name ) THEN 0 91 | ELSE 92 | 1 93 | END 94 | AS is_custom 95 | FROM 96 | analytics_settings_database.ga4_conversion_events 97 | WHERE 98 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS conversion_events 99 | ON 100 | property_summaries.property = conversion_events.property_id 101 | LEFT JOIN ( 102 | SELECT 103 | property AS property_id, 104 | customer_id 105 | FROM 106 | analytics_settings_database.ga4_google_ads_links 107 | WHERE 108 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS google_ads_links 109 | ON 110 | property_summaries.property = google_ads_links.property_id 111 | LEFT JOIN ( 112 | SELECT 113 | property AS property_id, 114 | name 115 | FROM 116 | analytics_settings_database.ga4_dv360_links 117 | WHERE 118 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS dv360_links 119 | ON 120 | property_summaries.property = google_ads_links.property_id 121 | LEFT JOIN ( 122 | SELECT 123 | property AS property_id, 124 | project 125 | FROM 126 | analytics_settings_database.ga4_firebase_links 127 | WHERE 128 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS firebase_links 129 | ON 130 | property_summaries.property = firebase_links.property_id 131 | LEFT JOIN ( 132 | SELECT 133 | property AS property_id, 134 | parameter_name 135 | FROM 136 | analytics_settings_database.ga4_custom_dimensions 137 | WHERE 138 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS custom_dimensions 139 | ON 140 | property_summaries.property = custom_dimensions.property_id 141 | LEFT JOIN ( 142 | SELECT 143 | property AS property_id, 144 | parameter_name 145 | FROM 146 | analytics_settings_database.ga4_custom_metrics 147 | WHERE 148 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS custom_metrics ON 149 | property_summaries.property = custom_metrics.property_id 150 | LEFT JOIN ( 151 | SELECT 152 | property AS property_id, 153 | secret_value 154 | FROM 155 | analytics_settings_database.ga4_measurement_protocol_secrets 156 | WHERE 157 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS measurement_protocol_secrets 158 | ON 159 | property_summaries.property = measurement_protocol_secrets.property_id 160 | LEFT JOIN ( 161 | SELECT 162 | property AS property_id, 163 | name, 164 | CASE 165 | WHEN REGEXP_CONTAINS('Purchasers|All Users', display_name) THEN 0 166 | ELSE 167 | 1 168 | END as is_custom 169 | FROM 170 | analytics_settings_database.ga4_audiences 171 | WHERE 172 | DATE(_PARTITIONTIME) = CURRENT_DATE()) AS audiences ON 173 | property_summaries.property = audiences.property_id 174 | WHERE 175 | DATE(_PARTITIONTIME) = CURRENT_DATE() 176 | GROUP BY 177 | 1, 2, 3, 4, 5, 6 178 | """ 179 | # table = TableReference(projectId=PROJECT_ID, datasetId=DATASET_ID, tableId=TABLE_ID) 180 | table_ref = client.dataset(DATASET_ID).table(TABLE_ID) 181 | job_config = bigquery.QueryJobConfig() 182 | job_config.write_disposition = 'WRITE_APPEND' 183 | job_config.destination = table_ref 184 | query_job = client.query(query, job_config=job_config) 185 | query_job.result() 186 | return 'complete' 187 | 188 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /settings_downloader_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Retrieves Google Analytics settings and saves them to BigQuery.""" 15 | 16 | import json 17 | import time 18 | import os 19 | import humps 20 | import google.auth 21 | from google.analytics.admin import AnalyticsAdminServiceClient 22 | from google.analytics.admin_v1alpha.types import ListPropertiesRequest 23 | from google.analytics.admin_v1alpha.types import DataStream 24 | from google.protobuf.json_format import MessageToDict 25 | from google.cloud import bigquery 26 | 27 | # Construct a BigQuery client object. 28 | bigquery_client = bigquery.Client() 29 | 30 | SERVICE_ACCOUNT_FILE = 'credentials.json' 31 | GA_SCOPES = ['https://www.googleapis.com/auth/analytics.readonly'] 32 | DATASET_ID = 'analytics_settings_database' 33 | 34 | REQUEST_DELAY = .2 35 | 36 | def ga_settings_download(event): 37 | """Retrieve GA settings and save them to a series of BigQuery Tables. 38 | 39 | Args: 40 | event (dict): Event payload. 41 | 42 | Returns: 43 | A string indicating that the settings were downloaded. 44 | """ 45 | # Authenticate and construct ga service. 46 | admin_api = authorize_ga_apis() 47 | 48 | # Get GA4 entitity settings. 49 | lists = {} 50 | lists |= list_ga4_resources(admin_api) 51 | 52 | dataset = bigquery_client.dataset(DATASET_ID) 53 | for key in lists: 54 | data = lists[key] 55 | if data: 56 | # Create newline delimited JSON file to to loaded into BigQuery. 57 | with open('/tmp/data.json', 'w') as json_file: 58 | for resource in data: 59 | json.dump(resource, json_file) 60 | json_file.write('\n') 61 | json_file.close() 62 | r = open('/tmp/data.json', 'rb') 63 | # Create load job. 64 | job_config = bigquery.LoadJobConfig() 65 | job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON 66 | table = dataset.table(key) 67 | job = bigquery_client.load_table_from_file( 68 | r, table, job_config=job_config) 69 | # Load the data into BigQuery. 70 | result = job.result() 71 | print(result) 72 | # Delete the temporary file. 73 | os.remove('/tmp/data.json') 74 | print('GA settings import complete') 75 | return 'success' 76 | 77 | def authorize_ga_apis(): 78 | """Fetches the Google Analytics Admin API client. 79 | 80 | Returns: 81 | The admin API client. 82 | """ 83 | source_credentials, project_id = google.auth.default(scopes=GA_SCOPES) 84 | ga_admin_api = AnalyticsAdminServiceClient(credentials=source_credentials) 85 | return ga_admin_api 86 | 87 | def list_ga4_resources(admin_api): 88 | """Get a dictionary of GA4 entity settings based on type. 89 | 90 | Args: 91 | admin_api: The Admin API client. 92 | 93 | Returns: 94 | A dictionary of GA4 resource lists. 95 | """ 96 | resources = { 97 | 'ga4_account_summaries': [], 98 | 'ga4_accounts': [], 99 | 'ga4_properties': [], 100 | 'ga4_data_streams': [], 101 | 'ga4_measurement_protocol_secrets': [], 102 | 'ga4_conversion_events': [], 103 | 'ga4_custom_dimensions': [], 104 | 'ga4_custom_metrics': [], 105 | 'ga4_dv360_link_proposals': [], 106 | 'ga4_dv360_links': [], 107 | 'ga4_firebase_links': [], 108 | 'ga4_google_ads_links': [], 109 | 'ga4_audiences': [], 110 | 'ga4_enhanced_measurement_settings': [], 111 | 'ga4_sa360_links': [], 112 | 'ga4_bigquery_links': [], 113 | 'ga4_expanded_data_sets': [], 114 | 'ga4_channel_groups': [], 115 | 'ga4_adsense_links': [], 116 | 'ga4_event_create_rules': [], 117 | 'ga4_sk_ad_network_conversion_value_schemas': [] 118 | } 119 | # Account summaries 120 | for account_summary in admin_api.list_account_summaries( 121 | request={'page_size': 200}): 122 | summaries_dict = humps.decamelize(MessageToDict(account_summary._pb)) 123 | resources['ga4_account_summaries'].append(summaries_dict) 124 | time.sleep(REQUEST_DELAY) 125 | # Accounts 126 | for account in admin_api.list_accounts(request={'page_size': 200}): 127 | account_dict = humps.decamelize(MessageToDict(account._pb)) 128 | resources['ga4_accounts'].append(account_dict) 129 | time.sleep(REQUEST_DELAY) 130 | # Properties 131 | for account_summary in resources['ga4_account_summaries']: 132 | prop_request = ListPropertiesRequest( 133 | filter=f"ancestor:{account_summary['account']}", 134 | page_size=200) 135 | # Settings specific to properties 136 | for prop in admin_api.list_properties(prop_request): 137 | property_dict = humps.decamelize(MessageToDict(prop._pb)) 138 | time.sleep(REQUEST_DELAY) 139 | # Set data retention settings for each property 140 | data_retention_settings = admin_api.get_data_retention_settings( 141 | name=(prop.name + '/dataRetentionSettings')) 142 | data_retention_dict = humps.decamelize( 143 | MessageToDict(data_retention_settings._pb)) 144 | property_dict['data_sharing_settings'] = data_retention_dict 145 | time.sleep(REQUEST_DELAY) 146 | # Set Google Signals settings for each property 147 | google_signals_settings = admin_api.get_google_signals_settings( 148 | name=(prop.name + '/googleSignalsSettings')) 149 | google_signals_dict = humps.decamelize( 150 | MessageToDict(google_signals_settings._pb)) 151 | property_dict['google_signals_settings'] = google_signals_dict 152 | time.sleep(REQUEST_DELAY) 153 | # Set Attribution settings for each property 154 | attribution_settings = admin_api.get_attribution_settings( 155 | name=(prop.name + '/attributionSettings')) 156 | attribution_dict = humps.decamelize( 157 | MessageToDict(attribution_settings._pb)) 158 | property_dict['attribution_settings'] = attribution_dict 159 | # Append property to list of properties 160 | resources['ga4_properties'].append(property_dict) 161 | time.sleep(REQUEST_DELAY) 162 | # Settings below the property level 163 | for property_summary in account_summary['property_summaries']: 164 | property_path = property_summary['property'] 165 | property_display_name = property_summary['display_name'] 166 | # Data streams 167 | for data_stream in admin_api.list_data_streams( 168 | request={'parent': property_path, 'page_size': 200}): 169 | data_stream_dict = format_resource_dict( 170 | data_stream, property_path, property_display_name) 171 | time.sleep(REQUEST_DELAY) 172 | # Measurement protocol secrets 173 | for mps in admin_api.list_measurement_protocol_secrets( 174 | parent=data_stream.name): 175 | mps_dict = format_resource_dict( 176 | mps, property_path, property_display_name) 177 | mps_dict['type'] = DataStream.DataStreamType(data_stream.type_).name 178 | mps_dict['stream_name'] = data_stream.name 179 | resources['ga4_measurement_protocol_secrets'].append(mps_dict) 180 | time.sleep(REQUEST_DELAY) 181 | # Event Create Rules 182 | for ecr in admin_api.list_event_create_rules( 183 | request={'page_size': 200, 'parent': data_stream.name}): 184 | resource_dict = format_resource_dict( 185 | resource, property_path, property_display_name) 186 | resource_dict['type'] = DataStream.DataStreamType(data_stream.type_).name 187 | resource_dict['stream_name'] = data_stream.name 188 | resources['ga4_event_create_rules'].append(resource_dict) 189 | time.sleep(REQUEST_DELAY) 190 | if DataStream.DataStreamType(data_stream.type_).name == 'WEB_DATA_STREAM': 191 | # Enhanced measurement settings 192 | ems = admin_api.get_enhanced_measurement_settings( 193 | name=data_stream.name + '/enhancedMeasurementSettings') 194 | ems_dict = format_resource_dict( 195 | ems, property_path, property_display_name) 196 | ems_dict['stream_name'] = data_stream.name 197 | resources['ga4_enhanced_measurement_settings'].append(ems_dict) 198 | time.sleep(REQUEST_DELAY) 199 | if DataStream.DataStreamType(data_stream.type_).name == 'IOS_APP_DATA_STREAM': 200 | # SK Ad Network Conversion Value Schema settings 201 | for resource in admin_api.list_sk_ad_network_conversion_value_schemas( 202 | request={'page_size': 200, 'parent': data_stream.name}): 203 | resource_dict = format_resource_dict( 204 | resource, property_path, property_display_name) 205 | resource_dict['stream_name'] = data_stream.name 206 | resources['ga4_sk_ad_network_conversion_value_schemas'].append( 207 | resource_dict) 208 | time.sleep(REQUEST_DELAY) 209 | # Events 210 | for event in admin_api.list_conversion_events( 211 | request={'parent': property_path, 'page_size': 200}): 212 | formatted_dict = format_resource_dict( 213 | event, property_path, property_display_name) 214 | resources['ga4_conversion_events'].append(formatted_dict) 215 | time.sleep(REQUEST_DELAY) 216 | # Custom dimensions 217 | for cd in admin_api.list_custom_dimensions( 218 | request={'parent': property_path, 'page_size': 200}): 219 | formatted_dict = format_resource_dict( 220 | cd, property_path, property_display_name) 221 | resources['ga4_custom_dimensions'].append(formatted_dict) 222 | time.sleep(REQUEST_DELAY) 223 | # Custom metrics 224 | for cm in admin_api.list_custom_metrics( 225 | request={'parent': property_path, 'page_size': 200}): 226 | formatted_dict = format_resource_dict( 227 | cm, property_path, property_display_name) 228 | resources['ga4_custom_metrics'].append(formatted_dict) 229 | time.sleep(REQUEST_DELAY) 230 | # Google ads links 231 | for link in admin_api.list_google_ads_links( 232 | request={'parent': property_path, 'page_size': 200}): 233 | formatted_dict = format_resource_dict( 234 | link, property_path, property_display_name) 235 | resources['ga4_google_ads_links'].append(formatted_dict) 236 | time.sleep(REQUEST_DELAY) 237 | # Firebase links 238 | for link in admin_api.list_firebase_links( 239 | request={'parent': property_path, 'page_size': 200}): 240 | formatted_dict = format_resource_dict( 241 | link, property_path, property_display_name) 242 | resources['ga4_firebase_links'].append(formatted_dict) 243 | time.sleep(REQUEST_DELAY) 244 | # DV360 advertiser links 245 | for link in admin_api.list_display_video360_advertiser_links( 246 | request={'parent': property_path, 'page_size': 200}): 247 | formatted_dict = format_resource_dict( 248 | link, property_path, property_display_name) 249 | resources['ga4_dv360_links'].append(formatted_dict) 250 | time.sleep(REQUEST_DELAY) 251 | # DV360 advertiser link proposals 252 | for proposal in ( 253 | admin_api.list_display_video360_advertiser_link_proposals( 254 | request={'parent': property_path, 'page_size': 200})): 255 | formatted_dict = format_resource_dict( 256 | proposal, property_path, property_display_name) 257 | resources['ga4_dv360_link_proposals'].append(formatted_dict) 258 | time.sleep(REQUEST_DELAY) 259 | # SA360 advertiser links 260 | for link in admin_api.list_search_ads360_links( 261 | request={'parent': property_path, 'page_size': 200}): 262 | formatted_dict = format_resource_dict( 263 | link, property_path, property_display_name) 264 | resources['ga4_sa360_links'].append(formatted_dict) 265 | time.sleep(REQUEST_DELAY) 266 | # BigQuery links 267 | for link in admin_api.list_big_query_links( 268 | request={'parent': property_path, 'page_size': 200}): 269 | formatted_dict = format_resource_dict( 270 | link, property_path, property_display_name) 271 | resources['ga4_bigquery_links'].append(formatted_dict) 272 | time.sleep(REQUEST_DELAY) 273 | # Audiences 274 | for audience in admin_api.list_audiences( 275 | request={'parent': property_path, 'page_size': 200}): 276 | formatted_dict = format_resource_dict( 277 | audience, property_path, property_display_name) 278 | if 'filter_clauses' in formatted_dict: 279 | string_clauses = json.dumps(formatted_dict['filter_clauses']) 280 | formatted_dict['filter_clauses'] = string_clauses 281 | resources['ga4_audiences'].append(formatted_dict) 282 | time.sleep(REQUEST_DELAY) 283 | # Expanded Data Sets 284 | for data_set in admin_api.list_expanded_data_sets( 285 | request={'parent': property_path, 'page_size': 200}): 286 | formatted_dict = format_resource_dict( 287 | data_set, property_path, property_display_name) 288 | if 'dimension_ilter_expression' in formatted_dict: 289 | string_clauses = json.dumps( 290 | formatted_dict['dimension_ilter_expression']) 291 | formatted_dict['dimension_ilter_expression'] = string_clauses 292 | resources['ga4_expanded_data_sets'].append(formatted_dict) 293 | time.sleep(REQUEST_DELAY) 294 | # Custom Channel Groups 295 | for resource in admin_api.list_channel_groups( 296 | request={'parent': property_path, 'page_size': 200}): 297 | formatted_dict = format_resource_dict( 298 | resource, property_path, property_display_name) 299 | if 'grouping_rule' in formatted_dict: 300 | string_clauses = json.dumps(formatted_dict['grouping_rule']) 301 | formatted_dict['grouping_rule'] = string_clauses 302 | resources['ga4_channel_groups'].append(formatted_dict) 303 | time.sleep(REQUEST_DELAY) 304 | # AdSense Links 305 | for resource in admin_api.list_ad_sense_links( 306 | request={'parent': property_path, 'page_size': 200}): 307 | formatted_dict = format_resource_dict( 308 | resource, property_path, property_display_name) 309 | resources['ga4_adsense_links'].append(formatted_dict) 310 | time.sleep(REQUEST_DELAY) 311 | return resources 312 | 313 | def format_resource_dict(data, property_path, property_display_name): 314 | data_dict = humps.decamelize(MessageToDict(data._pb)) 315 | data_dict['property'] = property_path 316 | data_dict['property_display_name'] = property_display_name 317 | return data_dict 318 | --------------------------------------------------------------------------------