├── CONTRIBUTING.md ├── LICENSE ├── ML_Model_Creation_Flow.md ├── README.md ├── bundle.js ├── bundle.js.map ├── explores ├── .gitkeep ├── model_info.explore.lkml └── selection_summary.explore.lkml ├── images ├── .gitkeep ├── 0 - Applications Menu.png ├── 1 - Landing Page.png ├── 10 - Confusion Matrix.png ├── 11 - ROC Curve.png ├── 12.1 - Explainable AI - Model Level.png ├── 12.2 - Explainable AI - Class Level.png ├── 13 - Select Data for Predictions.png ├── 14 - Generate Predictions.png ├── 15 - BQ Resources Created by Application.png ├── 2 - Choose Objective.png ├── 3 - Choose Looker Explore.png ├── 4 - Select Input Data.png ├── 5 - Select Model Target.png ├── 6 - Review Training Data Summary.png ├── 7 - Advanced Model Settings Modal.png ├── 8 - Model Training Screen.png ├── 9.1 - Evaluation Metrics.png ├── 9.2 - Evaluation Metrics with Description.png └── AIPlatform-36-color.svg ├── manifest.lkml ├── marketplace.json ├── ml_accelerator.model.lkml └── views ├── .gitkeep ├── model_info.view.lkml └── selection_summary.view.lkml /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement (CLA). You (or your employer) retain the copyright to your 10 | contribution; this simply gives us permission to use and redistribute your 11 | contributions as part of the project. Head over to 12 | to see your current agreements on file or 13 | to sign a new one. 14 | 15 | You generally only need to submit a CLA once, so if you've already submitted one 16 | (even if it was for a different project), you probably don't need to do it 17 | again. 18 | 19 | ## Code Reviews 20 | 21 | All submissions, including submissions by project members, require review. We 22 | use GitHub pull requests for this purpose. Consult 23 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 24 | information on using pull requests. 25 | 26 | ## Community Guidelines 27 | 28 | This project follows 29 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Looker Open Source 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /ML_Model_Creation_Flow.md: -------------------------------------------------------------------------------- 1 | ## Example ML Model Creation Flow 2 | 3 | #### Open the Machnine Learning Accelerator from the Left Side Panel 4 | 5 | ![Applications Menu](https://github.com/looker-open-source/app-ml-accelerator/blob/marketplace-release-v1.0.1/images/0%20-%20Applications%20Menu.png) 6 | 7 | #### From the **Landing Page** select **+ Create New Model** 8 | 9 | ![Landing Page](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/1%20-%20Landing%20Page.png) 10 | 11 | #### Choose an Objective 12 | 13 | ![Choose Objective](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/2%20-%20Choose%20Objective.png) 14 | 15 | #### Choose a Looker Explore 16 | 17 | ![Choose Looker Explore](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/3%20-%20Choose%20Looker%20Explore.png) 18 | 19 | #### Select Input Data 20 | 21 | ![Select Input Data](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/4%20-%20Select%20Input%20Data.png) 22 | 23 | #### Select Model Target 24 | 25 | ![Select Model Target](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/5%20-%20Select%20Model%20Target.png) 26 | 27 | #### Review Training Data Summary 28 | 29 | ![Review Training Data Summary](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/6%20-%20Review%20Training%20Data%20Summary.png) 30 | 31 | #### Review Model Evaluation Metrics 32 | 33 | ![Review Model Evaluation Metrics](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/9.2%20-%20Evaluation%20Metrics%20with%20Description.png) 34 | 35 | #### Review Model Confusion Matrix 36 | 37 | ![Review Model Confusion Matrix](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/10%20-%20Confusion%20Matrix.png) 38 | 39 | #### Review Model Feature Importance 40 | 41 | ![Review Model Feature Importance](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/12.1%20-%20Explainable%20AI%20-%20Model%20Level.png) 42 | 43 | #### Select Data for Predictions 44 | 45 | ![Select Data for Predictions](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/13%20-%20Select%20Data%20for%20Predictions.png) 46 | 47 | #### Generate Predictions 48 | 49 | ![Generate Predictions](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/14%20-%20Generate%20Predictions.png) 50 | 51 | #### BigQuery Resources Created by Application 52 | 53 | ![BigQuery Resources Created by Application](https://github.com/looker-open-source/app-ml-accelerator/blob/main/images/15%20-%20BQ%20Resources%20Created%20by%20Application.png) 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine Learning Accelerator 2 | 3 | The ML Accelerator is a purpose-built Looker application designed to give business users access to BigQuery and Vertex AI's machine learning capabilities. It provides a user-friendly interface designed to guide the user through each step of creating a machine learning model. Because of its simple, no-code interface, the application serves as a pathway for business analysts to learn and use predictive analytics in Looker. 4 | 5 | View the [ML Model Creation Flow](https://github.com/looker-open-source/app-ml-accelerator/blob/main/ML_Model_Creation_Flow.md) document for an example of an end-to-end user journey. 6 | 7 | #### **IMPORTANT** 8 | 9 | **Additional configuration is required after installation. A Looker Admin should complete the application configuration using the Installation Instructions below.** 10 | 11 | Report bugs or feature requests with [this form](https://docs.google.com/forms/d/e/1FAIpQLSd97ptoU3TUuasZeFjSBHT9FQiyrDzjHUm7NTspEjz5kwNSAA/viewform). Contact [ml-accelerator-feedback@google.com](mailto:ml-accelerator-feedback@google.com) with questions or comments. 12 | 13 | --- 14 | 15 | ### INSTALLATION INSTRUCTIONS 16 | 17 | #### 1. Choose a BigQuery Connection 18 | 19 | You will need to select a BigQuery connection during installation. The application can only be used with a single connection to prevent data from moving between connections. The connection chosen will determine which Looker Explores will be accessible from within the application. 20 | 21 | #### 2. Adjust Service Account Roles 22 | 23 | The service account used by the BigQuery connection chosen in Step 1 should have the following IAM predefined roles. 24 | - BigQuery Data Editor 25 | - BigQuery Job User 26 | - Vertex AI User 27 | 28 | #### 3. Create BigQuery Dataset for ML Models 29 | 30 | Create a dataset (e.g., `looker_bqml`) in the BigQuery connection's GCP project. 31 | 32 | #### 4. Install Application 33 | 34 | The application can be installed directly from [Looker Marketplace](https://marketplace.looker.com/) (recommended) or manually installed following the steps below. 35 | 36 | ##### Option A: Marketplace Install 37 | Refer to the [Looker Docs for installing a tool from Marketplace](https://cloud.google.com/looker/docs/marketplace#installing_a_tool_from_the_marketplace). Select the BigQuery connection name chosen in Step 1 during installation. 38 | 39 | ##### Option B: Manual Install 40 | - [Fork this GitHub repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository) 41 | - [Create a blank LookML project](https://cloud.google.com/looker/docs/create-projects#creating_a_blank_project) named `marketplace_bqml_ext` 42 | 43 | **IMPORTANT: The LookML project must be named `marketplace_bqml_ext`** 44 | 45 | - [Connect the new LookML project to the forked repository](https://cloud.google.com/looker/docs/setting-up-git-connection) 46 | - Update the value of the CONNECTION_NAME constant in the `manifest.lkml` file 47 | - [Commit and deploy changes to production](https://cloud.google.com/looker/docs/version-control-and-deploying-changes#getting_your_changes_to_production) 48 | 49 | #### 5. Configure Application with User Attributes 50 | 51 | The application uses three [Looker user attributes](https://cloud.google.com/looker/docs/admin-panel-users-user-attributes) to store its configuration settings. The following user attributes are required for the application to work properly. Each user attribute needs to be named exactly as listed below with a data type of `String`. The recommended setting for user access is `View`. 52 | 53 | Create the following user attributes and set their default values. 54 | 55 | | **Required User Attribute Name** | **Default Value Description** | 56 | |-----------------------------------------------------------------|-------------------------------------------------------------------| 57 | | marketplace_bqml_ext_ml_accelerator_bigquery_connection_name | Connection name chosen in Step 1 | 58 | | marketplace_bqml_ext_ml_accelerator_gcp_project | Projectd ID of the BigQuery dataset created in Step 3 | 59 | | marketplace_bqml_ext_ml_accelerator_bqml_model_dataset_name | Name of BigQuery dataset created in Step 3 (e.g., `looker_bqml`) | 60 | 61 | #### 6. Create a Looker Role to Manage User Access 62 | 63 | The application is designed for users with access to Explores and SQL Runner in Looker. Users will need the following permissions to use the application. 64 | - `explore` 65 | - `use_sql_runner` 66 | 67 | We recommend creating a new Looker role to easily manage user access to the application and guarantee users have the required permissions above. 68 | - Create a new Looker model set named `ML Accelerator` containing the LookML model `ml_accelerator` 69 | - Create a new Looker permission set named `ML Accelerator` containing all the permisions in the [default User permission set](https://cloud.google.com/looker/docs/admin-panel-users-roles#default_permission_sets) AND the `use_sql_runner` permission 70 | - Create a new Looker role named `ML Accelerator` using the new model and permission set 71 | - Assign the `ML Accelerator` role to Looker users and groups 72 | 73 | #### 7. Setup AI-Generated Model Evaluation Summaries 74 | 75 | After release 2.2, the application can use text generating AI to summarize the model evaluation to more clearly communicate model performance. This optional feature requires additional setup. 76 | 77 | ##### 7a: Add an External Connection from Bigquery to Vertex 78 | In BigQuery, an [external connection](https://cloud.google.com/bigquery/docs/external-data-sources) is required to connect it to pre-trained models in Vertex AI. If one is not already set up, you must do so. A tutorial can also be found [here](https://cloud.google.com/bigquery/docs/generate-text-tutorial). 79 | 1. Under the same gcp project already in use for the application, verify the [BigQuery Connection](https://console.cloud.google.com/apis/library/bigqueryconnection.googleapis.com) and [Vertex AI](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com) APIs are both enabled. 80 | 2. In BigQuery click “add,” then "Connections to external data sources." 81 | 3. Select "BigLake and remote function" and use the same location as the dataset already in use by the application 82 | 4. The ID will be the name of your connection. Since it could be used to connect to any number of pre-trained models in vertex it is wise to choose something generic, such as “ext-vertex-ai” 83 | 5. Create the connection 84 | 6. Go to the connection and copy the service account ID. In order to access remote functions from Vertex AI, the [BigQuery connection delegation service agent](https://cloud.google.com/iam/docs/service-agents#bigquery-connection-delegation-service-agent) (of the form bqcx-[#]@gcp-sa-bigquery-condel.iam.gserviceaccount.com) that is associated with this connection must have the "Vertex AI User" role, which can be added in IAM. 85 | 86 | ##### 7b: Create the Remote Text-Generation Model 87 | 88 | In BigQuery, enter the following statement in the query editor (this code uses the suggested naming conventions for the the steps above and assumes region is US-Multi). The text-bison@002 model is suggested, but other LLM models with good performance generating text could also be used. The model_name will be later added as a User Attribute value. A suggestion for model_name is "mla-text-bison" 89 | ``` 90 | CREATE OR REPLACE MODEL project_id.dataset_id.model_name 91 | REMOTE WITH CONNECTION `us.ext-vertex-ai` 92 | OPTIONS (endpoint = 'text-bison@002'); 93 | ``` 94 | This will take a few minutes to load and will not return any results. 95 | 96 | ##### 7c: Update the Relevant User Attribute 97 | 98 | Similar to section 5 above. 99 | 100 | | **Required User Attribute Name** | **Default Value Description** | 101 | |-----------------------------------------------------------------|--------------------------------| 102 | | marketplace_bqml_ext_ml_accelerator_generate_text_model_name | Name chosen in step 7b above | 103 | -------------------------------------------------------------------------------- /explores/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/explores/.gitkeep -------------------------------------------------------------------------------- /explores/model_info.explore.lkml: -------------------------------------------------------------------------------- 1 | # Purpose of Explore: Explore Model List View, which will display all created models and relevant metadata. 2 | 3 | include: "/views/model_info.view.lkml" 4 | 5 | explore: model_info { 6 | hidden: yes 7 | persist_for: "0 minutes" 8 | } 9 | -------------------------------------------------------------------------------- /explores/selection_summary.explore.lkml: -------------------------------------------------------------------------------- 1 | include: "/views/selection_summary.view" 2 | 3 | explore: selection_summary { 4 | hidden: yes 5 | } 6 | 7 | explore: arima_selection_summary { 8 | hidden: yes 9 | } 10 | -------------------------------------------------------------------------------- /images/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /images/0 - Applications Menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/0 - Applications Menu.png -------------------------------------------------------------------------------- /images/1 - Landing Page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/1 - Landing Page.png -------------------------------------------------------------------------------- /images/10 - Confusion Matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/10 - Confusion Matrix.png -------------------------------------------------------------------------------- /images/11 - ROC Curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/11 - ROC Curve.png -------------------------------------------------------------------------------- /images/12.1 - Explainable AI - Model Level.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/12.1 - Explainable AI - Model Level.png -------------------------------------------------------------------------------- /images/12.2 - Explainable AI - Class Level.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/12.2 - Explainable AI - Class Level.png -------------------------------------------------------------------------------- /images/13 - Select Data for Predictions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/13 - Select Data for Predictions.png -------------------------------------------------------------------------------- /images/14 - Generate Predictions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/14 - Generate Predictions.png -------------------------------------------------------------------------------- /images/15 - BQ Resources Created by Application.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/15 - BQ Resources Created by Application.png -------------------------------------------------------------------------------- /images/2 - Choose Objective.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/2 - Choose Objective.png -------------------------------------------------------------------------------- /images/3 - Choose Looker Explore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/3 - Choose Looker Explore.png -------------------------------------------------------------------------------- /images/4 - Select Input Data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/4 - Select Input Data.png -------------------------------------------------------------------------------- /images/5 - Select Model Target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/5 - Select Model Target.png -------------------------------------------------------------------------------- /images/6 - Review Training Data Summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/6 - Review Training Data Summary.png -------------------------------------------------------------------------------- /images/7 - Advanced Model Settings Modal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/7 - Advanced Model Settings Modal.png -------------------------------------------------------------------------------- /images/8 - Model Training Screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/8 - Model Training Screen.png -------------------------------------------------------------------------------- /images/9.1 - Evaluation Metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/9.1 - Evaluation Metrics.png -------------------------------------------------------------------------------- /images/9.2 - Evaluation Metrics with Description.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/images/9.2 - Evaluation Metrics with Description.png -------------------------------------------------------------------------------- /images/AIPlatform-36-color.svg: -------------------------------------------------------------------------------- 1 | Icon_36px_MLEngine_Color -------------------------------------------------------------------------------- /manifest.lkml: -------------------------------------------------------------------------------- 1 | project_name: "marketplace_bqml_ext" 2 | 3 | application: ml-accelerator { 4 | label: "Machine Learning Accelerator" 5 | file: "bundle.js" 6 | sri_hash: "spnMLxVFVZ71aj3VUpreZGBwpLD9+lzINATVhn47QEAgD5WGV85ID3R9wFiSakLq" 7 | entitlements: { 8 | core_api_methods: [ 9 | "all_lookml_models", 10 | "all_users", 11 | "create_query", 12 | "run_query", 13 | "lookml_model_explore", 14 | "get_model", 15 | "model_fieldname_suggestions", 16 | "me", 17 | "user_attribute_user_values", 18 | "create_sql_query", 19 | "run_sql_query" 20 | ] 21 | 22 | use_form_submit: yes 23 | use_embeds: yes 24 | use_iframes: yes 25 | new_window: yes 26 | new_window_external_urls: ["https://en.wikipedia.org/wiki", "https://developers.google.com/machine-learning/glossary", "https://cloud.google.com/vertex-ai/docs/model-registry/introduction"] 27 | scoped_user_attributes: [ 28 | "bigquery_connection_name", 29 | "bqml_model_dataset_name", 30 | "generate_text_model_name", 31 | "gcp_project", 32 | ] 33 | } 34 | } 35 | 36 | constant: CONNECTION_NAME { 37 | value: "ml-accelerator" 38 | export: override_required 39 | } 40 | 41 | constant: GCP_PROJECT { 42 | value: "{{_user_attributes['marketplace_bqml_ext_ml_accelerator_gcp_project']}}" 43 | } 44 | 45 | constant: BQML_MODEL_DATASET_NAME { 46 | value: "{{_user_attributes['marketplace_bqml_ext_ml_accelerator_bqml_model_dataset_name']}}" 47 | } 48 | 49 | constant: GENERATE_TEXT_MODEL_NAME { 50 | value: "{{_user_attributes['marketplace_bqml_ext_ml_accelerator_generate_text_model_name']}}" 51 | } 52 | # First create an LLM model in the same dataset as specified in constant "BQML_MODEL_DATASET_NAME", then provide model name here 53 | # https://cloud.google.com/bigquery/docs/generate-text 54 | # Also, modify the service account used for the connection to obtain a new permission: bigquery.connections.use 55 | # This is available to users with role Bigquery Connection User (https://cloud.google.com/iam/docs/understanding-roles#bigquery.connectionUser) 56 | -------------------------------------------------------------------------------- /marketplace.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Machine Learning Accelerator", 3 | "category_label": "Applications", 4 | "branding": { 5 | "image_uri": "https://marketplace-api.looker.com/app-icons/bqml-ext.png", 6 | "tagline": "Gain deeper insights with machine learning.", 7 | "brand_color": "#4285F4" 8 | }, 9 | "constants": { 10 | "CONNECTION_NAME": { 11 | "label": "LookML Constant: BigQuery Connection Name", 12 | "description": "The BigQuery connection the application will be allowed to use. Must be the same connection as chosen below.", 13 | "value_constraint": "connection" 14 | } 15 | }, 16 | "user_attributes": { 17 | "bigquery_connection_name": { 18 | "label": "BigQuery Connection Name", 19 | "description": "The BigQuery connection the application will be allowed to use. Must be the same connection as chosen above.", 20 | "type": "string", 21 | "required": true, 22 | "value_is_hidden": false, 23 | "user_can_view": true, 24 | "user_can_edit": false, 25 | "default_value": "", 26 | "value_constraint": "connection" 27 | }, 28 | "gcp_project": { 29 | "label": "GCP Project ID", 30 | "description": "The GCP project ID for the BigQuery dataset where ML models will be saved.", 31 | "type": "string", 32 | "required": true, 33 | "value_is_hidden": false, 34 | "user_can_view": true, 35 | "user_can_edit": false, 36 | "default_value": "" 37 | }, 38 | "bqml_model_dataset_name": { 39 | "label": "BQML Model Dataset Name", 40 | "description": "The dataset where ML models will be saved. Create a new dataset for BQML models (recommended) or choose the same dataset used for Looker PDTs.", 41 | "type": "string", 42 | "required": true, 43 | "value_is_hidden": false, 44 | "user_can_view": true, 45 | "user_can_edit": false, 46 | "default_value": "" 47 | }, 48 | "generate_text_model_name": { 49 | "label": "GenAI Text Model Name", 50 | "description": "Name of an LLM model to generate text summaries (optional feature). Must be in same dataset as above. See https://github.com/looker-open-source/app-ml-accelerator/blob/main/README.md#7-setup-ai-generated-evaluation-summaries for setup instructions.", 51 | "type": "string", 52 | "required": false, 53 | "value_is_hidden": false, 54 | "user_can_view": true, 55 | "user_can_edit": false, 56 | "default_value": "" 57 | } 58 | }, 59 | "models": [ 60 | { 61 | "name": "ml_accelerator", 62 | "connection_constant": "CONNECTION_NAME" 63 | } 64 | ] 65 | } 66 | -------------------------------------------------------------------------------- /ml_accelerator.model.lkml: -------------------------------------------------------------------------------- 1 | connection: "@{CONNECTION_NAME}" 2 | 3 | include: "/explores/model_info.explore" 4 | include: "/explores/selection_summary.explore" 5 | -------------------------------------------------------------------------------- /views/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/app-ml-accelerator/7bdd79ebb3fccdfcb93e81b3702b128d1df1462b/views/.gitkeep -------------------------------------------------------------------------------- /views/model_info.view.lkml: -------------------------------------------------------------------------------- 1 | view: model_info { 2 | sql_table_name: `@{GCP_PROJECT}`.@{BQML_MODEL_DATASET_NAME}.bqml_model_info ;; 3 | 4 | dimension: model_name { 5 | primary_key: yes 6 | type: string 7 | sql: ${TABLE}.model_name ;; 8 | } 9 | 10 | dimension: state_json { 11 | type: string 12 | sql: ${TABLE}.state_json ;; 13 | } 14 | 15 | dimension: created_by_email { 16 | type: string 17 | sql: ${TABLE}.created_by_email ;; 18 | } 19 | 20 | dimension: created_by_first_name { 21 | type: string 22 | sql: ${TABLE}.created_by_first_name ;; 23 | } 24 | 25 | dimension: created_by_last_name { 26 | type: string 27 | sql: ${TABLE}.created_by_last_name ;; 28 | } 29 | 30 | dimension: shared_with_emails { 31 | type: string 32 | sql: ${TABLE}.shared_with_emails ;; 33 | } 34 | 35 | dimension: full_email_list { 36 | type: string 37 | sql: "\""||${created_by_email}||"\" "||IFNULL(${shared_with_emails},'') ;; 38 | } 39 | 40 | dimension: model_created_at { 41 | type: date_time 42 | sql: TIMESTAMP_MILLIS(${TABLE}.model_created_at) ;; 43 | } 44 | 45 | dimension: model_updated_at { 46 | type: date_time 47 | sql: TIMESTAMP_MILLIS(${TABLE}.model_updated_at) ;; 48 | } 49 | 50 | measure: count { 51 | type: count 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /views/selection_summary.view.lkml: -------------------------------------------------------------------------------- 1 | view: selection_summary { 2 | derived_table: { 3 | sql: SELECT column_stats.column_name 4 | , column_stats._nulls AS count_nulls 5 | , column_stats._non_nulls AS count_not_nulls 6 | , column_stats.pct_not_null AS pct_not_null 7 | , column_stats.count_distinct_values 8 | , column_stats.pct_unique 9 | , column_metadata.data_type 10 | , column_metadata.input_data_column_count 11 | , column_stats.input_data_row_count 12 | , column_stats._min_value 13 | , column_stats._max_value 14 | , column_stats._avg_value 15 | 16 | FROM (SELECT column_name, 17 | COUNT(0) AS input_data_row_count 18 | , COUNT(DISTINCT column_value) AS count_distinct_values 19 | , SAFE_DIVIDE(COUNT(DISTINCT column_value),COUNT(*)) AS pct_unique 20 | , COUNTIF(column_value IS NULL) AS _nulls 21 | , COUNTIF(column_value IS NOT NULL) AS _non_nulls 22 | , COUNTIF(column_value IS NOT NULL) / COUNT(0) AS pct_not_null 23 | , MIN(SAFE_CAST(column_value as numeric)) AS _min_value 24 | , MAX(SAFE_CAST(column_value as numeric)) AS _max_value 25 | , AVG(SAFE_CAST(column_value AS numeric)) AS _avg_value 26 | 27 | 28 | -- unpivot input data into column_name, column_value 29 | -- capture all fields in each row AS JSON string (e.g., "field_a": valueA, "field_b": valueB) 30 | -- unnest array created by split of row_json by ',' 31 | -- "field_a": valueA 32 | -- "field_b": valueB 33 | -- split further on : to get separate columns for name and value 34 | -- format to trim "" from column name and replace any string nulls with true NULLs 35 | FROM (SELECT trim(column_name, '"') AS column_name 36 | , IF(SAFE_CAST(column_value AS STRING)='null',NULL, column_value) AS column_value 37 | 38 | FROM (SELECT REGEXP_REPLACE(TO_JSON_STRING(t), r'^{|}$', '') AS row_json 39 | FROM `@{GCP_PROJECT}.@{BQML_MODEL_DATASET_NAME}.{% parameter selection_summary.input_data_view_name %}` AS t ) table_AS_json 40 | , UNNEST(SPLIT(row_json, ',"')) AS cols 41 | , UNNEST([SPLIT(cols, ':')[SAFE_OFFSET(0)]]) AS column_name 42 | , UNNEST([SPLIT(cols, ':')[SAFE_OFFSET(1)]]) AS column_value 43 | 44 | ) AS col_val 45 | 46 | WHERE column_name <> '' AND column_name NOT LIKE '%-%' 47 | GROUP BY column_name 48 | ) AS column_stats 49 | 50 | INNER JOIN (SELECT table_catalog 51 | , table_schema 52 | , table_name 53 | , column_name 54 | , data_type 55 | , count(0) over (partition by 1) AS input_data_column_count 56 | FROM `@{GCP_PROJECT}.@{BQML_MODEL_DATASET_NAME}`.INFORMATION_SCHEMA.COLUMNS 57 | WHERE table_name = '{% parameter selection_summary.input_data_view_name %}' 58 | ) column_metadata 59 | ON column_stats.column_name = column_metadata.column_name 60 | ;; 61 | } 62 | 63 | 64 | parameter: input_data_view_name { 65 | type: unquoted 66 | } 67 | 68 | parameter: target_field_name { 69 | type: unquoted 70 | } 71 | 72 | dimension: target_column { 73 | type: string 74 | sql: '{% parameter target_field_name %}' ;; 75 | hidden: yes 76 | } 77 | 78 | dimension: column_name { 79 | primary_key: yes 80 | type: string 81 | sql: ${TABLE}.column_name ;; 82 | } 83 | 84 | dimension: count_nulls { 85 | type: number 86 | sql: ${TABLE}.count_nulls ;; 87 | } 88 | 89 | dimension: count_not_nulls { 90 | type: number 91 | sql: ${TABLE}.count_not_nulls ;; 92 | } 93 | 94 | dimension: pct_not_null { 95 | type: number 96 | hidden: yes 97 | sql: ${TABLE}.pct_not_null ;; 98 | value_format_name: percent_4 99 | } 100 | 101 | dimension: pct_null { 102 | type: number 103 | sql: 1 - ${pct_not_null} ;; 104 | value_format_name: percent_2 105 | } 106 | 107 | dimension: count_distinct_values { 108 | label: "Distinct Values" 109 | type: number 110 | sql: ${TABLE}.count_distinct_values ;; 111 | } 112 | 113 | dimension: pct_unique { 114 | type: number 115 | sql: ${TABLE}.pct_unique ;; 116 | value_format_name: percent_2 117 | } 118 | 119 | dimension: data_type { 120 | type: string 121 | sql: ${TABLE}.data_type ;; 122 | } 123 | 124 | dimension: _min_value { 125 | type: string 126 | sql: ${TABLE}._min_value ;; 127 | } 128 | 129 | dimension: _max_value { 130 | type: string 131 | sql: ${TABLE}._max_value ;; 132 | } 133 | 134 | dimension: _avg_value { 135 | type: number 136 | sql: ${TABLE}._avg_value ;; 137 | } 138 | 139 | dimension: input_data_column_count { 140 | type: number 141 | sql: ${TABLE}.input_data_column_count ;; 142 | } 143 | 144 | dimension: input_data_row_count { 145 | type: number 146 | sql: ${TABLE}.input_data_row_count ;; 147 | } 148 | } 149 | 150 | view: arima_selection_summary { 151 | derived_table: { 152 | sql: 153 | {% if arima_selection_summary.arimaTimeframe._parameter_value == 'date_minute' %}{% assign date_fmt = '%Y-%m-%d %H:%M' %} 154 | {% elsif arima_selection_summary.arimaTimeframe._parameter_value == 'date_hour' %}{% assign date_fmt = '%Y-%m-%d %H' %} 155 | {% elsif arima_selection_summary.arimaTimeframe._parameter_value == 'date_month' %}{% assign date_fmt = '%Y-%m' %} 156 | {% elsif arima_selection_summary.arimaTimeframe._parameter_value == 'date_quarter' %}{% assign date_fmt = '%Y-%m' %} 157 | {% elsif arima_selection_summary.arimaTimeframe._parameter_value == 'date_year' %}{% assign date_fmt = '%Y' %} 158 | {% else %}{% assign date_fmt = '%Y-%m-%d' %}{% endif %} 159 | 160 | SELECT column_stats.column_name 161 | , column_metadata.data_type 162 | , column_metadata.input_data_column_count 163 | , column_stats.input_data_row_count 164 | , column_stats._min_value 165 | , column_stats._max_value 166 | , column_stats._min_date_value 167 | , column_stats._max_date_value 168 | , column_stats._avg_value 169 | 170 | FROM (SELECT column_name, 171 | COUNT(0) AS input_data_row_count 172 | , MIN(CASE WHEN '{% parameter arima_selection_summary.arimaTimeColumn %}' = column_name THEN SAFE.PARSE_DATETIME('{{date_fmt}}', column_value) ELSE NULL END) AS _min_date_value 173 | , MAX(CASE WHEN '{% parameter arima_selection_summary.arimaTimeColumn %}' = column_name THEN SAFE.PARSE_DATETIME('{{date_fmt}}', column_value) ELSE NULL END) AS _max_date_value 174 | , MIN(CASE WHEN '{% parameter arima_selection_summary.arimaTimeColumn %}' = column_name THEN NULL ELSE SAFE_CAST(column_value AS numeric) END) AS _min_value 175 | , MAX(CASE WHEN '{% parameter arima_selection_summary.arimaTimeColumn %}' = column_name THEN NULL ELSE SAFE_CAST(column_value AS numeric) END) AS _max_value 176 | 177 | , AVG(SAFE_CAST(column_value AS numeric)) AS _avg_value 178 | 179 | FROM (SELECT TRIM(column_name, '"') AS column_name 180 | , TRIM(IF(SAFE_CAST(column_value AS STRING)='null', NULL, column_value), '"') AS column_value 181 | 182 | FROM (SELECT REGEXP_REPLACE(TO_JSON_STRING(t), r'^{|}$', '') AS row_json 183 | FROM `@{GCP_PROJECT}.@{BQML_MODEL_DATASET_NAME}.{% parameter arima_selection_summary.input_data_view_name %}` AS t ) table_AS_json 184 | , UNNEST(SPLIT(row_json, ',"')) AS cols 185 | , UNNEST([SPLIT(cols, '":')[SAFE_OFFSET(0)]]) AS column_name 186 | , UNNEST([SPLIT(cols, '":')[SAFE_OFFSET(1)]]) AS column_value 187 | WHERE TRIM(column_name, '"') <> '' AND column_name NOT LIKE '%-%' 188 | AND TRIM(column_name, '"') IN ('{% parameter arima_selection_summary.arimaTimeColumn %}', '{% parameter arima_selection_summary.target_field_name %}') 189 | ) AS col_val 190 | 191 | GROUP BY column_name 192 | ) AS column_stats 193 | 194 | INNER JOIN (SELECT table_catalog 195 | , table_schema 196 | , table_name 197 | , column_name 198 | , data_type 199 | , count(0) over (partition by 1) AS input_data_column_count 200 | FROM `@{GCP_PROJECT}.@{BQML_MODEL_DATASET_NAME}`.INFORMATION_SCHEMA.COLUMNS 201 | WHERE table_name = '{% parameter arima_selection_summary.input_data_view_name %}' 202 | ) column_metadata 203 | ON column_stats.column_name = column_metadata.column_name 204 | ;; 205 | } 206 | 207 | parameter: target_field_name { 208 | type: unquoted 209 | } 210 | 211 | parameter: arimaTimeColumn { 212 | type: unquoted 213 | } 214 | 215 | parameter: arimaTimeframe { 216 | type: unquoted 217 | } 218 | 219 | parameter: input_data_view_name { 220 | type: unquoted 221 | } 222 | 223 | dimension: target_column { 224 | type: string 225 | sql: '{% parameter target_field_name %}' ;; 226 | hidden: yes 227 | } 228 | 229 | dimension: column_name { 230 | primary_key: yes 231 | type: string 232 | sql: ${TABLE}.column_name ;; 233 | } 234 | 235 | dimension: data_type { 236 | type: string 237 | sql: ${TABLE}.data_type ;; 238 | } 239 | 240 | dimension: _min_value { 241 | type: string 242 | sql: ${TABLE}._min_value ;; 243 | } 244 | 245 | dimension: _max_value { 246 | type: string 247 | sql: ${TABLE}._max_value ;; 248 | } 249 | 250 | dimension: _min_date_value { 251 | type: string 252 | sql: ${TABLE}._min_date_value ;; 253 | } 254 | 255 | dimension: _max_date_value { 256 | type: string 257 | sql: ${TABLE}._max_date_value ;; 258 | } 259 | 260 | dimension: _avg_value { 261 | type: number 262 | sql: ${TABLE}._avg_value ;; 263 | } 264 | 265 | dimension: input_data_column_count { 266 | type: number 267 | sql: ${TABLE}.input_data_column_count ;; 268 | } 269 | 270 | dimension: input_data_row_count { 271 | type: number 272 | sql: ${TABLE}.input_data_row_count ;; 273 | } 274 | 275 | 276 | } 277 | --------------------------------------------------------------------------------