├── .github ├── CODEOWNERS └── workflows │ ├── configurator_deploy.yaml │ └── pythonapp.yml ├── .gitignore ├── .vim └── coc-settings.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── __main__.py ├── dist ├── lmanage-0.1.0-py3-none-any.whl ├── lmanage-0.1.0.tar.gz ├── lmanage-0.1.1-py3-none-any.whl ├── lmanage-0.1.1.tar.gz ├── lmanage-0.1.2-py3-none-any.whl ├── lmanage-0.1.2.tar.gz ├── lmanage-0.1.3-py3-none-any.whl ├── lmanage-0.1.3.tar.gz ├── lmanage-0.1.4-py3-none-any.whl ├── lmanage-0.1.4.tar.gz ├── lmanage-0.1.5-py3-none-any.whl ├── lmanage-0.1.5.tar.gz ├── lmanage-0.1.6-py3-none-any.whl ├── lmanage-0.1.6.tar.gz ├── lmanage-0.1.7-py3-none-any.whl ├── lmanage-0.1.7.tar.gz ├── lmanage-0.1.8-py3-none-any.whl ├── lmanage-0.1.8.tar.gz ├── lmanage-0.1.9-py3-none-any.whl ├── lmanage-0.1.9.tar.gz ├── lmanage-0.2.0-py3-none-any.whl ├── lmanage-0.2.0.tar.gz ├── lmanage-0.2.1-py3-none-any.whl ├── lmanage-0.2.1.tar.gz ├── lmanage-0.2.2-py3-none-any.whl ├── lmanage-0.2.2.tar.gz ├── lmanage-0.2.3-py3-none-any.whl ├── lmanage-0.2.3.tar.gz ├── lmanage-0.2.4-py3-none-any.whl ├── lmanage-0.2.4.tar.gz ├── lmanage-0.2.5-py3-none-any.whl ├── lmanage-0.2.5.tar.gz ├── lmanage-0.2.6.1-py3-none-any.whl ├── lmanage-0.2.6.1.tar.gz ├── lmanage-0.2.6.2-py3-none-any.whl ├── lmanage-0.2.6.2.tar.gz ├── lmanage-0.2.6.3-py3-none-any.whl ├── lmanage-0.2.6.3.tar.gz ├── lmanage-0.2.6.4-py3-none-any.whl ├── lmanage-0.2.6.4.tar.gz ├── lmanage-0.2.7-py3-none-any.whl ├── lmanage-0.2.7.tar.gz ├── lmanage-0.2.8-py3-none-any.whl ├── lmanage-0.2.8.tar.gz ├── lmanage-0.2.81-py3-none-any.whl ├── lmanage-0.2.81.tar.gz ├── lmanage-0.2.82-py3-none-any.whl ├── lmanage-0.2.82.tar.gz ├── lmanage-0.2.83-py3-none-any.whl ├── lmanage-0.2.83.tar.gz ├── lmanage-0.2.9-py3-none-any.whl ├── lmanage-0.2.9.tar.gz ├── lmanage-0.2.91-py3-none-any.whl ├── lmanage-0.2.91.tar.gz ├── lmanage-0.2.92-py3-none-any.whl ├── lmanage-0.2.92.tar.gz ├── lmanage-0.3.0-py3-none-any.whl └── lmanage-0.3.0.tar.gz ├── file.log ├── images ├── hermes.png ├── lmanage_folder_diagram.png ├── lmanage_mapview_chart.pdf ├── mapview_walkthru.jpeg ├── my_diagram.png └── web_service.png ├── instructions ├── capturator_README.md ├── configurator_README.md ├── looker_settings_capture.md └── mapview_README.md ├── lmanage ├── .vimspector.json ├── __init__.py ├── capturator │ ├── __init__.py │ ├── content_capturation │ │ ├── __init__.py │ │ ├── board_capture.py │ │ ├── dashboard_capture.py │ │ └── look_capture.py │ ├── folder_capturation │ │ ├── __init__.py │ │ ├── create_folder_yaml_structure.py │ │ └── folder_config.py │ ├── looker_api_reader.py │ ├── looker_config_saver.py │ ├── user_attribute_capturation │ │ ├── __init__.py │ │ └── capture_ua_permissions.py │ └── user_group_capturation │ │ ├── __init__.py │ │ ├── group_config.py │ │ ├── role_config.py │ │ └── user_permission.py ├── cli.py ├── configurator │ ├── .vimspector.json │ ├── __init__.py │ ├── content_configuration │ │ ├── __init__.py │ │ ├── clean_instance_content.py │ │ ├── create_boards.py │ │ ├── create_dashboards.py │ │ └── create_looks.py │ ├── create_object.py │ ├── folder_configuration │ │ ├── __init__.py │ │ ├── create_and_provision_instance_folders.py │ │ ├── create_instance_folders.py │ │ └── folder_config.py │ ├── instance_configuration_settings │ │ └── fullinstance.yaml │ ├── looker_config_reader.py │ ├── looker_provisioner.py │ ├── user_attribute_configuration │ │ ├── __init__.py │ │ └── create_and_assign_user_attributes.py │ └── user_group_configuration │ │ ├── __init__.py │ │ ├── create_instance_groups.py │ │ ├── create_instance_roles.py │ │ └── create_role_base.py ├── find_replace.sh ├── logger_config.py ├── looker_auth.py ├── looker_config.py ├── mapview │ ├── .vimspector.json │ ├── __init__.py │ ├── compare_content_to_lookml.py │ ├── hugo.csv │ ├── instancedata.py │ ├── legacy_get_content_with_views.py │ ├── main.py │ ├── mapexplores.py │ ├── mapview_execute.py │ ├── matched_field_analysis.py │ ├── pandas_multiple.xlsx │ ├── parse_sql_tables.py │ └── utils │ │ ├── .vimspector.json │ │ ├── __init__.py │ │ ├── create_df.py │ │ ├── parse_lookml.py │ │ └── parsing_sql.py └── utils │ ├── __init__.py │ ├── errorhandling.py │ ├── helpers.py │ ├── logger_creation.py │ ├── looker_object_constructors.py │ └── parse_yaml.py ├── output ├── demo_output.yaml ├── demo_output_content.yaml ├── hugo.csv ├── instance_mapview.xlsx ├── l233_output.yaml ├── l233_output_content.yaml ├── profserv_output.yaml └── profserv_output_content.yaml ├── poetry.lock ├── pyproject.toml └── tests ├── __init__.py ├── example_yamls ├── Looker22_12_output.yaml ├── Looker22_18_output.yaml ├── clustered_output.yaml ├── demo_output.yaml ├── dev_output.yaml ├── folder_permission_input.yaml ├── folder_yaml.yaml ├── fullinstance.yaml ├── k8_output.yaml ├── profserv_output.yaml ├── test_demo_output.yaml ├── test_output.yaml ├── test_test_output.yaml └── user_permission_output.yaml ├── fake_methods_data.py ├── migrate_schedule.py ├── test_capture_looks.py ├── test_create_folders.py └── test_lookml_files └── the_look ├── dashboards ├── .gitkeep ├── brand_lookup.dashboard.lookml ├── business_pulse.dashboard.lookml ├── customer_lookup.dashboard.lookml ├── shipping_logistics_overview.dashboard.lookml └── web_analytics_overview.dashboard.lookml ├── models ├── .gitkeep ├── bq.model.lkml ├── order_items.model.lkml └── thelook.model.lkml ├── models_aws ├── .gitkeep ├── tests.txt └── thelook_redshift.model.lkml └── views ├── .gitkeep ├── 01_order_items.view.lkml ├── 02_users.view.lkml ├── 03_inventory_items.view.lkml ├── 04_products.view.lkml ├── 05_distribution_centers.view.lkml ├── 11_order_facts.view.lkml ├── 12_user_order_facts.view.lkml ├── 13_repeat_purchase_facts.view.lkml ├── 22_affinity.view.lkml ├── 25_trailing_sales_snapshot.view.lkml ├── 51_events.view.lkml ├── explores.lkml └── test_ndt.view.lkml /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @looker-open-source/cloud-looker-devrel @hugoselbie @rbob86 @belvederej 2 | -------------------------------------------------------------------------------- /.github/workflows/configurator_deploy.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 16 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 17 | 18 | name: Looker Validation Suite 19 | 20 | on: 21 | push: 22 | branches: [ main ] 23 | pull_request: 24 | branches: [ main ] 25 | 26 | jobs: 27 | build: 28 | runs-on: ubuntu-latest 29 | 30 | steps: 31 | - uses: actions/checkout@v2 32 | - name: Set up Python 3.8 33 | uses: actions/setup-python@v2 34 | with: 35 | python-version: "3.8" 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install --upgrade pip 39 | pip install lmanage 40 | # Run Looker Instance Configuratory 41 | - name: Run LManage Configurator 42 | env: 43 | LOOKERSDK_API_VERSION: ${{ secrets.LOOKERSDK_API_VERSION }} 44 | LOOKERSDK_BASE_URL: ${{ secrets.LOOKERSDK_BASE_URL }} 45 | LOOKERSDK_CLIENT_ID: ${{ secrets.LOOKERSDK_CLIENT_ID }} 46 | LOOKERSDK_CLIENT_SECRET: ${{ secrets.LOOKERSDK_CLIENT_SECRET }} 47 | run: | 48 | cd .github 49 | echo ${{ github.head_ref }} 50 | echo ${{ github.event.push.ref }} 51 | lmanage configurator -yp /output/dev_output.yaml 52 | -------------------------------------------------------------------------------- /.github/workflows/pythonapp.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: python_application 16 | 17 | on: 18 | push: 19 | branches: 20 | - main 21 | pull_request: 22 | branches: 23 | - main 24 | 25 | jobs: 26 | build_37: 27 | 28 | runs-on: ubuntu-latest 29 | 30 | steps: 31 | - uses: actions/checkout@v1 32 | - name: Set up Python 3.8 33 | uses: actions/setup-python@v1 34 | with: 35 | python-version: 3.8 36 | - name: Install dependencies 37 | run: | 38 | pip install flake8 pytest pytest-mock 39 | pip install . 40 | - name: Lint with flake8 41 | run: | 42 | # stop the build if there are Python syntax errors or undefined names 43 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 44 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 45 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 46 | - name: Test with pytest 47 | run: | 48 | pytest 49 | 50 | build_38: 51 | runs-on: ubuntu-latest 52 | 53 | steps: 54 | - uses: actions/checkout@v1 55 | - name: Set up Python 3.8 56 | uses: actions/setup-python@v1 57 | with: 58 | python-version: 3.8 59 | - name: Install dependencies 60 | run: | 61 | pip install flake8 pytest pytest-mock 62 | pip install . 63 | - name: Lint with flake8 64 | run: | 65 | # stop the build if there are Python syntax errors or undefined names 66 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 67 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 68 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 69 | - name: Test with pytest 70 | run: | 71 | pytest 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | look_backup/ 2 | looker_backup_content/ 3 | 4 | dist/ 5 | tests/example_yamls/ 6 | output/ 7 | .history/ 8 | #Ignore all setup files 9 | .venv/ 10 | .vscode/ 11 | folder_walk.sh 12 | None 13 | .DS_Store 14 | 15 | #Ignore test files 16 | test.py 17 | test.csv 18 | output.csv 19 | delinquentcontent.csv 20 | spam.log 21 | map_view.log 22 | utils/test.py 23 | utils/old_files/ 24 | 25 | #Ignore legacy files 26 | lmanage/getContentWithViews.py 27 | lmanage/delinquent_content.py 28 | lmanage/delinquent_user.py 29 | lmanage/validate_content.py 30 | lmanage/find_content_with_lookml_params.py 31 | 32 | #Ignore test files for early content 33 | tests/test_getContentWithViews.py 34 | tests/test_delinquent_content.py 35 | tests/test_delinquent_user.py 36 | tests/test_find_content_with_lookml_params.py 37 | tests/test_validate_content.py 38 | tests/test_lookml_files.py 39 | tests/snap_db_response.json 40 | tests/test_lookml_files/aws_thelook_redshift/ 41 | logs/ 42 | 43 | #Ignore pycache 44 | __pycache__/ 45 | *.pyc 46 | 47 | # Ignore config & capturator files 48 | looker.ini 49 | config/ 50 | -------------------------------------------------------------------------------- /.vim/coc-settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.linting.pylintEnabled": false, 3 | "python.linting.pep8Enabled": false, 4 | "python.linting.enabled": true, 5 | "python.linting.flake8Enabled": true 6 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of 9 | experience, education, socio-economic status, nationality, personal appearance, 10 | race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or reject 41 | comments, commits, code, wiki edits, issues, and other contributions that are 42 | not aligned to this Code of Conduct, or to ban temporarily or permanently any 43 | contributor for other behaviors that they deem inappropriate, threatening, 44 | offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | This Code of Conduct also applies outside the project spaces when the Project 56 | Steward has a reasonable belief that an individual's behavior may have a 57 | negative impact on the project or its community. 58 | 59 | ## Conflict Resolution 60 | 61 | We do not believe that all conflict is bad; healthy debate and disagreement 62 | often yield positive results. However, it is never okay to be disrespectful or 63 | to engage in behavior that violates the project’s code of conduct. 64 | 65 | If you see someone violating the code of conduct, you are encouraged to address 66 | the behavior directly with those involved. Many issues can be resolved quickly 67 | and easily, and this gives people more control over the outcome of their 68 | dispute. If you are unable to resolve the matter for any reason, or if the 69 | behavior is threatening or harassing, report it. We are dedicated to providing 70 | an environment where participants feel welcome and safe. 71 | 72 | Reports should be directed to *Hugo Selbie* hugoselbie@google.com the 73 | Project Steward(s) for *lmanage*. It is the Project Steward’s duty to 74 | receive and address reported violations of the code of conduct. They will then 75 | work with a committee consisting of representatives from the Open Source 76 | Programs Office and the Google Open Source Strategy team. If for any reason you 77 | are uncomfortable reaching out to the Project Steward, please email 78 | opensource@google.com. 79 | 80 | We will investigate every complaint, but you may not receive a direct response. 81 | We will use our discretion in determining when and how to follow up on reported 82 | incidents, which may range from not taking action to permanent expulsion from 83 | the project and project-sponsored spaces. We will notify the accused of the 84 | report and provide them an opportunity to discuss it before any action is taken. 85 | The identity of the reporter will be omitted from the details of the report 86 | supplied to the accused. In potentially harmful situations, such as ongoing 87 | harassment or threats to anyone's safety, we may take action without notice. 88 | 89 | ## Attribution 90 | 91 | This Code of Conduct is adapted from the Contributor Covenant, version 1.4, 92 | available at 93 | https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 94 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement (CLA). You (or your employer) retain the copyright to your 10 | contribution; this simply gives us permission to use and redistribute your 11 | contributions as part of the project. Head over to 12 | to see your current agreements on file or 13 | to sign a new one. 14 | 15 | You generally only need to submit a CLA once, so if you've already submitted one 16 | (even if it was for a different project), you probably don't need to do it 17 | again. 18 | 19 | ## Code Reviews 20 | 21 | All submissions, including submissions by project members, require review. We 22 | use GitHub pull requests for this purpose. Consult 23 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 24 | information on using pull requests. 25 | 26 | ## Community Guidelines 27 | 28 | This project follows 29 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LManage 2 | ## What is it. 3 | LManage is a collection of useful tools for [Looker](https://looker.com/) admins to help curate and cleanup content and it's associated source [LookML](https://docs.looker.com/data-modeling/learning-lookml/what-is-lookml). 4 | 5 | ## How do i Install it. 6 | Lmanage can be found on [pypi](#). 7 | ``` 8 | pip install lmanage 9 | ``` 10 | 11 | ## How do I Use it. 12 | ### Commands 13 | LManage will ultimately will have many different commands as development continues 14 | | Status | Command | Rationale | 15 | |---------|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 16 | | Live | Object Migrator Tool | Migrate Looker Objects such as Content, Folders and Permissions, User Groups, Roles and Attributes between a Looker Instance or for Version Control [instructions](https://github.com/looker-open-source/lmanage/tree/main/instructions/looker_settings_capture.md) | 17 | | Planned | scoper | Takes in a model file, elminates the * includes, iterate through the explores and joins and creates a fully scoped model include list for validation performance and best practice code organization | 18 | | Planned | removeuser | Based on last time logged in, prune Looker users to ensure a performant, compliant Looker instance | 19 | | Planned | [mapview](https://github.com/looker-open-source/lmanage/tree/main/instructions/mapview_README.md) | Find the LookML fields and tables that are associated with a piece of Looker content | 20 | 21 | #### help and version 22 | ``` 23 | lmanage --help 24 | Usage: lmanage [OPTIONS] COMMAND [ARGS]... 25 | 26 | Options: 27 | --version Show the version and exit. 28 | --help Show this message and exit. 29 | 30 | Commands: 31 | capturator 32 | configurator 33 | ``` 34 | #### Looker Object Migrator 35 | The object migrator allows you to preserve a point in time representation of your Looker content (Looks and Dashboards), Folder structure, Content access settings, User groups, User roles, User Attributes and preserve these as a Yaml file. This tool then lets you configure a new instance based on that Yaml file. 36 | 37 | [instructions](https://github.com/looker-open-source/lmanage/tree/main/instructions/looker_settings_capture.md) 38 | 39 | 40 | **This is not an officially supported Google Product.** 41 | -------------------------------------------------------------------------------- /dist/lmanage-0.1.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.0-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.0.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.1-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.1-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.1.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.2-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.2-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.2.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.3-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.3-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.3.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.4-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.4-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.4.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.5-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.5-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.5.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.5.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.6-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.6-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.6.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.6.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.7-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.7-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.7.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.7.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.8-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.8-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.8.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.8.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.1.9-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.9-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.1.9.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.9.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.0-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.0.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.1-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.1-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.1.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.2-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.2-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.2.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.3-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.3-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.3.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.4-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.4-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.4.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.5-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.5-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.5.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.5.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.6.1-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.1-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.6.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.1.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.6.2-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.2-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.6.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.2.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.6.3-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.3-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.6.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.3.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.6.4-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.4-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.6.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.4.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.7-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.7-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.7.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.7.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.8-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.8-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.8.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.8.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.81-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.81-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.81.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.81.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.82-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.82-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.82.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.82.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.83-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.83-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.83.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.83.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.9-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.9-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.9.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.9.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.91-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.91-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.91.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.91.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.2.92-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.92-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.2.92.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.92.tar.gz -------------------------------------------------------------------------------- /dist/lmanage-0.3.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.3.0-py3-none-any.whl -------------------------------------------------------------------------------- /dist/lmanage-0.3.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.3.0.tar.gz -------------------------------------------------------------------------------- /file.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/file.log -------------------------------------------------------------------------------- /images/hermes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/hermes.png -------------------------------------------------------------------------------- /images/lmanage_folder_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/lmanage_folder_diagram.png -------------------------------------------------------------------------------- /images/lmanage_mapview_chart.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/lmanage_mapview_chart.pdf -------------------------------------------------------------------------------- /images/mapview_walkthru.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/mapview_walkthru.jpeg -------------------------------------------------------------------------------- /images/my_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/my_diagram.png -------------------------------------------------------------------------------- /images/web_service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/web_service.png -------------------------------------------------------------------------------- /instructions/looker_settings_capture.md: -------------------------------------------------------------------------------- 1 | # Looker Instance Object Migrator Tool 2 | ## The tool is opinionated in functionality by not migrating content or settings that are established on a per user basis. 3 | 4 | The Looker Object Migrator Tool is an API based CLI tool to capture Looker Objects preserving a point in time snapshot as a text based file. This file can then be version controlled and used to configure or amend existing objects in a Looker instance. 5 | 6 | The advantages of having Looker Objects set out in a text file are numerous, for example: 7 | - version controlling Looker objects such as content or Folder structure and permissions (or even setting up automated processes such as gitops) 8 | - massively reducing number of clicks required to set up an instance from scratch, or revert an instance back to desired state 9 | - defining one source of truth across multiple looker instances, for instance shards 10 | - having a clearly defined security permission doc that can interact with other services such as SAML. 11 | 12 | The Object Migrator Tool is comprised of two commands a capture tool and configure tool. The tools are designed to work in concert, i.e. the configure tool is expecting the output of the capture tool, although it's entirely possible to use them individually. 13 | 14 | ## Impacted Settings 15 | The Looker Instance settings that are impacted by these tools are: 16 | - User Attributes (creation and value setting on groups) 17 | - Nested Folder Structure 18 | - Looker Content Access Settings 19 | - User Groups 20 | - User Roles 21 | - User Permission Sets 22 | - User Model Sets 23 | - Looker Content (looks and dashboards) 24 | - Looker Schedules and Alerts 25 | 26 | ## Typical Workflow 27 | 28 | 1. Run the LManage Capturator commmand to generate a Yaml file [example yaml file](#) 29 | 2. Make sure you understand the existing settings, 30 | 3. Check Yaml file into a version control system of choice 31 | 4. Restore Looker system settings using the LManage Configurator command 32 | 33 | #### [Capture Tool aka Capturator specific documentation](https://github.com/looker-open-source/lmanage/blob/main/instructions/capturator_README.md) 34 | #### [Configure Tool aka Configurator specific documentation](https://github.com/looker-open-source/lmanage/blob/main/instructions/configurator_README.md) 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /instructions/mapview_README.md: -------------------------------------------------------------------------------- 1 | # mapview 2 | # This is currently a legacy tool and has been deliberately broken while undergoing a rewrite. 3 | The mapview command will find the etymology of the content on your dashboard, exporting a CSV that looks like [this](https://docs.google.com/spreadsheets/d/1TzeJW46ml0uzO9RdLOOLxwtvUWjhmZxoa-xq4pbznV0/edit?resourcekey=0-xbWC87hXYFNgy1As06NncA#gid=900312158). 4 | 5 | ##### example usage 6 | `lmanage mapview --path ./output/my_output.csv --ini-file ~/py/projects/ini/k8.ini --project /test_lookml_files/the_look -table "order_items"` 7 | ##### flags 8 | - **lookml_file_path** (`--lookml_file_path`, `-lfp`) This is the path where you would like the outputfile for your returned dataset to be. 9 | - **ini-file** (`--ini-file`, `-i`) This is the file path to the ini file that you've created to use the Looker SDK 10 | - **output_path** (`--output_path`, `-op`) File path to export an xlsx tabbed file of your results input your file path to save a tabbed xls of results 11 | ``` 12 | #example Ini file 13 | [Looker_Instance] 14 | base_url=https://looker-dev.company.com:19999 (or 443 if hosted in GCP) 15 | client_id=abc 16 | client_secret=xyz 17 | verify_ssl=True 18 | ``` 19 | 20 | ![](./images/mapview_walkthru.jpeg) 21 | 22 | 23 | ## Excel Tabbed Returns 24 | ### DashboardToLookMLMapping 25 | - **used_views**: the view_names extracted from the query 26 | - **matched_fields**: the fields matched with inputted lookml 27 | - **dashboard**: the id of the looker dashboard 28 | - **element_id**: the id of the visualization element on the looker dashboard 29 | 30 | ### MostUsedDimensions 31 | - **matched_fields**: the fields matched with inputted lookml 32 | - **dashboard_count**: how many dashboards are used by this field 33 | 34 | ### MostUsedViews 35 | - **used_views**: the views extracted from the query 36 | - **dashboard_count**: how many dashboards are used by this field 37 | 38 | ##### n.b. 39 | **Multi Project Usage** 40 | Dashboards can hold tiles from multiple projects, in this case if you create one local folder of lookml see example below, then pass the value of that one meta folder the the `--project` flag. Doing this will enable the underlying LookML parsing engine driven by [pyLookML](https://github.com/llooker/pylookml) to iterate over all the relevant files and find the appropriate cross project matches. 41 | 42 | ``` 43 | ├── test_lookml_files 44 | │ ├── dashboards 45 | │ │ ├── brand_lookup.dashboard.lookml 46 | │ │ ├── business_pulse.dashboard.lookml 47 | │ │ ├── customer_lookup.dashboard.lookml 48 | │ ├── models_proj1 49 | │ │ └── thelook.model.lkml 50 | │ ├── models_proj2 51 | │ │ └── thelook_redshift.model.lkml 52 | │ ├── view_proj1 53 | │ │ ├── 01_order_items.view.lkml 54 | │ │ ├── 02_users.view.lkml 55 | │ │ ├── 03_inventory_items.view.lkml 56 | │ │ ├── 04_products.view.lkml 57 | │ │ ├── 05_distribution_centers.view.lkml 58 | │ └── view_proj2 59 | │ ├── 01_order_items.view.lkml 60 | │ ├── 02_users.view.lkml 61 | │ ├── 03_inventory_items.view.lkml 62 | │ ├── 04_products.view.lkml 63 | │ ├── 05_distribution_centers.view.lkml 64 | │ ├── 11_order_facts.view.lkml 65 | │ ├── 12_user_order_facts.view.lkml 66 | │ ├── 13_repeat_purchase_facts.view.lkml 67 | │ ├── 22_affinity.view.lkml 68 | │ ├── 25_trailing_sales_snapshot.view.lkml 69 | │ ├── 51_events.view.lkml 70 | │ ├── explores.lkml 71 | │ └── test_ndt.view.lkml. 72 | ``` 73 | 74 | 75 | 76 | **This is not an officially supported Google Product.** 77 | -------------------------------------------------------------------------------- /lmanage/.vimspector.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": { 3 | "capturator": { 4 | "adapter": "debugpy", 5 | "configuration": { 6 | "python": "/usr/local/google/home/hugoselbie/.cache/pypoetry/virtualenvs/lmanage-UKKoJ9La-py3.8/bin/python", 7 | "request": "launch", 8 | "protocol": "auto", 9 | "stopOnEntry": true, 10 | "console": "integratedTerminal", 11 | "program": "${workspaceRoot}/capture_instance_permission_structure.py", 12 | "cwd": "${workspaceRoot}" 13 | } 14 | }, 15 | "configuration": { 16 | "adapter": "debugpy", 17 | "configuration": { 18 | "python": "/usr/local/google/home/hugoselbie/.cache/pypoetry/virtualenvs/lmanage-UKKoJ9La-py3.8/bin/python", 19 | "request": "launch", 20 | "protocol": "auto", 21 | "stopOnEntry": true, 22 | "console": "integratedTerminal", 23 | "program": "${workspaceRoot}/provision_instance_permission_structure.py", 24 | "cwd": "${workspaceRoot}" 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /lmanage/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __version__ = '0.1.4' 16 | -------------------------------------------------------------------------------- /lmanage/capturator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __version__ = '0.1.4' 16 | -------------------------------------------------------------------------------- /lmanage/capturator/content_capturation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/capturator/content_capturation/__init__.py -------------------------------------------------------------------------------- /lmanage/capturator/content_capturation/board_capture.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | import logging 3 | from lmanage.utils.errorhandling import return_sleep_message 4 | from lmanage.utils.looker_object_constructors import BoardObject 5 | from lmanage.utils import logger_creation as log_color 6 | from yaspin import yaspin 7 | #logger = log_color.init_logger(__name__, logger_level) 8 | 9 | 10 | class CaptureBoards(): 11 | def __init__(self, sdk): 12 | self.sdk = sdk 13 | 14 | def handle_board_structure(self, board_response: dict) -> dict: 15 | board_section_list = board_response.board_sections 16 | bs_list = [] 17 | 18 | for board_section in board_section_list: 19 | board_item_list = board_section.board_items 20 | new_bi_list = [board_item.__dict__ for board_item in board_item_list] 21 | board_section['board_items'] = new_bi_list 22 | bs_dict = board_section.__dict__ 23 | bs_list.append(bs_dict) 24 | 25 | board_response['board_sections'] = bs_list 26 | 27 | return board_response 28 | 29 | 30 | def get_all_boards(self) -> dict: 31 | response = [] 32 | with yaspin().white.bold.shark.on_blue as sp: 33 | sp.text="getting all system board metadata (can take a while)" 34 | all_system_boards = self.sdk.all_boards() 35 | 36 | for board in all_system_boards: 37 | bresponse = self.handle_board_structure(board_response=board) 38 | b = BoardObject( 39 | content_metadata_id=board.content_metadata_id, 40 | section_order=board.section_order, 41 | title=board.title, 42 | primary_homepage=board.primary_homepage, 43 | description=board.description, 44 | board_sections=bresponse.board_sections 45 | ) 46 | response.append(b) 47 | return response 48 | 49 | 50 | def execute(self): 51 | all_boards = self.get_all_boards() 52 | return all_boards -------------------------------------------------------------------------------- /lmanage/capturator/content_capturation/dashboard_capture.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | import logging 3 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color 4 | from tenacity import retry, wait_fixed, wait_random, stop_after_attempt 5 | from yaspin import yaspin 6 | from looker_sdk import error 7 | from collections import Counter 8 | 9 | class CaptureDashboards(): 10 | def __init__(self, sdk, content_folders: dict, logger): 11 | self.sdk = sdk 12 | self.content_folders = content_folders 13 | self.logger = logger 14 | self.all_alerts = self.sdk.search_alerts(all_owners=True) 15 | all_dashboard_elements_with_alerts = [alert.dashboard_element_id for alert in self.all_alerts] 16 | self.dashboard_element_alert_counts = dict(Counter(all_dashboard_elements_with_alerts)) 17 | 18 | def get_all_dashboards(self) -> dict: 19 | scrub_dashboards = {} 20 | with yaspin().white.bold.shark.on_blue as sp: 21 | sp.text = "getting all system dashboard metadata (can take a while)" 22 | all_dashboards = self.sdk.all_dashboards(fields="id,folder,slug") 23 | 24 | for dash in all_dashboards: 25 | self.logger.debug(self.content_folders) 26 | if dash.folder.id in self.content_folders: 27 | scrub_dashboards[dash.id] = {} 28 | scrub_dashboards[dash.id]['folder_id'] = dash.folder.id 29 | scrub_dashboards[dash.id]['slug'] = dash.slug 30 | # logger.info(scrub_dashboards) 31 | return scrub_dashboards 32 | 33 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 34 | def get_looker_dashboard(self, looker_dashboard_id, fields='dashboard_elements'): 35 | dashboard_elements = self.sdk.dashboard( 36 | looker_dashboard_id, fields) 37 | return dashboard_elements 38 | 39 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 40 | def get_looker_schedule_plan_for_dashboard(self, looker_dashboard_id): 41 | dash_schedule_plans = self.sdk.scheduled_plans_for_dashboard( 42 | looker_dashboard_id, all_users=True) 43 | return dash_schedule_plans 44 | 45 | def get_dashboard_lookml(self, all_dashboards: dict) -> list: 46 | # logging.info("Beginning Dashboard Capture:") 47 | response = [] 48 | for dashboard_id in tqdm(all_dashboards, desc="Dashboard Capture", unit="dashboards", colour="#2c8558"): 49 | lookml = None 50 | trys = 0 51 | if "::" in dashboard_id: 52 | continue 53 | else: 54 | dashboard_elements = self.get_looker_dashboard( 55 | looker_dashboard_id=dashboard_id)['dashboard_elements'] 56 | dashboard_element_ids = [e['id'] for e in dashboard_elements] 57 | dashboard_element_alert_counts = [self.dashboard_element_alert_counts.get(id, 0) for id in dashboard_element_ids] 58 | scheduled_plans = self.get_looker_schedule_plan_for_dashboard( 59 | looker_dashboard_id=dashboard_id) 60 | alerts = [ 61 | loc.AlertObject(alert) for alert in self.all_alerts if alert['dashboard_element_id'] in dashboard_element_ids] 62 | 63 | while lookml is None and trys < 5: 64 | trys += 1 65 | try: 66 | lookml = self.sdk.dashboard_lookml( 67 | dashboard_id=dashboard_id) 68 | except error.SDKError as e: 69 | self.logger.error(f"LookML download error for dashboard {dashboard_id}") 70 | eh.return_sleep_message(call_number=trys, quiet=True) 71 | 72 | if lookml: 73 | self.logger.debug(lookml.lookml) 74 | captured_dashboard = loc.DashboardObject( 75 | legacy_folder_id=all_dashboards.get(dashboard_id), 76 | lookml=lookml.lookml, 77 | dashboard_id=dashboard_id, 78 | dashboard_slug=all_dashboards.get( 79 | dashboard_id).get('slug'), 80 | dashboard_element_alert_counts=dashboard_element_alert_counts, 81 | scheduled_plans=scheduled_plans, 82 | alerts=alerts 83 | ) 84 | response.append(captured_dashboard) 85 | return response 86 | 87 | def execute(self): 88 | all_dashboards = self.get_all_dashboards() 89 | captured_dash = self.get_dashboard_lookml(all_dashboards) 90 | return captured_dash 91 | -------------------------------------------------------------------------------- /lmanage/capturator/content_capturation/look_capture.py: -------------------------------------------------------------------------------- 1 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color 2 | from tqdm import tqdm 3 | from yaspin import yaspin 4 | from tenacity import retry, wait_fixed, wait_random, stop_after_attempt 5 | 6 | # logger = log_color.init_logger(__name__, logger_level) 7 | 8 | 9 | class LookCapture: 10 | def __init__(self, sdk, content_folders, logger): 11 | self.sdk = sdk 12 | self.content_folders = content_folders 13 | self.logger = logger 14 | 15 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 16 | def get_all_looks_metadata(self) -> list: 17 | response = self.sdk.all_looks(fields='id,folder') 18 | return response 19 | 20 | def all_looks(self): 21 | system_folders = ['Users', 'Embed Users', 'Embed Groups'] 22 | all_look_meta = None 23 | with yaspin().white.bold.shark.on_blue as sp: 24 | sp.text = "getting all system look metadata (can take a while)" 25 | all_look_meta = self.get_all_looks_metadata() 26 | 27 | scrub_looks = {} 28 | for look in all_look_meta: 29 | if look.folder.id in self.content_folders or look.folder.id == '1': 30 | scrub_looks[look.id] = look.folder.id 31 | else: 32 | continue 33 | 34 | return scrub_looks 35 | 36 | def get_look_metadata(self, look_id: str) -> dict: 37 | look_meta = None 38 | trys = 0 39 | while look_meta is None: 40 | try: 41 | look_meta = self.sdk.look(look_id=look_id) 42 | except: 43 | eh.return_sleep_message 44 | return look_meta 45 | 46 | def clean_query_obj(self, query_metadata: dict) -> dict: 47 | metadata_keep_keys = [ 48 | "model", 49 | "view", 50 | "fields", 51 | "pivots", 52 | "fill_fields", 53 | "filters", 54 | "filter_expression", 55 | "sorts", 56 | "limit", 57 | "column_limit", 58 | "total", 59 | "row_total", 60 | "subtotals", 61 | "vis_config", 62 | "filter_config", 63 | "visible_ui_sections", 64 | "dynamic_fields", 65 | "query_timezone"] 66 | restricted_look_metadata = dict( 67 | (k, query_metadata[k]) for k in metadata_keep_keys) 68 | return restricted_look_metadata 69 | 70 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 71 | def get_scheduled_plans_for_look(self, looker_look_id): 72 | return self.sdk.scheduled_plans_for_look( 73 | look_id=looker_look_id, all_users=True) 74 | 75 | def execute(self): 76 | ''' 77 | 1. get all the looks and extract the id's 78 | 2. iterate through the looks and extract metadata, only keep necessary fields to make a query for look transference 79 | 3. create custom look object and add to list 80 | ''' 81 | all_look_data = self.all_looks() 82 | looks = [] 83 | content = 1 84 | for look in tqdm(all_look_data, desc="Look Capture", unit=" looks", colour="#2c8558"): 85 | lmetadata = self.get_look_metadata(look_id=look) 86 | scheduled_plans = self.get_scheduled_plans_for_look( 87 | looker_look_id=lmetadata.id) 88 | query_object = lmetadata.query.__dict__ 89 | nq_obj = self.clean_query_obj(query_metadata=query_object) 90 | self.logger.debug(nq_obj) 91 | 92 | legacy_folder = lmetadata.folder_id 93 | look_id = lmetadata.id 94 | title = lmetadata.title 95 | look_obj = loc.LookObject( 96 | query_obj=nq_obj, 97 | description=lmetadata.description, 98 | legacy_folder_id=legacy_folder, 99 | look_id=look_id, 100 | title=title, 101 | scheduled_plans=scheduled_plans) 102 | looks.append(look_obj) 103 | content += 1 104 | return looks 105 | -------------------------------------------------------------------------------- /lmanage/capturator/folder_capturation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/capturator/folder_capturation/__init__.py -------------------------------------------------------------------------------- /lmanage/capturator/folder_capturation/create_folder_yaml_structure.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color 4 | 5 | #logger = log_color.init_logger(__name__, logger_level) 6 | 7 | class CreateFolderOutput(): 8 | def __init__(self, folder_list): 9 | self.folder_list = folder_list 10 | 11 | def create_folder_structure(self, folder_obj: object, data_storage: list): 12 | temp = {} 13 | temp['name'] = folder_obj.name 14 | temp['permissions'] = folder_obj.content_access_list 15 | temp['subfolder'] = folder_obj.children 16 | data_storage.append(temp) 17 | 18 | if isinstance(folder_obj.children, list): 19 | for subfolder in folder_obj.children: 20 | self.create_folder_structure(subfolder, data_storage) 21 | 22 | return data_storage 23 | 24 | def make_dict(self): 25 | x = [] 26 | f = self.folder_list 27 | 28 | for folders in f: 29 | data_storage = [] 30 | x = self.create_folder_structure( 31 | folder_obj=folders, data_storage=data_storage) 32 | return x 33 | 34 | def execute(self): 35 | ''' 36 | goal: extract all the folders into the appropriate markdown folder structure 37 | ''' 38 | x = self.make_dict() 39 | return x 40 | -------------------------------------------------------------------------------- /lmanage/capturator/looker_api_reader.py: -------------------------------------------------------------------------------- 1 | from lmanage.capturator.folder_capturation import folder_config as fc 2 | from lmanage.capturator.user_group_capturation import role_config as rc 3 | from lmanage.capturator.user_attribute_capturation import capture_ua_permissions as cup 4 | from lmanage.capturator.content_capturation import dashboard_capture as dc, look_capture as lc, board_capture as bc 5 | from lmanage.looker_config import LookerConfig 6 | from lmanage.logger_config import setup_logger 7 | from lmanage.looker_auth import LookerAuth 8 | 9 | logger = setup_logger() 10 | 11 | 12 | class LookerApiReader(): 13 | shared_folder_id = '1' 14 | 15 | def __init__(self, ini_file): 16 | self.sdk = LookerAuth().authenticate(ini_file) 17 | 18 | def get_config(self) -> LookerConfig: 19 | settings = self.__read_settings() 20 | content = self.__read_content() 21 | return LookerConfig(settings, content) 22 | 23 | def __read_settings(self): 24 | # Folder configuration 25 | folder_returns = fc.CaptureFolderConfig( 26 | sdk=self.sdk, logger=logger).execute() 27 | folder_structure_list = folder_returns[1] 28 | self.content_folders = [self.shared_folder_id] + folder_returns[0] 29 | 30 | # Roles, Permission Sets & Model Sets 31 | role_info = rc.ExtractRoleInfo(sdk=self.sdk, logger=logger) 32 | permission_sets = role_info.extract_permission_sets() 33 | model_sets = role_info.extract_model_sets() 34 | roles = role_info.extract_role_info() 35 | 36 | # User Attributes 37 | user_attributes = cup.ExtractUserAttributes( 38 | sdk=self.sdk, logger=logger).create_user_attributes() 39 | 40 | return { 41 | 'folder_permissions': folder_structure_list, 42 | 'permission_sets': permission_sets, 43 | 'model_sets': model_sets, 44 | 'roles': roles, 45 | 'user_attributes': user_attributes 46 | } 47 | 48 | def __read_content(self): 49 | # Boards 50 | # boards = bc.CaptureBoards(sdk=sdk).execute() 51 | 52 | # Looks 53 | looks = lc.LookCapture( 54 | sdk=self.sdk, content_folders=self.content_folders, logger=logger).execute() 55 | 56 | # Dashboards 57 | dashboards = dc.CaptureDashboards( 58 | sdk=self.sdk, content_folders=self.content_folders, logger=logger).execute() 59 | 60 | return { 61 | 'looks': looks, 62 | 'dashboards': dashboards 63 | } 64 | -------------------------------------------------------------------------------- /lmanage/capturator/looker_config_saver.py: -------------------------------------------------------------------------------- 1 | import looker_sdk 2 | import ruamel.yaml 3 | import os 4 | from lmanage.looker_config import LookerConfig 5 | from lmanage.logger_config import setup_logger 6 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh 7 | 8 | logger = setup_logger() 9 | 10 | 11 | class LookerConfigSaver(): 12 | def __init__(self, config_dir): 13 | self.__init_yaml() 14 | self.config_dir = config_dir 15 | if not os.path.exists(config_dir): 16 | os.makedirs(config_dir) 17 | 18 | def save(self, config: LookerConfig): 19 | self.__save_settings(config.settings) 20 | self.__save_content(config.content) 21 | 22 | def __init_yaml(self): 23 | self.yaml = ruamel.yaml.YAML() 24 | self.yaml.register_class(loc.LookerFolder) 25 | self.yaml.register_class(loc.LookerModelSet) 26 | self.yaml.register_class(loc.LookerPermissionSet) 27 | self.yaml.register_class(loc.LookerRoles) 28 | self.yaml.register_class(loc.LookerUserAttribute) 29 | self.yaml.register_class(loc.LookObject) 30 | self.yaml.register_class(loc.DashboardObject) 31 | self.yaml.register_class(loc.AlertObject) 32 | self.yaml.register_class(loc.AlertAppliedDashboardFilterObject) 33 | self.yaml.register_class(loc.AlertDestinationObject) 34 | self.yaml.register_class(loc.AlertFieldObject) 35 | self.yaml.register_class(loc.AlertFieldFilterObject) 36 | self.yaml.register_class(loc.BoardObject) 37 | self.yaml.register_class(looker_sdk.sdk.api40.models.ScheduledPlan) 38 | self.yaml.register_class( 39 | looker_sdk.sdk.api40.models.ScheduledPlanDestination) 40 | self.yaml.register_class(looker_sdk.sdk.api40.models.UserPublic) 41 | 42 | def __save_settings(self, settings): 43 | # Folder Permissions 44 | with open(self.__get_settings_path(), 'w') as file: 45 | fd_yml_txt = '''# FOLDER_PERMISSIONS\n# Opening Session Welcome to the Capturator, this is the Folder place\n# -----------------------------------------------------\n\n''' 46 | file.write(fd_yml_txt) 47 | self.yaml.dump(settings['folder_permissions'], file) 48 | 49 | # Roles, Permission Sets & Model Sets 50 | with open(self.__get_settings_path(), 'a') as file: 51 | r_yml_txt = '''# Looker Role\n# Opening Session Welcome to the Capturator, this is the Role place\n# -----------------------------------------------------\n\n''' 52 | file.write(r_yml_txt) 53 | file.write('\n\n# PERMISSION SETS\n') 54 | self.yaml.dump(settings['permission_sets'], file) 55 | file.write('\n\n# MODEL SETS\n') 56 | self.yaml.dump(settings['model_sets'], file) 57 | file.write('\n\n# LOOKER ROLES\n') 58 | self.yaml.dump(settings['roles'], file) 59 | 60 | # User Attributes 61 | with open(self.__get_settings_path(), 'a') as file: 62 | file.write('\n\n# USER_ATTRIBUTES\n') 63 | self.yaml.dump(settings['user_attributes'], file) 64 | 65 | def __save_content(self, content): 66 | # Boards 67 | # with open(self.__get_content_path(), 'w+') as file: 68 | # file.write('\n\n# BoardData\n') 69 | # self.yaml.dump(self.content.boards, file) 70 | 71 | # Looks 72 | looks = content['looks'] 73 | with open(self.__get_content_path(), 'wb') as file: 74 | file.write(bytes('\n\n# LookData\n', 'utf-8')) 75 | if eh.test_object_data(looks): 76 | self.yaml.dump(looks, file) 77 | else: 78 | file.write(bytes('#No Captured Looks', 'utf-8')) 79 | 80 | # Dashboards 81 | dashboards = content['dashboards'] 82 | with open(self.__get_content_path(), 'ab') as file: 83 | file.write(bytes('\n\n# Dashboard Content\n', 'utf-8')) 84 | if eh.test_object_data(dashboards): 85 | self.yaml.dump(dashboards, file) 86 | else: 87 | file.write(bytes('#No Captured Dashboards', 'utf-8')) 88 | 89 | def __get_settings_path(self): 90 | return f'{self.config_dir}/settings.yaml' 91 | 92 | def __get_content_path(self): 93 | return f'{self.config_dir}/content.yaml' 94 | -------------------------------------------------------------------------------- /lmanage/capturator/user_attribute_capturation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/capturator/user_attribute_capturation/__init__.py -------------------------------------------------------------------------------- /lmanage/capturator/user_attribute_capturation/capture_ua_permissions.py: -------------------------------------------------------------------------------- 1 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color 2 | from tqdm import tqdm 3 | from tenacity import retry, wait_fixed, wait_random, stop_after_attempt 4 | 5 | class ExtractUserAttributes(): 6 | def __init__(self, sdk, logger): 7 | self.sdk = sdk 8 | self.logger = logger 9 | self.user_attribute_metadata = self.existing_user_attributes() 10 | self.all_group_metadata = self.all_group_metadatas() 11 | 12 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 13 | def get_all_user_attributes(self) -> dict: 14 | response = self.sdk.all_user_attributes() 15 | return response 16 | 17 | def existing_user_attributes(self) -> dict: 18 | ex_ua = self.get_all_user_attributes() 19 | resp = [ua for ua in ex_ua if not ua.is_system] 20 | return resp 21 | 22 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 23 | def all_group_metadatas(self): 24 | response=self.sdk.all_groups() 25 | return response 26 | 27 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 28 | def all_user_attribute_group_value_metadata(self, uaid: int): 29 | response = self.sdk.all_user_attribute_group_values(uaid) 30 | return response 31 | 32 | 33 | def create_user_attributes(self): 34 | group_metadata = { 35 | group.id: group.name for group in self.all_group_metadata} 36 | response = [] 37 | for ua in tqdm(self.user_attribute_metadata, desc = "User Attribute Capture", unit=" user att", colour="#2c8558"): 38 | group_assign = self.all_user_attribute_group_value_metadata(uaid=ua.id) 39 | team_values = [] 40 | 41 | for group in group_assign: 42 | group_name = group_metadata.get(group.group_id) 43 | teams = {} 44 | teams[group_name] = group.value 45 | team_values.append(teams) 46 | looker_ua = loc.LookerUserAttribute( 47 | name=ua.name, 48 | uatype=ua.type, 49 | hidden_value=ua.value_is_hidden, 50 | user_view=ua.user_can_view, 51 | user_edit=ua.user_can_edit, 52 | default_value=ua.default_value, 53 | teams_val=team_values 54 | ) 55 | response.append(looker_ua) 56 | return response 57 | -------------------------------------------------------------------------------- /lmanage/capturator/user_group_capturation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/capturator/user_group_capturation/__init__.py -------------------------------------------------------------------------------- /lmanage/capturator/user_group_capturation/group_config.py: -------------------------------------------------------------------------------- 1 | from looker_sdk import error 2 | from looker_sdk.sdk.api40 import models 3 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color 4 | 5 | 6 | class GetInstanceGroups(): 7 | def __init__(self, sdk, logger) -> None: 8 | self.sdk = sdk 9 | self.logger 10 | 11 | def extract_group_names(self): 12 | capture_groups = self.sdk.all_groups() 13 | group_names = [group.name for group in capture_groups] 14 | 15 | return group_names 16 | 17 | def extract_folder_names(self): 18 | folders = self.sdk.all_folders() 19 | test = {} 20 | for folder in folders: 21 | print('high', folder.name) 22 | if folder.parent_id is not None: 23 | print('low', folder.name) 24 | 25 | test[folder.name] = folder.parent_id 26 | else: 27 | print('test', folder.name) 28 | 29 | # for x, y in test.items(): 30 | 31 | return test 32 | 33 | def create_group_if_not_exists(self, 34 | sdk, 35 | group_name: str) -> dict: 36 | """ Create a Looker Group and add Group attributes 37 | 38 | :group_name: Name of a Looker group to create. 39 | :rtype: Looker Group object. 40 | """ 41 | # get group if exists 42 | try: 43 | self.logger.info(f'Creating group "{group_name}"') 44 | group = sdk.create_group( 45 | body=models.WriteGroup( 46 | can_add_to_content_metadata=True, 47 | name=group_name 48 | ) 49 | ) 50 | return group 51 | except error.SDKError as grouperr: 52 | err_msg = eh.return_error_message(grouperr) 53 | self.logger.info( 54 | 'you have an error in your group; error = %s', err_msg) 55 | self.logger.debug(grouperr) 56 | group = sdk.search_groups(name=group_name) 57 | return group[0] 58 | 59 | def get_instance_group_metadata(self, 60 | sdk, 61 | unique_group_list: list) -> list: 62 | group_metadata = [] 63 | 64 | for group_name in unique_group_list: 65 | group = self.create_group_if_not_exists(sdk, group_name) 66 | temp = {} 67 | temp['group_id'] = group.id 68 | temp['group_name'] = group.name 69 | group_metadata.append(temp) 70 | 71 | return group_metadata 72 | 73 | def sync_groups(self, 74 | sdk, 75 | group_name_list: list) -> str: 76 | 77 | all_groups = sdk.all_groups() 78 | group_dict = {group.name: group.id for group in all_groups} 79 | # Deleting Standard Groups 80 | del group_dict['All Users'] 81 | 82 | for group_name in group_dict.keys(): 83 | if group_name not in group_name_list: 84 | sdk.delete_group(group_id=group_dict[group_name]) 85 | self.logger.info( 86 | f'deleting group {group_name} to sync with yaml config') 87 | 88 | return 'your groups are in sync with your yaml file' 89 | 90 | def execute(self): 91 | team_list = [] 92 | # extracting user attribute teams 93 | self.extract_teams(container=self.user_attribute_metadata, 94 | data_storage=team_list) 95 | # extracting role based teams 96 | self.extract_teams( 97 | container=self.role_metadata, data_storage=team_list) 98 | # extract nested folder access teams 99 | self.extract_folder_teams( 100 | container=self.folder_metadata, data_storage=team_list) 101 | 102 | team_list = list(set(team_list)) 103 | 104 | # create all the groups 105 | self.get_instance_group_metadata( 106 | sdk=self.sdk, unique_group_list=team_list) 107 | -------------------------------------------------------------------------------- /lmanage/capturator/user_group_capturation/role_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from tqdm import tqdm 3 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color 4 | 5 | # logger = log_color.init_logger(__name__, logger_level) 6 | 7 | 8 | class ExtractRoleInfo(): 9 | def __init__(self, sdk, logger): 10 | self.sdk = sdk 11 | self.logger = logger 12 | self.role_base = self.get_all_roles() 13 | 14 | def get_all_roles(self): 15 | sdk = self.sdk 16 | response = sdk.all_roles() 17 | for role in enumerate(response): 18 | 19 | if role[1].name == 'Admin': 20 | response.pop(role[0]) 21 | 22 | return response 23 | 24 | def create_list_of_permission_sets(self): 25 | response = [] 26 | for role in self.role_base: 27 | if role.permission_set is None: 28 | raise Exception( 29 | f'Role name {role.name} has no permission_set, please add one to be captured or delete the role.') 30 | temp = {} 31 | temp['name'] = role.permission_set.name 32 | temp['permissions'] = role.permission_set.permissions 33 | response.append(temp) 34 | return response 35 | 36 | def extract_permission_sets(self): 37 | response = [] 38 | p_list = eh.dedup_list_of_dicts( 39 | self.create_list_of_permission_sets()) 40 | for role in p_list: 41 | self.logger.debug(role) 42 | perm_set = loc.LookerPermissionSet( 43 | name=role.get('name'), 44 | permissions=role.get('permissions')) 45 | response.append(perm_set) 46 | 47 | return response 48 | 49 | def create_list_of_roles(self): 50 | response = [] 51 | for role in self.role_base: 52 | temp = {} 53 | temp['name'] = role.model_set.name 54 | temp['models'] = role.model_set.models 55 | response.append(temp) 56 | return response 57 | 58 | def extract_model_sets(self): 59 | response = [] 60 | role_list = eh.dedup_list_of_dicts( 61 | self.create_list_of_roles()) 62 | for role in role_list: 63 | model_set = loc.LookerModelSet( 64 | models=role.get('models'), name=role.get('name')) 65 | response.append(model_set) 66 | return response 67 | 68 | def extract_role_info(self): 69 | response = [] 70 | for role in tqdm(self.role_base, desc='Role Capture', unit=' roles', colour='#2c8558'): 71 | groups = None 72 | trys = 0 73 | while groups is None: 74 | trys += 1 75 | try: 76 | groups = self.sdk.role_groups(role_id=role.id) 77 | except: 78 | eh.return_sleep_message(call_number=trys) 79 | role_groups = [group.name for group in groups] 80 | lookerrole = loc.LookerRoles(permission_set=role.permission_set.name, 81 | model_set=role.model_set.name, teams=role_groups, name=role.name) 82 | response.append(lookerrole) 83 | 84 | return response 85 | -------------------------------------------------------------------------------- /lmanage/capturator/user_group_capturation/user_permission.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import looker_sdk 3 | from looker_sdk.sdk.api40 import models 4 | from role_config import CreateRoleBase 5 | from lmanage.utils import logger_creation as log_color 6 | 7 | logging.setLoggerClass(log_color.ColoredLogger) 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | class CreateInstanceRoles(CreateRoleBase): 12 | def __init__(self, roles, sdk): 13 | self.role_metadata = roles 14 | self.sdk = sdk 15 | 16 | def create_permission_lookup(self): 17 | all_permission_sets = self.get_all_permission_sets() 18 | instance_permission_set_dict = { 19 | perm.name: perm.id for perm in all_permission_sets} 20 | return instance_permission_set_dict 21 | 22 | def create_model_lookup(self): 23 | all_model_sets = self.get_all_model_sets() 24 | instance_model_set_dict = { 25 | model.name: model.id for model in all_model_sets} 26 | return instance_model_set_dict 27 | 28 | def create_allgroup_lookup(self): 29 | all_groups = self.sdk.all_groups() 30 | group_dict = {group.name: group.id for group in all_groups} 31 | return group_dict 32 | 33 | def create_allrole_lookup(self): 34 | all_roles = self.sdk.all_roles() 35 | role_dict = {role.name: role.id for role in all_roles} 36 | return role_dict 37 | 38 | def create_instance_roles(self): 39 | model_lookup = self.create_model_lookup() 40 | permission_lookup = self.create_permission_lookup() 41 | 42 | role_output = [] 43 | 44 | for role_name, metadata in self.role_metadata.items(): 45 | model_set_id = model_lookup.get(metadata.get('model_set')) 46 | permission_set_id = permission_lookup.get( 47 | metadata.get('permission_set')) 48 | 49 | body = models.WriteRole( 50 | name=role_name, 51 | permission_set_id=permission_set_id, 52 | model_set_id=model_set_id 53 | ) 54 | try: 55 | role = self.sdk.create_role( 56 | body=body 57 | ) 58 | except looker_sdk.error.SDKError as roleerror: 59 | logger.debug(roleerror) 60 | role_id = self.sdk.search_roles(name=role_name)[0].id 61 | role = self.sdk.update_role(role_id=role_id, body=body) 62 | temp = {} 63 | temp['role_id'] = role.id 64 | temp['role_name'] = role_name 65 | role_output.append(temp) 66 | logger.info(role_output) 67 | return role_output 68 | 69 | def set_role(self, role_id: str, group_id: list) -> str: 70 | try: 71 | self.sdk.set_role_groups(role_id, group_id) 72 | return logger.info(f'attributing {group_id} permissions on instance') 73 | except looker_sdk.error.SDKError: 74 | return logger.info('something went wrong') 75 | 76 | def attribute_instance_roles(self, created_role_metadata: list): 77 | role_lookup = self.create_allrole_lookup() 78 | group_lookup = self.create_allgroup_lookup() 79 | 80 | for role_name, metadata in self.role_metadata.items(): 81 | teams = metadata.get('team') 82 | role_id = role_lookup.get(role_name) 83 | group_id_list = [] 84 | for team in teams: 85 | group_id = group_lookup.get(team) 86 | group_id_list.append(group_id) 87 | self.set_role(role_id=role_id, group_id=group_id_list) 88 | 89 | def sync_roles(self): 90 | all_role_lookup = self.create_allrole_lookup() 91 | all_role_lookup.pop('Admin') 92 | yaml_role = [role for role in self.role_metadata] 93 | logger.debug(all_role_lookup) 94 | 95 | for role_name in all_role_lookup.keys(): 96 | if role_name not in yaml_role: 97 | role_id = self.sdk.search_roles(name=role_name)[0].id 98 | self.sdk.delete_role(role_id=role_id) 99 | 100 | def execute(self): 101 | created_roles = self.create_instance_roles() 102 | self.attribute_instance_roles(created_role_metadata=created_roles) 103 | self.sync_roles() 104 | -------------------------------------------------------------------------------- /lmanage/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | import click 17 | import configparser 18 | import os 19 | from functools import wraps 20 | from lmanage.capturator.looker_api_reader import LookerApiReader 21 | from lmanage.capturator.looker_config_saver import LookerConfigSaver 22 | from lmanage.configurator.looker_config_reader import LookerConfigReader 23 | from lmanage.configurator.looker_provisioner import LookerProvisioner 24 | from lmanage.utils import logger_creation as log_color 25 | 26 | # logger = log_color.init_logger(__name__, testing_mode=False) 27 | 28 | 29 | @click.group() 30 | @click.version_option() 31 | def lmanage(): 32 | pass 33 | 34 | 35 | def log_args(**kwargs): 36 | log_level = kwargs.get('verbose', False) 37 | logger = log_color.init_logger(__name__, testing_mode=log_level) 38 | for k, v in kwargs.items(): 39 | if {v} != None: 40 | logger.info( 41 | f'You have set {v} for your {k} variable') 42 | else: 43 | logger.debug( 44 | f'There is no value set for {k} please use the `--help` flag to see input parameters.') 45 | 46 | 47 | def clean_args(**kwargs): 48 | kwargs['config_dir'] = kwargs['config_dir'].rstrip('/') 49 | return kwargs 50 | 51 | 52 | def common_options(f): 53 | @click.option('-i', '--ini-file', 54 | help='Specify API Credentials in an ini file, if no path is given program will assume these values are set as environmental variables as denoted at https://github.com/looker-open-source/sdk-codegen#environment-variable-configuration.') 55 | @click.option('-cd', '--config-dir', 56 | help='Where to save the YAML files to use for instance configuration.', required=True) 57 | @click.option('-v', '--verbose', 58 | is_flag=True, 59 | help='Add this flag to get a more verbose version of the returned stout text.') 60 | @click.option('-f', '--force', 61 | is_flag=True, 62 | help='Add this flag to skip confirmation prompt for capturing and configuring instance.') 63 | @click.pass_context 64 | @wraps(f) 65 | def decorated_function(ctx, *args, **kwargs): 66 | log_args(**kwargs) 67 | kwargs = clean_args(**kwargs) 68 | return ctx.invoke(f, *args, **kwargs) 69 | return decorated_function 70 | 71 | 72 | @lmanage.command() 73 | @common_options 74 | def capturator(**kwargs): 75 | """Captures security settings for your looker instance""" 76 | ini_file, config_dir, verbose, force = kwargs.get('ini_file'), kwargs.get( 77 | 'config_dir'), kwargs.get('verbose'), kwargs.get('force') 78 | 79 | try: 80 | target_url = get_target_url(ini_file) 81 | if force or (not force and click.confirm(f'\nYou are about to capture settings and content from instance {target_url}. Proceed?')): 82 | config = LookerApiReader(ini_file).get_config() 83 | LookerConfigSaver(config_dir).save(config) 84 | except RuntimeError as e: 85 | print(e) 86 | 87 | 88 | @lmanage.command() 89 | @common_options 90 | def configurator(**kwargs): 91 | """Configures security settings for your looker instance""" 92 | ini_file, config_dir, verbose, force = kwargs.get('ini_file'), kwargs.get( 93 | 'config_dir'), kwargs.get('verbose'), kwargs.get('force') 94 | 95 | try: 96 | target_url = get_target_url(ini_file) 97 | config_reader = LookerConfigReader(config_dir) 98 | config_reader.read() 99 | summary = config_reader.get_summary() 100 | if force or (not force and click.confirm(f'\nYou are about configure instance {target_url} with:\n{summary}\nAre you sure you want to proceed?')): 101 | LookerProvisioner(ini_file, verbose).provision(config_reader.config) 102 | except RuntimeError as e: 103 | print(e) 104 | 105 | 106 | def get_target_url(ini_file): 107 | if ini_file: 108 | config_parser = configparser.ConfigParser() 109 | successful_read = config_parser.read(ini_file) 110 | 111 | if not successful_read: 112 | raise FileNotFoundError( 113 | f'Configuration file "{ini_file}" not found.') 114 | 115 | sections = config_parser.sections() 116 | return config_parser.get(sections[0], 'base_url') 117 | else: 118 | target_url = os.getenv('LOOKERSDK_BASE_URL') 119 | if target_url is None: 120 | raise ValueError( 121 | 'LOOKERSDK_BASE_URL environment variable not set.') 122 | 123 | 124 | if __name__ == '__main__': 125 | lmanage() 126 | -------------------------------------------------------------------------------- /lmanage/configurator/.vimspector.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": { 3 | "run": { 4 | "adapter": "debugpy", 5 | "configuration": { 6 | "request": "launch", 7 | "protocol": "auto", 8 | "stopOnEntry": true, 9 | "console": "integratedTerminal", 10 | "program": "${workspaceRoot}/provision_instance_permission_structure.py", 11 | "cwd": "${workspaceRoot}" 12 | } 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lmanage/configurator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __version__ = '0.1.4' 16 | -------------------------------------------------------------------------------- /lmanage/configurator/content_configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/configurator/content_configuration/__init__.py -------------------------------------------------------------------------------- /lmanage/configurator/content_configuration/clean_instance_content.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | from lmanage.configurator.create_object import CreateObject 3 | 4 | 5 | class CleanInstanceContent(CreateObject): 6 | def __init__(self, sdk, logger) -> None: 7 | self.sdk = sdk 8 | self.logger = logger 9 | 10 | def execute(self): 11 | # self.__delete_boards() 12 | self.__delete_dashboards() 13 | self.__delete_looks() 14 | self.__delete_scheduled_plans() 15 | self.__empty_dashboard_trash() 16 | self.__empty_look_trash() 17 | 18 | def __delete_boards(self) -> None: 19 | all_boards = self.sdk.all_boards() 20 | delete_board_list = [board.id for board in all_boards] 21 | for board_id in tqdm(delete_board_list, 'Scrubbing Instance Boards', unit='boards', colour="#2c8558"): 22 | self.sdk.delete_board(board_id=board_id) 23 | self.logger.debug( 24 | f'cleaning the trash board {board_id} from instance') 25 | 26 | def __delete_dashboards(self) -> None: 27 | all_dashboards = self.sdk.all_dashboards() 28 | delete_id_list = [ 29 | dash.id for dash in all_dashboards if '::' not in dash.id] 30 | for dash_id in tqdm(delete_id_list, 'Scrubbing Instance Dashboards', unit='dashboards', colour="#2c8558"): 31 | self.sdk.delete_dashboard(dashboard_id=dash_id) 32 | self.logger.debug( 33 | f'cleaning the trash dashboard {dash_id} from instance') 34 | 35 | def __delete_looks(self) -> None: 36 | all_looks = self.sdk.all_looks() 37 | delete_id_list = [look.id for look in all_looks] 38 | for look_id in tqdm(delete_id_list, 'Scrubbing Instance Looks', unit='looks', colour="#2c8558"): 39 | self.logger.debug( 40 | f'Deleting look {look_id} from instance') 41 | self.sdk.delete_look(look_id=look_id) 42 | 43 | def __delete_scheduled_plans(self) -> None: 44 | all_schedules = self.sdk.all_scheduled_plans(all_users=True) 45 | schedule_plan_id_list = [schedule.id for schedule in all_schedules] 46 | for schedule_plan_id in tqdm(schedule_plan_id_list, 'Scrubbing Instance Schedules', unit='schedules', colour="#2c8558"): 47 | self.sdk.delete_scheduled_plan(scheduled_plan_id=schedule_plan_id) 48 | self.logger.debug( 49 | f'cleaning the trash schedule {schedule_plan_id}') 50 | 51 | def __empty_dashboard_trash(self) -> None: 52 | trash_dash = self.sdk.search_dashboards(deleted=True) 53 | trash_dash_id_list = [dash.id for dash in trash_dash] 54 | for dash_id in tqdm(trash_dash_id_list, 'Emptying Dashboard Trash', unit='dashboards', colour="#2c8558"): 55 | self.sdk.delete_dashboard(dashboard_id=dash_id) 56 | self.logger.debug( 57 | f'cleaning the trash dashboard {dash_id} from instance') 58 | 59 | def __empty_look_trash(self) -> None: 60 | trash_look = self.sdk.search_looks(deleted=True) 61 | trash_look_id_list = [look.id for look in trash_look] 62 | for look_id in tqdm(trash_look_id_list, 'Emptying Look Trash', unit='Look', colour="#2c8558"): 63 | self.sdk.delete_look(look_id=look_id) 64 | self.logger.debug( 65 | f'cleaning the trash dashboard {look_id} from instance') 66 | -------------------------------------------------------------------------------- /lmanage/configurator/content_configuration/create_boards.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | from looker_sdk import models40 as models 3 | from lmanage.configurator.create_object import CreateObject 4 | from lmanage.utils import logger_creation as log_color 5 | # logger = log_color.init_logger(__name__, logger_level) 6 | 7 | 8 | class CreateBoards(CreateObject): 9 | def __init__(self, sdk, board_metadata, dashboard_mapping, look_mapping) -> None: 10 | self.sdk = sdk 11 | self.dashboard_mapping = dashboard_mapping 12 | self.look_mapping = look_mapping 13 | self.board_metadata = board_metadata 14 | 15 | def create_boards(self, bmeta: dict): 16 | board_body = models.WriteBoard( 17 | description=bmeta.get('description'), 18 | title=bmeta.get('title'), 19 | section_order=bmeta.get('section_order') 20 | ) 21 | 22 | board = self.sdk.create_board(body=board_body) 23 | 24 | return board 25 | 26 | def create_board_section(self, section_object, board_id): 27 | new_board_section = models.WriteBoardSection( 28 | board_id=board_id, 29 | description=section_object.description, 30 | item_order=section_object.item_order, 31 | title=section_object.title 32 | ) 33 | resp = self.sdk.create_board_section(body=new_board_section) 34 | return resp.id 35 | 36 | def match_dashboard_id(self, source_dashboard_id: str) -> str: 37 | pass 38 | 39 | def match_look_id(self, source_look_id: str) -> str: 40 | pass 41 | 42 | def create_board_item(self, target_board_section_id): 43 | 44 | dashboard_id = None 45 | look_id = None 46 | 47 | if target_board_section_id.dashboard_id: 48 | dashboard_id = self.match_dashboard_id( 49 | target_board_section_id.dashboard_id) 50 | if target_board_section_id.look_id: 51 | look_id = self.match_look_id(target_board_section_id.look_id) 52 | 53 | new_board_item = models.WriteBoardItem() 54 | new_board_item.__dict__.update(source_board_item_object.__dict__) 55 | new_board_item.dashboard_id = dashboard_id 56 | new_board_item.look_id = look_id 57 | new_board_item.homepage_section_id = target_board_section_id 58 | 59 | logger.info( 60 | "Creating item", 61 | extra={ 62 | "section_id": new_board_item.homepage_section_id, 63 | "dashboard_id": new_board_item.dashboard_id, 64 | "look_id": new_board_item.look_id, 65 | "url": new_board_item.url 66 | } 67 | ) 68 | resp = target_sdk.create_homepage_item(new_board_item) 69 | logger.info("Item created", extra={"id": resp.id}) 70 | 71 | return resp 72 | 73 | def execute(self): 74 | ''' 75 | create boards 76 | in boards create sections 77 | in sections create board items 78 | check content access provisions 79 | ''' 80 | for board in self.board_metadata: 81 | nb = self.create_boards(bmeta=board) 82 | 83 | print(nb) 84 | -------------------------------------------------------------------------------- /lmanage/configurator/create_object.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class CreateObject(ABC): 5 | @abstractmethod 6 | def execute(): 7 | pass 8 | -------------------------------------------------------------------------------- /lmanage/configurator/folder_configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/configurator/folder_configuration/__init__.py -------------------------------------------------------------------------------- /lmanage/configurator/folder_configuration/create_instance_folders.py: -------------------------------------------------------------------------------- 1 | from looker_sdk import error 2 | from looker_sdk.sdk.api40 import models 3 | from tqdm import tqdm 4 | from lmanage.utils import logger_creation as log_color 5 | import typing 6 | # logger = log_color.init_logger(__name__, logger_level) 7 | 8 | 9 | class CreateInstanceFolders(): 10 | def __init__(self, folder_metadata, sdk, logger): 11 | self.sdk = sdk 12 | self.folder_metadata = folder_metadata 13 | self.logger = logger 14 | 15 | def create_folders(self): 16 | self.__delete_top_level_folders() 17 | for folder_tree in tqdm(self.folder_metadata, desc="Unesting Folder Data", unit="unested", colour="#2c8558"): 18 | metadata_list = [] 19 | metadata_list = self.__walk_folder_structure( 20 | folder_tree=folder_tree, data_storage=metadata_list) 21 | self.logger.debug('Retrieved YAML folder files') 22 | self.logger.debug(f'Older metadata = {metadata_list}') 23 | return metadata_list 24 | 25 | def create_folder_mapping_dict(self, folder_metadata: list) -> dict: 26 | ''' 27 | create a folder mapping dict with legacy folder id as it's key and new folder id as the value for use in content migration 28 | ''' 29 | response = {} 30 | for folder_obj in folder_metadata: 31 | lid = folder_obj.get('old_folder_id') 32 | fid = folder_obj.get('new_folder_id') 33 | response[lid] = fid 34 | 35 | return response 36 | 37 | def __delete_top_level_folders(self): 38 | def is_top_level_folder(folder): 39 | return folder['parent_id'] is not None and int(folder['parent_id']) == 1 40 | all_folders = self.sdk.all_folders('id, name, parent_id') 41 | for folder in tqdm(all_folders): 42 | if is_top_level_folder(folder): 43 | try: 44 | self.logger.debug( 45 | f'Deleting folder "{folder["name"]}"') 46 | self.sdk.delete_folder(folder_id=folder['id']) 47 | except error.SDKError as e: 48 | self.logger.error(e) 49 | 50 | def __walk_folder_structure(self, folder_tree: dict, data_storage: list, parent_id: typing.Optional[str] = None): 51 | folder_name = folder_tree.get('name') 52 | try: 53 | if parent_id is None: 54 | folder_id = '1' 55 | else: 56 | folder_id = self.__create_folder(folder_name=folder_name, 57 | parent_id=parent_id).get('id') 58 | temp = { 59 | 'name': folder_name, 60 | 'old_folder_id': folder_tree.get('id'), 61 | 'new_folder_id': folder_id, 62 | 'content_metadata_id': folder_tree.get('content_metadata_id'), 63 | 'team_view': folder_tree.get('team_view'), 64 | 'team_edit': folder_tree.get('team_edit'), 65 | } 66 | self.logger.debug(f'Data structure to be appended = {temp}') 67 | data_storage.append(temp) 68 | if isinstance(folder_tree.get('subfolder'), list): 69 | for subfolder in folder_tree.get('subfolder'): 70 | self.__walk_folder_structure(subfolder, data_storage, 71 | parent_id=folder_id) 72 | except error.SDKError as e: 73 | self.logger.warn('error %s', e) 74 | self.logger.warn( 75 | 'you have a duplicate folder called %s with the same parent, this is against best practice and LManage is ignoring it', folder_name) 76 | 77 | # if isinstance(dict_obj.get('subfolder'), list): 78 | # for subfolder in dict_obj.get('subfolder'): 79 | # self.walk_folder_structure(subfolder, data_storage, 80 | # parent_name=new_folder_id) 81 | 82 | return data_storage 83 | 84 | def __create_folder(self, folder_name: str, parent_id: str): 85 | self.logger.debug(f'Creating folder "{folder_name}"') 86 | folder = self.sdk.create_folder( 87 | body=models.CreateFolder( 88 | name=folder_name, 89 | parent_id=parent_id 90 | ) 91 | ) 92 | return folder 93 | 94 | # def sync_folders(self, created_folder: list): 95 | # all_folders = self.sdk.all_folders() 96 | # created_folder_ids = [fobj.get('folder_id') for fobj in created_folder] 97 | # all_folder_ids = [] 98 | 99 | # for folder in all_folders: 100 | # if folder.is_personal: 101 | # pass 102 | # elif folder.parent_id is None: 103 | # pass 104 | # else: 105 | # all_folder_ids.append(folder.id) 106 | 107 | # for fids in all_folder_ids: 108 | # if fids not in created_folder_ids: 109 | # try: 110 | # self.sdk.delete_folder(folder_id=fids) 111 | # self.logger.debug( 112 | # 'deleting folder %s to sync with yaml config', fids) 113 | 114 | # except error.SDKError as InheritanceError: 115 | # self.logger.debug('root folder has been deleted so %s', 116 | # InheritanceError) 117 | # return 'your folders are in sync with your yaml file' 118 | -------------------------------------------------------------------------------- /lmanage/configurator/folder_configuration/folder_config.py: -------------------------------------------------------------------------------- 1 | from os import name 2 | from lmanage.utils import logger_creation as log_color 3 | 4 | # logger = log_color.init_logger(__name__, logger_level) 5 | 6 | 7 | class FolderConfig(): 8 | ''' Class to read in folder metadata and unested_folder_data ''' 9 | 10 | def __init__(self, folders): 11 | self.folder_metadata = folders 12 | 13 | def unnest_folder_data(self): 14 | response = [] 15 | 16 | for d in self.folder_metadata: 17 | folder_dict = d 18 | metadata_list = [] 19 | metadata_list = self.walk_folder_structure( 20 | dict_obj=folder_dict, 21 | data_storage=metadata_list, 22 | parent_id='1') 23 | 24 | response.append(metadata_list) 25 | 26 | logger.info('retrieved yaml folder files') 27 | logger.debug('folder metadata = %s', response) 28 | 29 | return response 30 | 31 | def walk_folder_structure(self, dict_obj: dict, data_storage: list, parent_id: str): 32 | temp = {} 33 | temp['name'] = dict_obj.get('name') 34 | temp['legacy_id'] = dict_obj.get('id') 35 | temp['team_edit'] = dict_obj.get('team_edit') 36 | temp['team_view'] = dict_obj.get('team_view') 37 | temp['parent_id'] = parent_id 38 | logger.debug('data_structure to be appended = %s', temp) 39 | data_storage.append(temp) 40 | 41 | if isinstance(dict_obj.get('subfolder'), list): 42 | for subfolder in dict_obj.get('subfolder'): 43 | self.walk_folder_structure(subfolder, data_storage, 44 | parent_id=dict_obj.get('name')) 45 | 46 | return data_storage 47 | -------------------------------------------------------------------------------- /lmanage/configurator/looker_config_reader.py: -------------------------------------------------------------------------------- 1 | from lmanage.utils import parse_yaml as py, logger_creation as log_color 2 | 3 | 4 | class LookerConfigReader: 5 | def __init__(self, config_dir): 6 | logger_level = "INFO" # kwargs.get('verbose') 7 | logger = log_color.init_logger(__name__, logger_level) 8 | logger.info( 9 | "----------------------------------------------------------------------------------------" 10 | ) 11 | logger.info("parsing yaml file") 12 | self.settings_yaml = py.Yaml(yaml_path=f"{config_dir}/settings.yaml") 13 | self.content_yaml = py.Yaml(yaml_path=f"{config_dir}/content.yaml") 14 | # self.logger = log_color.init_logger(__name__, logger_level) 15 | 16 | def read(self): 17 | self.config = { 18 | "folder_metadata": self.settings_yaml.get_metadata("folder_metadata"), 19 | "permission_set_metadata": self.settings_yaml.get_metadata( 20 | "permission_set_metadata" 21 | ), 22 | "model_set_metadata": self.settings_yaml.get_metadata("model_set_metadata"), 23 | "role_metadata": self.settings_yaml.get_metadata("role_metadata"), 24 | "user_attribute_metadata": self.settings_yaml.get_metadata( 25 | "user_attribute_metadata" 26 | ), 27 | "look_metadata": self.content_yaml.get_metadata("look_metadata"), 28 | "dashboard_metadata": self.content_yaml.get_metadata("dashboard_metadata"), 29 | # 'board_metadata': self.content_yaml.get_metadata('board_metadata'), 30 | } 31 | 32 | def get_summary(self): 33 | config = self.config 34 | look_scheduled_plan_count = sum( 35 | [ 36 | len(l.get("scheduled_plans")) if l.get("scheduled_plans") else 0 37 | for l in config["look_metadata"] 38 | ] 39 | ) 40 | scheduled_plan_and_alert_counts = [ 41 | ( 42 | len(d.get("scheduled_plans")) if d.get("scheduled_plans") else 0, 43 | len(d.get("alerts")) if d.get("alerts") else 0, 44 | ) 45 | for d in config["dashboard_metadata"] 46 | ] 47 | scheduled_plan_count = 0 48 | alert_count = 0 49 | 50 | for first, second in scheduled_plan_and_alert_counts: 51 | scheduled_plan_count += first 52 | alert_count += second 53 | 54 | return f""" 55 | {len(config['permission_set_metadata']):<5} permissions 56 | {len(config['model_set_metadata']):<5} models 57 | {len(config['role_metadata']):<5} roles 58 | {len(config['user_attribute_metadata']):<5} user attributes 59 | {len(config['look_metadata']):<5} looks 60 | {look_scheduled_plan_count:<5} look scheduled plans 61 | {len(config['dashboard_metadata']):<5} dashboards 62 | {scheduled_plan_count:<5} dashboard scheduled plans 63 | {alert_count:<5} dashboard alerts 64 | """ 65 | -------------------------------------------------------------------------------- /lmanage/configurator/looker_provisioner.py: -------------------------------------------------------------------------------- 1 | from lmanage.looker_auth import LookerAuth 2 | from lmanage.configurator.user_attribute_configuration import create_and_assign_user_attributes as cuap 3 | from lmanage.configurator.folder_configuration import create_and_provision_instance_folders as cfp, create_instance_folders as cf 4 | from lmanage.configurator.user_group_configuration import create_instance_groups as gc, create_instance_roles as up, create_role_base as rc 5 | from lmanage.configurator.content_configuration import clean_instance_content as ccp, create_looks as cl, create_dashboards as cd, create_boards as cb 6 | from lmanage.utils import logger_creation as log_color 7 | 8 | 9 | class LookerProvisioner(): 10 | def __init__(self, ini_file, verbose): 11 | self.sdk = LookerAuth().authenticate(ini_file) 12 | self.logger = log_color.init_logger(__name__, testing_mode=verbose) 13 | 14 | def provision(self, metadata): 15 | ############### 16 | # Role Config # 17 | ############### 18 | # Create Permission and Model Sets 19 | rc.CreateRoleBase(permissions=metadata['permission_set_metadata'], 20 | model_sets=metadata['model_set_metadata'], sdk=self.sdk, logger=self.logger).execute() 21 | 22 | ################# 23 | # Folder Config # 24 | ################# 25 | # CREATE NEW FOLDERS 26 | folder_objects = cf.CreateInstanceFolders( 27 | folder_metadata=metadata['folder_metadata'], sdk=self.sdk, logger=self.logger) 28 | created_folder_metadata = folder_objects.create_folders() 29 | 30 | # CREATE FOLDER TO FOLDER Dict 31 | folder_mapping_obj = folder_objects.create_folder_mapping_dict( 32 | folder_metadata=created_folder_metadata) 33 | 34 | ################ 35 | # Group Config # 36 | ################ 37 | # CREATE NEW GROUPS FROM YAML FILE TEAM VALUES 38 | gc.CreateInstanceGroups( 39 | folders=created_folder_metadata, 40 | user_attributes=metadata['user_attribute_metadata'], 41 | roles=metadata['role_metadata'], 42 | sdk=self.sdk, 43 | logger=self.logger).execute() 44 | 45 | ########################### 46 | # Folder Provision Config # 47 | ########################### 48 | # CREATE NEW GROUPS FROM YAML FILE TEAM VALUES 49 | cfp.CreateAndProvisionInstanceFolders( 50 | folders=created_folder_metadata, 51 | sdk=self.sdk, logger=self.logger).execute() 52 | 53 | ############### 54 | # Role Config # 55 | ############### 56 | up.CreateInstanceRoles(roles=metadata['role_metadata'], 57 | sdk=self.sdk, 58 | logger=self.logger).execute() 59 | 60 | ######################### 61 | # User Attribute Config # 62 | ######################### 63 | # FIND UNIQUE USER ATTRIBUTES AND ATTRIBUTE TO TEAM 64 | cuap.CreateAndAssignUserAttributes( 65 | user_attributes=metadata['user_attribute_metadata'], 66 | sdk=self.sdk, 67 | logger=self.logger).execute() 68 | 69 | ############################ 70 | # Content Transport Config # 71 | ############################ 72 | # EMPTY TRASH CAN OF ALL DELETED CONTENT 73 | ccp.CleanInstanceContent(sdk=self.sdk, logger=self.logger).execute() 74 | 75 | # FIND LOOKS AND REMAKE THEM 76 | cl.CreateLooks( 77 | sdk=self.sdk, 78 | folder_mapping=folder_mapping_obj, 79 | content_metadata=metadata['look_metadata'], 80 | logger=self.logger).execute() 81 | 82 | # Find DASHBOARDS AND REMAKE THEM 83 | cd.CreateDashboards( 84 | sdk=self.sdk, 85 | folder_mapping=folder_mapping_obj, 86 | content_metadata=metadata['dashboard_metadata'], 87 | logger=self.logger).execute() 88 | -------------------------------------------------------------------------------- /lmanage/configurator/user_attribute_configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/configurator/user_attribute_configuration/__init__.py -------------------------------------------------------------------------------- /lmanage/configurator/user_attribute_configuration/create_and_assign_user_attributes.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from looker_sdk import error 3 | from looker_sdk.sdk.api40 import models 4 | from tqdm import tqdm 5 | from tenacity import retry, wait_random, wait_fixed, stop_after_attempt 6 | from lmanage.configurator.create_object import CreateObject 7 | 8 | 9 | class CreateAndAssignUserAttributes(CreateObject): 10 | def __init__(self, user_attributes, sdk, logger): 11 | self.user_attribute_metadata = user_attributes 12 | self.sdk = sdk 13 | self.logger = logger 14 | 15 | def existing_user_attributes(self) -> dict: 16 | all_instance_ua = self.sdk.all_user_attributes() 17 | 18 | all_ua = {ua.name: ua.id for ua in all_instance_ua} 19 | return all_ua 20 | 21 | def create_user_attribute_if_not_exists(self): 22 | existing_ua = self.existing_user_attributes() 23 | yaml_user_attributes = self.user_attribute_metadata 24 | 25 | for ua in tqdm(yaml_user_attributes, desc="User Attribute Creation", unit=" attributes", colour="#2c8558"): 26 | name = ua.get('name') 27 | if name in existing_ua.keys(): 28 | logging.warning( 29 | f'user attribute {name} already exists on this instance') 30 | else: 31 | datatype = ua.get('uatype') 32 | value_is_hidden = ua.get('hidden_value') 33 | user_view = ua.get('user_view') 34 | user_edit = ua.get('user_edit') 35 | 36 | try: 37 | ua_permissions = models.WriteUserAttribute( 38 | name=name, 39 | label=name, 40 | type=datatype, 41 | value_is_hidden=value_is_hidden, 42 | user_can_view=user_view, 43 | user_can_edit=user_edit, 44 | default_value=ua.get('default_value') 45 | ) 46 | response = self.sdk.create_user_attribute( 47 | body=ua_permissions) 48 | logging.info(f'created user attribute {response.label}') 49 | except error.SDKError as err: 50 | logging.error('skipping permission because already exists') 51 | 52 | def sync_user_attributes(self): 53 | instance_ua = self.existing_user_attributes() 54 | config_ua = [ua.get('name') for ua in self.user_attribute_metadata] 55 | sys_default_ua = [ 56 | 'email', 57 | 'first_name', 58 | 'id', 59 | 'google_user_id', 60 | 'landing_page', 61 | 'last_name', 62 | 'number_format', 63 | 'locale', 64 | 'name', 65 | "looker_internal_email_domain_allowlist", 66 | ] 67 | 68 | for ua in sys_default_ua: 69 | if ua in instance_ua: 70 | instance_ua.pop(ua, None) 71 | 72 | for ua_name in instance_ua.keys(): 73 | if ua_name not in config_ua: 74 | ua_id = instance_ua.get(ua_name) 75 | self.sdk.delete_user_attribute(ua_id) 76 | logging.info( 77 | f'''deleting ua {ua_name} because 78 | it is not listed in the yaml config''') 79 | 80 | def add_group_values_to_ua(self): 81 | instance_ua = self.existing_user_attributes() 82 | all_instance_groups = self.sdk.all_groups() 83 | group_metadata = { 84 | group.name: group.id for group in all_instance_groups} 85 | yaml_user_attributes = self.user_attribute_metadata 86 | for ua in yaml_user_attributes: 87 | if len(ua.get('teams')) > 0: 88 | for team_val in ua.get('teams'): 89 | group_id = group_metadata.get(list(team_val.keys())[0]) 90 | 91 | meta_value = list(team_val.values())[0] 92 | ua_id = instance_ua.get(ua.get('name')) 93 | 94 | params_to_add = models.UserAttributeGroupValue( 95 | value=meta_value, 96 | value_is_hidden=False 97 | ) 98 | 99 | self.sdk.update_user_attribute_group_value( 100 | group_id=group_id, 101 | user_attribute_id=ua_id, 102 | body=params_to_add) 103 | 104 | def execute(self): 105 | # CREATE NEW USER ATTRIBUTES 106 | self.create_user_attribute_if_not_exists() 107 | 108 | # DELETE ALL USER ATTRIBUTES THAT DON'T MATCH WITH YAML 109 | self.sync_user_attributes() 110 | 111 | # ADD VALUES TO INSCOPE USER ATTRIBUTES 112 | self.add_group_values_to_ua() 113 | -------------------------------------------------------------------------------- /lmanage/configurator/user_group_configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/configurator/user_group_configuration/__init__.py -------------------------------------------------------------------------------- /lmanage/configurator/user_group_configuration/create_instance_groups.py: -------------------------------------------------------------------------------- 1 | from looker_sdk import error 2 | from looker_sdk.sdk.api40 import models 3 | from lmanage.utils.errorhandling import return_error_message 4 | from tqdm import tqdm 5 | from tenacity import retry, wait_fixed, wait_random, stop_after_attempt 6 | from lmanage.configurator.create_object import CreateObject 7 | 8 | 9 | class CreateInstanceGroups(CreateObject): 10 | def __init__(self, folders, user_attributes, roles, sdk, logger) -> None: 11 | self.folder_metadata = folders 12 | self.user_attribute_metadata = user_attributes 13 | self.role_metadata = roles 14 | self.sdk = sdk 15 | self.logger = logger 16 | 17 | def extract_teams(self, container, data_storage, ua: bool): 18 | for team in container: 19 | team_list = team.get('teams') 20 | if len(team_list) > 0: 21 | if ua: 22 | for team_val in team_list: 23 | team_app = list(team_val.keys())[0] 24 | data_storage.append(team_app) 25 | else: 26 | for team_val in team_list: 27 | data_storage.append(team_val) 28 | return data_storage 29 | 30 | def extract_folder_teams(self, container, data_storage): 31 | for folder_element in container: 32 | if isinstance(folder_element.get('team_edit'), list): 33 | edit_group = folder_element.get('team_edit') 34 | for group in edit_group: 35 | data_storage.append(group) 36 | if isinstance(folder_element.get('team_view'), list): 37 | view_group = folder_element.get('team_view') 38 | for group in view_group: 39 | data_storage.append(group) 40 | else: 41 | pass 42 | 43 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 44 | def search_looker_groups(self, sdk, group_name: str) -> list: 45 | s = sdk.search_groups(name=group_name) 46 | return s 47 | 48 | def create_group_if_not_exists(self, 49 | sdk, 50 | group_name: str) -> dict: 51 | """ Create a Looker Group and add Group attributes 52 | 53 | :group_name: Name of a Looker group to create. 54 | :rtype: Looker Group object. 55 | """ 56 | # get group if exists 57 | try: 58 | self.logger.debug(f'Creating group "{group_name}"') 59 | group = sdk.create_group( 60 | body=models.WriteGroup( 61 | can_add_to_content_metadata=True, 62 | name=group_name 63 | ) 64 | ) 65 | return group 66 | except error.SDKError as grouperr: 67 | err_msg = return_error_message(grouperr) 68 | self.logger.debug( 69 | 'You have hit a warning creating your group; warning = %s', err_msg) 70 | self.logger.debug(grouperr) 71 | group = self.search_looker_groups( 72 | sdk=self.sdk, group_name=group_name) 73 | return group[0] 74 | 75 | def get_instance_group_metadata(self, 76 | sdk, 77 | unique_group_list: list) -> list: 78 | group_metadata = [] 79 | 80 | for group_name in tqdm(unique_group_list, desc="Creating User Groups", unit=" user groups", colour="#2c8558"): 81 | group = self.create_group_if_not_exists(sdk, group_name) 82 | temp = {} 83 | temp['group_id'] = group.id 84 | temp['group_name'] = group.name 85 | group_metadata.append(temp) 86 | 87 | return group_metadata 88 | 89 | def sync_groups(self, 90 | group_name_list: list) -> str: 91 | 92 | all_groups = self.sdk.all_groups() 93 | group_dict = {group.name: group.id for group in all_groups} 94 | # Deleting Standard Groups 95 | try: 96 | del group_dict['All Users'] 97 | except: 98 | pass 99 | 100 | for group_name in group_dict.keys(): 101 | if group_name not in group_name_list: 102 | self.sdk.delete_group(group_id=group_dict[group_name]) 103 | self.logger.info( 104 | f'deleting group {group_name} to sync with yaml config') 105 | 106 | return 'your groups are in sync with your yaml file' 107 | 108 | def execute(self): 109 | team_list = [] 110 | # extracting user attribute teams 111 | self.extract_teams(container=self.user_attribute_metadata, 112 | data_storage=team_list, ua=True) 113 | # extracting role based teams 114 | self.extract_teams( 115 | container=self.role_metadata, data_storage=team_list, ua=False) 116 | # extract nested folder access teams 117 | self.extract_folder_teams( 118 | container=self.folder_metadata, data_storage=team_list) 119 | 120 | team_list = list(set(team_list)) 121 | 122 | # create all the groups 123 | self.get_instance_group_metadata( 124 | sdk=self.sdk, unique_group_list=team_list) 125 | self.sync_groups(group_name_list=team_list) 126 | -------------------------------------------------------------------------------- /lmanage/configurator/user_group_configuration/create_instance_roles.py: -------------------------------------------------------------------------------- 1 | from looker_sdk import error 2 | from looker_sdk.sdk.api40 import models 3 | from lmanage.configurator.user_group_configuration.create_role_base import CreateRoleBase 4 | from lmanage.utils.errorhandling import return_error_message 5 | from tqdm import tqdm 6 | from tenacity import retry, wait_random, wait_fixed, stop_after_attempt 7 | 8 | 9 | class CreateInstanceRoles(CreateRoleBase): 10 | def __init__(self, roles, sdk, logger): 11 | self.role_metadata = roles 12 | self.sdk = sdk 13 | self.logger = logger 14 | 15 | def create_permission_lookup(self): 16 | all_permission_sets = self.get_all_permission_sets() 17 | instance_permission_set_dict = { 18 | perm.name: perm.id for perm in all_permission_sets} 19 | return instance_permission_set_dict 20 | 21 | def create_model_lookup(self): 22 | all_model_sets = self.get_all_model_sets() 23 | instance_model_set_dict = { 24 | model.name: model.id for model in all_model_sets} 25 | return instance_model_set_dict 26 | 27 | def create_allgroup_lookup(self): 28 | all_groups = self.sdk.all_groups() 29 | group_dict = {group.name: group.id for group in all_groups} 30 | return group_dict 31 | 32 | def create_allrole_lookup(self): 33 | all_roles = self.sdk.all_roles() 34 | role_dict = {role.name: role.id for role in all_roles} 35 | return role_dict 36 | 37 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5)) 38 | def update_looker_role(self, role_id: str, body: models.WriteRole): 39 | role = self.sdk.update_role(role_id=role_id, body=body) 40 | return role 41 | 42 | def create_instance_roles(self): 43 | model_lookup = self.create_model_lookup() 44 | permission_lookup = self.create_permission_lookup() 45 | 46 | role_output = [] 47 | 48 | for r_metadata in tqdm(self.role_metadata, desc="User Role Creation", unit=" user roles", colour="#2c8558"): 49 | role_name = r_metadata.get('name') 50 | model_set_id = model_lookup.get( 51 | r_metadata.get('model_set') if r_metadata.get('model_set') == 'All' else r_metadata.get('model_set').lower()) 52 | permission_set_id = permission_lookup.get( 53 | r_metadata.get('permission_set').lower()) 54 | if role_name != 'Admin': 55 | body = models.WriteRole( 56 | name=role_name, 57 | permission_set_id=str(permission_set_id), 58 | model_set_id=str(model_set_id) 59 | ) 60 | try: 61 | role = self.sdk.create_role( 62 | body=body 63 | ) 64 | 65 | except error.SDKError as roleerror: 66 | err_msg = return_error_message(roleerror) 67 | self.logger.debug( 68 | 'You have hit a warning creating your role \'%s\'; warning = %s', role_name, err_msg) 69 | role_id = self.sdk.search_roles(name=role_name)[0].id 70 | role = self.update_looker_role( 71 | role_id=str(role_id), body=body) 72 | temp = {} 73 | temp['role_id'] = role.id 74 | temp['role_name'] = role_name 75 | role_output.append(temp) 76 | else: 77 | pass 78 | self.logger.debug(role_output) 79 | return role_output 80 | 81 | def set_role(self, role_id: str, group_id: list) -> str: 82 | try: 83 | self.sdk.set_role_groups(role_id, group_id) 84 | return self.logger.debug(f'attributing {group_id} permissions on instance') 85 | except error.SDKError as role_err: 86 | err_msg = return_error_message(role_err) 87 | self.logger.debug( 88 | 'You have hit a warning setting your role, warning = %s', err_msg) 89 | self.logger.debug(role_err) 90 | 91 | def attribute_instance_roles(self, created_role_metadata: list): 92 | role_lookup = self.create_allrole_lookup() 93 | group_lookup = self.create_allgroup_lookup() 94 | 95 | for r_metadata in tqdm(self.role_metadata, desc="Instance Role Attribution", unit=" roles", colour="#2c8558"): 96 | role_name = r_metadata.get('name') 97 | teams = r_metadata.get('teams') 98 | role_id = role_lookup.get(role_name) 99 | group_id_list = [] 100 | for team in teams: 101 | group_id = group_lookup.get(team) 102 | group_id_list.append(group_id) 103 | self.set_role(role_id=role_id, group_id=group_id_list) 104 | 105 | def sync_roles(self): 106 | all_role_lookup = self.create_allrole_lookup() 107 | all_role_lookup.pop('Admin') 108 | yaml_role = [role.get('name') for role in self.role_metadata] 109 | 110 | for role_name in all_role_lookup.keys(): 111 | if role_name not in yaml_role: 112 | role_id = all_role_lookup.get(role_name) 113 | self.sdk.delete_role(role_id=role_id) 114 | 115 | def execute(self): 116 | created_roles = self.create_instance_roles() 117 | self.attribute_instance_roles(created_role_metadata=created_roles) 118 | self.sync_roles() 119 | -------------------------------------------------------------------------------- /lmanage/find_replace.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # find_and_replace.sh 3 | # set -x #echo on 4 | echo 'p for poetry run added in or np for vimspector' 5 | read INPUT 6 | 7 | echo "your input is $INPUT" 8 | 9 | POETRY="p" 10 | NOTPOETRY="np" 11 | 12 | if [ "$POETRY" == "$INPUT" ] 13 | then 14 | find . -type f -name "*.py" -print0 | xargs -0 sed -i -e 's/from configurator/from lmanage.configurator/g' -e 's/from utils/from lmanage.utils/g' -e 's/from capturator/from lmanage.capturator/g' 15 | 16 | else 17 | echo 'unpoetizing' 18 | fi 19 | 20 | if [ $NOTPOETRY == $INPUT ] 21 | then 22 | find . -type f -name "*.py" -print0 | xargs -0 sed -i -e 's/from lmanage.configurator/from configurator/g' -e 's/from lmanage.utils/from utils/g' -e 's/from lmanage.capturator/from capturator/g' 23 | 24 | else 25 | echo 'poetizing' 26 | fi 27 | -------------------------------------------------------------------------------- /lmanage/logger_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def setup_logger(): 5 | # Create a custom logger 6 | logger = logging.getLogger(__name__) 7 | 8 | # Set the level of logging. 9 | # You might want to use logging.DEBUG or logging.ERROR in other scenarios 10 | logger.setLevel(logging.INFO) 11 | 12 | # Create handlers. Here, we create a StreamHandler (for stdout) and a FileHandler (for writing logs to a file) 13 | c_handler = logging.StreamHandler() 14 | f_handler = logging.FileHandler('file.log') 15 | c_handler.setLevel(logging.INFO) 16 | f_handler.setLevel(logging.ERROR) 17 | 18 | # Create formatters and add it to handlers 19 | c_format = logging.Formatter('%(name)s - %(levelname)s - %(message)s') 20 | f_format = logging.Formatter( 21 | '%(asctime)s - %(name)s - %(levelname)s - %(message)s') 22 | c_handler.setFormatter(c_format) 23 | f_handler.setFormatter(f_format) 24 | 25 | # Add handlers to the logger 26 | logger.addHandler(c_handler) 27 | logger.addHandler(f_handler) 28 | 29 | return logger 30 | -------------------------------------------------------------------------------- /lmanage/looker_auth.py: -------------------------------------------------------------------------------- 1 | import looker_sdk 2 | from lmanage.logger_config import setup_logger 3 | from looker_sdk import error 4 | 5 | logger = setup_logger() 6 | 7 | 8 | class LookerAuth(): 9 | def authenticate(self, ini_file): 10 | sdk = looker_sdk.init40( 11 | config_file=ini_file) if ini_file else looker_sdk.init40() 12 | if self.__auth_check(sdk): 13 | logger.info('User is successfully authenticated to the API') 14 | return sdk 15 | else: 16 | raise Exception( 17 | 'User is not successfully authenticated. Please verify credentials in .ini file.') 18 | 19 | def __auth_check(self, sdk): 20 | try: 21 | sdk.me() 22 | return True 23 | except: 24 | return False 25 | -------------------------------------------------------------------------------- /lmanage/looker_config.py: -------------------------------------------------------------------------------- 1 | class LookerConfig(): 2 | def __init__(self, settings, content): 3 | self.settings = settings 4 | self.content = content 5 | -------------------------------------------------------------------------------- /lmanage/mapview/.vimspector.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": { 3 | "run": { 4 | "adapter": "debugpy", 5 | "configuration": { 6 | "python": "/usr/local/google/home/hugoselbie/.cache/pypoetry/virtualenvs/lmanage-UKKoJ9La-py3.8/bin/python", 7 | "request": "launch", 8 | "protocol": "auto", 9 | "stopOnEntry": true, 10 | "console": "integratedTerminal", 11 | "program": "${workspaceRoot}/mapview_execute.py", 12 | "cwd": "${workspaceRoot}" 13 | } 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /lmanage/mapview/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /lmanage/mapview/compare_content_to_lookml.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import coloredlogs 3 | import logging 4 | from lmanage.mapview.utils import parse_lookml 5 | 6 | logger = logging.getLogger(__name__) 7 | coloredlogs.install(level='INFO') 8 | logging.getLogger("looker_sdk").setLevel(logging.WARNING) 9 | 10 | 11 | def format_str_list(list_str): 12 | new_list = ast.literal_eval(list_str) 13 | return new_list 14 | 15 | 16 | def collate_view_files(parsed_lookml): 17 | response = [] 18 | for instance_lookml in parsed_lookml: 19 | for path in instance_lookml.keys(): 20 | if 'view.lkml' in path: 21 | response.append(instance_lookml) 22 | return response 23 | 24 | 25 | def extract_field_names_from_lookml(parsed_lookml, data_storage): 26 | 27 | try: 28 | dimensions = parsed_lookml['dimensions'] 29 | except KeyError: 30 | dimensions = [] 31 | logger.warning('no dimensions present in view') 32 | try: 33 | measures = parsed_lookml['measures'] 34 | except KeyError: 35 | measures = [] 36 | logger.warning('no measures present in view') 37 | 38 | view_name = parsed_lookml.get('name') 39 | for dim in dimensions: 40 | field_name = dim.get('name') 41 | full_name = f'{view_name}.{field_name}' 42 | data_storage[full_name] = view_name 43 | 44 | for meas in measures: 45 | if len(meas) > 0: 46 | field_name = meas.get('name') 47 | full_name = f'{view_name}.{field_name}' 48 | data_storage[full_name] = view_name 49 | 50 | return data_storage 51 | 52 | 53 | def match_fields_to_lookml(field_list, lookml): 54 | matched_fields = [] 55 | for field in field_list: 56 | match = lookml.get(field, 'no match') 57 | if match != 'no match': 58 | matched_fields.append(field) 59 | 60 | return matched_fields 61 | 62 | 63 | def extract_view_from_field(field_name): 64 | view_name = field_name.split('.')[0] 65 | return view_name 66 | 67 | 68 | def create_lookml_field_lookup(view_file): 69 | field_lookup = {} 70 | for lookml in view_file: 71 | parsed_lookml = parse_lookml.LookML(lookml) 72 | if parsed_lookml.has_views(): 73 | for v in parsed_lookml.views(): 74 | extract_field_names_from_lookml( 75 | parsed_lookml=v, data_storage=field_lookup) 76 | return field_lookup 77 | 78 | 79 | def create_viewname_lookup(view_file): 80 | sql_table_name_list = [] 81 | for lookml in view_file: 82 | parsed_lookml = parse_lookml.LookML(lookml) 83 | if parsed_lookml.has_views(): 84 | for v in parsed_lookml.views(): 85 | try: 86 | sql_table_name = v['sql_table_name'] 87 | sql_table_name_list.append(sql_table_name) 88 | except KeyError: 89 | logger.warning('no sql_table_name present in view') 90 | return sql_table_name_list 91 | 92 | 93 | def match_element_to_lookml(instancedata, view_file): 94 | field_lookup = create_lookml_field_lookup(view_file) 95 | viewname_lookup = create_viewname_lookup(view_file) 96 | 97 | matches = [] 98 | for dashboard in instancedata: 99 | fields_used = format_str_list(dashboard.get('query.formatted_fields')) 100 | views_in_dash = [] 101 | for field in fields_used: 102 | x = extract_view_from_field(field) 103 | if x not in views_in_dash: 104 | views_in_dash.append(x) 105 | matched_fields = match_fields_to_lookml(field_list=fields_used, 106 | lookml=field_lookup) 107 | 108 | for views in viewname_lookup: 109 | if views not in views_in_dash: 110 | views_in_dash.append(views) 111 | 112 | temp = {} 113 | temp['used_views'] = views_in_dash 114 | temp['matched_fields'] = matched_fields 115 | temp['dashboard'] = dashboard['dashboard.id'] 116 | temp['element_id'] = dashboard['dashboard_element.id'] 117 | matches.append(temp) 118 | print(matches) 119 | return matches 120 | -------------------------------------------------------------------------------- /lmanage/mapview/hugo.csv: -------------------------------------------------------------------------------- 1 | ,used_views,matched_fields 2 | 0,"['content_usage', 'public.distribution_centers', 'public.products', 'public.users', 'public.order_items', 'public.inventory_items', 'public.events', '`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']",[] 3 | 1,"['history', 'source_query_rank', 'public.distribution_centers', 'public.products', 'public.users', 'public.order_items', 'public.inventory_items', 'public.events', '`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']",[] 4 | 2,"['order_items', 'public.distribution_centers', 'public.products', 'public.users', 'public.order_items', 'public.inventory_items', 'public.events', '`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items.id', 'order_items.order_id', 'order_items.sale_price', 'order_items.average_days_to_process']" 5 | 3,"['products', 'order_items', 'public.distribution_centers', 'public.products', 'public.users', 'public.order_items', 'public.inventory_items', 'public.events', '`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['products.brand', 'order_items.gross_margin', 'order_items.id']" 6 | -------------------------------------------------------------------------------- /lmanage/mapview/instancedata.py: -------------------------------------------------------------------------------- 1 | from looker_sdk.sdk.api40 import models 2 | import json 3 | 4 | 5 | class GetCleanInstanceData(): 6 | def __init__(self, sdk): 7 | self.sdk = sdk 8 | 9 | def get_dashboards(self): 10 | """Uses the Looker SDK System__Activity model to extract dashboard 11 | and dashboard_element metadata. 12 | """ 13 | query_config = models.WriteQuery( 14 | model="system__activity", 15 | view="dashboard", 16 | fields=[ 17 | "dashboard.id", 18 | "dashboard_element.id", 19 | "dashboard_element.type", 20 | "dashboard_element.result_source", 21 | "query.formatted_pivots", 22 | "query.model", 23 | "query.view", 24 | "query.formatted_fields", 25 | "query.id", 26 | "dashboard.title", 27 | "look.id" 28 | ], 29 | filters={ 30 | "dashboard_element.type": "-text", 31 | "dashboard.deleted_date": "NULL" 32 | }, 33 | limit='5000' 34 | ) 35 | query_response = self.sdk.run_inline_query( 36 | result_format='json', 37 | body=query_config 38 | ) 39 | 40 | query_response = json.loads(query_response) 41 | 42 | return query_response 43 | 44 | def unpivot_query(self, query_id): 45 | query_metadata = self.sdk.query(query_id=query_id) 46 | 47 | query_body = models.WriteQuery( 48 | model=query_metadata.model, 49 | view=query_metadata.view, 50 | fields=query_metadata.fields, 51 | filters=query_metadata.filters 52 | ) 53 | 54 | new_query = self.sdk.create_query(body=query_body) 55 | return new_query.id 56 | 57 | def execute(self): 58 | instance_data = self.get_dashboards() 59 | clean_data = [] 60 | for db_element in instance_data: 61 | if db_element['query.formatted_pivots'] is None: 62 | clean_data.append(db_element) 63 | else: 64 | query_id = db_element['query.id'] 65 | nqid = self.unpivot_query(query_id) 66 | db_element['query.id'] = nqid 67 | clean_data.append(db_element) 68 | return clean_data 69 | -------------------------------------------------------------------------------- /lmanage/mapview/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | get dashboards 3 | find elements on dashboards 4 | get query elements of queries 5 | get model files 6 | get explores 7 | get fields 8 | 9 | get folder of lookml, or git repo 10 | match dash field to lookml files 11 | match how many times a lookml file has been referenced 12 | show table definition of matched table 13 | ''' 14 | from looker_sdk import init31 15 | import logging 16 | import coloredlogs 17 | import instancedata as idata 18 | import parse_lookml as pl 19 | import mapexplores as me 20 | import matched_field_analysis as mfa 21 | 22 | logger = logging.getLogger(__name__) 23 | coloredlogs.install(level='INFO') 24 | logging.getLogger("looker_sdk").setLevel(logging.WARNING) 25 | 26 | 27 | def main(**kwargs): 28 | div = '-----------------------------------------------------------------------------' 29 | 30 | ini_file = kwargs.get("ini_file") 31 | lookml_file_path = kwargs.get("file_path") 32 | logger.info(div) 33 | sdk = init31(config_file=ini_file) 34 | 35 | dash_to_dash_elements = idata.get_dashboards(sdk=sdk) 36 | lookml_files = pl.get_all_lkml_filepaths(starting_path=lookml_file_path) 37 | 38 | parsed_lookml_dict = pl.get_parsed_lookml(lookml_file_paths=lookml_files) 39 | 40 | elements = me.match_element_to_lookml( 41 | instancedata=dash_to_dash_elements, view_file=lookml_files) 42 | da = mfa.MatchedFieldDataAnalysis(matched_fields=elements) 43 | print(da.most_used_object(looker_object='fields')) 44 | print(da.field_to_dashboard_map(dashboard_id_filter=7)) 45 | da.data_analysis() 46 | 47 | 48 | if __name__ == "__main__": 49 | IP = ('/usr/local/google/home/hugoselbie/code_sample/py/ini/k8.ini') 50 | FP = ('/usr/local/google/home/hugoselbie/code_sample/py/mapview/test_lookml_files/the_look') 51 | 52 | main( 53 | ini_file=IP, 54 | file_path=FP) 55 | -------------------------------------------------------------------------------- /lmanage/mapview/mapexplores.py: -------------------------------------------------------------------------------- 1 | import lkml 2 | import glob 3 | import logging 4 | import coloredlogs 5 | import ast 6 | 7 | import parse_lookml 8 | 9 | logger = logging.getLogger(__name__) 10 | coloredlogs.install(level='INFO') 11 | logging.getLogger("looker_sdk").setLevel(logging.WARNING) 12 | 13 | 14 | def format_str_list(list_str): 15 | new_list = ast.literal_eval(list_str) 16 | return new_list 17 | 18 | 19 | def collate_view_files(parsed_lookml): 20 | response = [] 21 | for instance_lookml in parsed_lookml: 22 | for path in instance_lookml.keys(): 23 | if 'view.lkml' in path: 24 | response.append(instance_lookml) 25 | return response 26 | 27 | 28 | def extract_field_names_from_lookml(parsed_lookml, data_storage): 29 | 30 | try: 31 | dimensions = parsed_lookml['dimensions'] 32 | except KeyError: 33 | dimensions = [] 34 | logger.warning('no dimensions present in view') 35 | try: 36 | measures = parsed_lookml['measures'] 37 | except KeyError: 38 | measures = [] 39 | logger.warning('no measures present in view') 40 | 41 | view_name = parsed_lookml.get('name') 42 | for dim in dimensions: 43 | field_name = dim.get('name') 44 | full_name = f'{view_name}.{field_name}' 45 | data_storage[full_name] = view_name 46 | 47 | for meas in measures: 48 | if len(meas) > 0: 49 | field_name = meas.get('name') 50 | full_name = f'{view_name}.{field_name}' 51 | data_storage[full_name] = view_name 52 | 53 | return data_storage 54 | 55 | 56 | def match_fields_to_lookml(field_list, lookml): 57 | matched_fields = [] 58 | for field in field_list: 59 | match = lookml.get(field, 'no match') 60 | if match != 'no match': 61 | matched_fields.append(field) 62 | 63 | print(matched_fields) 64 | return matched_fields 65 | 66 | 67 | def extract_view_from_field(field_name): 68 | view_name = field_name.split('.')[0] 69 | return view_name 70 | 71 | 72 | def match_element_to_lookml(instancedata, view_file): 73 | field_lookup = {} 74 | for lookml in view_file: 75 | parsed_lookml = parse_lookml.LookML(lookml) 76 | if parsed_lookml.has_views(): 77 | for v in parsed_lookml.views(): 78 | extract_field_names_from_lookml( 79 | parsed_lookml=v, data_storage=field_lookup) 80 | 81 | matches = [] 82 | for dashboard in instancedata: 83 | fields_used = format_str_list(dashboard.get('query.formatted_fields')) 84 | views_in_dash = [] 85 | for field in fields_used: 86 | x = extract_view_from_field(field) 87 | if x not in views_in_dash: 88 | views_in_dash.append(x) 89 | matched_fields = match_fields_to_lookml(field_list=fields_used, 90 | lookml=field_lookup) 91 | 92 | temp = {} 93 | temp['used_views'] = views_in_dash 94 | temp['matched_fields'] = matched_fields 95 | temp['dashboard'] = dashboard.get('dashboard.id') 96 | matches.append(temp) 97 | return matches 98 | -------------------------------------------------------------------------------- /lmanage/mapview/mapview_execute.py: -------------------------------------------------------------------------------- 1 | from looker_sdk import init31 2 | import logging 3 | import coloredlogs 4 | import mapview.instancedata as idata 5 | import mapview.parse_sql_tables as pst 6 | import mapview.compare_content_to_lookml as me 7 | import mapview.matched_field_analysis as mfa 8 | from lmanage.mapview.utils import parse_lookml as pl 9 | 10 | logger = logging.getLogger(__name__) 11 | coloredlogs.install(level='INFO') 12 | logging.getLogger("looker_sdk").setLevel(logging.WARNING) 13 | 14 | 15 | def main(**kwargs): 16 | div = '-------------------------------------------------------------------' 17 | 18 | ini_file = kwargs.get("ini_file") 19 | lookml_file_path = kwargs.get("lookml_file_path") 20 | file_output = kwargs.get("output_path") 21 | logger.info(div) 22 | sdk = init31(config_file=ini_file) 23 | 24 | dashboard_metadata = idata.GetCleanInstanceData(sdk=sdk).execute() 25 | 26 | query_elements = pst.ParseSqlTables(dataextract=dashboard_metadata, 27 | sdk=sdk).get_sql_from_elements() 28 | 29 | lookml_files = pl.get_all_lkml_filepaths(starting_path=lookml_file_path) 30 | 31 | elements = me.match_element_to_lookml( 32 | instancedata=query_elements, view_file=lookml_files) 33 | da = mfa.MatchedFieldDataAnalysis(matched_fields=elements) 34 | 35 | da.export_to_excel(file_output) 36 | logger.info('i have finished') 37 | 38 | 39 | if __name__ == "__main__": 40 | IP = ('/usr/local/google/home/hugoselbie/code_sample/py/ini/k8.ini') 41 | FP = ('/usr/local/google/home/hugoselbie/code_sample/py/mapview/test_lookml_files/the_look') 42 | 43 | main( 44 | ini_file=IP, 45 | file_path=FP) 46 | -------------------------------------------------------------------------------- /lmanage/mapview/matched_field_analysis.py: -------------------------------------------------------------------------------- 1 | from lmanage.attr import field 2 | import lkml 3 | import glob 4 | import logging 5 | import coloredlogs 6 | import ast 7 | import pandas as pd 8 | import xlsxwriter 9 | 10 | logger = logging.getLogger(__name__) 11 | coloredlogs.install(level='INFO') 12 | logging.getLogger("looker_sdk").setLevel(logging.WARNING) 13 | 14 | 15 | class MatchedFieldDataAnalysis(): 16 | def __init__(self, matched_fields): 17 | self.matched_data = matched_fields 18 | self.df = pd.DataFrame(self.matched_data) 19 | 20 | def create_df(self): 21 | df = pd.DataFrame(self.matched_data) 22 | return df 23 | 24 | def most_used_object(self, looker_object): 25 | if looker_object == 'views': 26 | xploded = self.df.explode('used_views') 27 | xploded = xploded.groupby(['used_views'])['dashboard'].count() 28 | return xploded 29 | elif looker_object == 'fields': 30 | xploded = self.df.explode('matched_fields') 31 | xploded = xploded.groupby(['matched_fields'])['dashboard'].count() 32 | xploded = xploded.groupby(['used_views'])[ 33 | 'dashboard'].count().rename('dashboard_count').sort_values(ascending=False) 34 | return xploded 35 | 36 | def field_to_dashboard_map(self, field_filter=None, dashboard_id_filter=None): 37 | xploded = self.df.explode('matched_fields') 38 | if field_filter: 39 | xploded = xploded.filter(like=field_filter, axis=0) 40 | if dashboard_id_filter: 41 | xploded = xploded.filter(like=dashboard_id_filter) 42 | return xploded 43 | 44 | def data_analysis(self): 45 | df = self.create_df() 46 | print(df) 47 | 48 | def export_to_excel(self, output): 49 | writer = pd.ExcelWriter(output, engine='xlsxwriter') 50 | dashboard_match = self.field_to_dashboard_map() 51 | used_views = self.most_used_object(looker_object='views') 52 | used_obj = self.most_used_object(looker_object='fields') 53 | 54 | # Write each dataframe to a different worksheet. 55 | dashboard_match.to_excel(writer, sheet_name='DashboardToLookMLMapping') 56 | used_obj.to_excel(writer, sheet_name='MostUsedDimensions') 57 | used_views.to_excel(writer, sheet_name='MostUsedViews') 58 | 59 | # Close the Pandas Excel writer and output the Excel file. 60 | writer.save() 61 | -------------------------------------------------------------------------------- /lmanage/mapview/pandas_multiple.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/mapview/pandas_multiple.xlsx -------------------------------------------------------------------------------- /lmanage/mapview/parse_sql_tables.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import coloredlogs 3 | from lmanage.mapview.utils import parsing_sql 4 | from looker_sdk import error 5 | 6 | 7 | logger = logging.getLogger(__name__) 8 | coloredlogs.install(level='DEBUG') 9 | logging.getLogger("looker_sdk").setLevel(logging.WARNING) 10 | 11 | 12 | class ParseSqlTables(): 13 | def __init__(self, dataextract, sdk): 14 | self.dataextract = dataextract 15 | self.sdk = sdk 16 | 17 | def parse_sql(self, qid: int): 18 | """ 19 | Idenfies the base tables and joins used by a Looker query. 20 | 21 | Iterates over a list of PyLookML files identifies view files aand returns a 22 | list of the sql_table_names if they exist in an object of type 'View' 23 | qid: (int) query_id from lmanage.a Looker query 24 | Returns: 25 | A list of all the tables that are found from lmanage.a Looker generated 26 | SQL query. 27 | For example: 28 | ['public.order_items','public.inventory_items','public.events'] 29 | Exception: 30 | If a query is broken for whatever reason the Exception is caught to 31 | continue the program running 32 | """ 33 | try: 34 | sql_response = self.sdk.run_query( 35 | query_id=qid, result_format="sql") 36 | if isinstance(sql_response, str): 37 | tables = parsing_sql.extract_tables(sql_response) 38 | return tables 39 | else: 40 | return sql_response 41 | except error.SDKError: 42 | return('No Content') 43 | 44 | def get_sql_from_elements(self): 45 | """Amends returned SDK System__Activity reponse with sql tables used 46 | from lmanage.the `parse_sql` function. 47 | 48 | Iterates over the response from lmanage.get_dashboards and runs the parse_sql 49 | function for each returned dashboard element, returns the list of tables 50 | and amends the dict response and returns it 51 | Args: 52 | sdk: Looker SDK object 53 | content_results: (dict) response from lmanage.get_dashboards function call 54 | Returns: 55 | An amended dict response with the sql columns used by each element 56 | extracted our of the Looker generated SQL for each dashboard object. 57 | For example: 58 | [{'dashboard.id': 1, 59 | 'dashboard_element.id': 1, 60 | 'dashboard_element.type': 'vis', 61 | 'dashboard_element.result_source': 'Lookless', 62 | 'query.model': 'bq', 63 | 'query.view': 'order_items', 64 | 'query.formatted_fields': '["order_items.count"]', 65 | 'query.id': 59, 66 | 'dashboard.title': 'dash_1', 67 | 'look.id': None, 68 | 'sql_joins': ['`looker-private-demo.ecomm.order_items`']}] 69 | """ 70 | for dash in self.dataextract: 71 | query_id = dash['query.id'] 72 | sql_value = self.parse_sql(query_id) 73 | 74 | dash['sql_joins'] = sql_value 75 | 76 | return self.dataextract 77 | -------------------------------------------------------------------------------- /lmanage/mapview/utils/.vimspector.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": { 3 | "run": { 4 | "adapter": "debugpy", 5 | "configuration": { 6 | "request": "launch", 7 | "protocol": "auto", 8 | "stopOnEntry": true, 9 | "console": "integratedTerminal", 10 | "program": "${workspaceRoot}/get_groups.py", 11 | "cwd": "${workspaceRoot}" 12 | } 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lmanage/mapview/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /lmanage/mapview/utils/create_df.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2021 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import os 17 | import logging 18 | from lmanage.pathlib import Path 19 | import json 20 | import pandas as pd 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | def create_df(data): 26 | logger.debug(type(data)) 27 | data = json.loads(data) if isinstance(data, str) else data 28 | df = pd.DataFrame(data) 29 | return df 30 | 31 | 32 | def check_ini(ini): 33 | cwd = Path.cwd() 34 | 35 | if ini: 36 | parsed_ini_file = cwd.joinpath(ini) 37 | else: 38 | parsed_ini_file = None 39 | 40 | if os.path.isfile(parsed_ini_file): 41 | logger.info(f'opening your ini file at {parsed_ini_file}') 42 | return parsed_ini_file 43 | else: 44 | logger.debug( 45 | f'the path to your ini file is not valid at {parsed_ini_file}') 46 | -------------------------------------------------------------------------------- /lmanage/mapview/utils/parse_lookml.py: -------------------------------------------------------------------------------- 1 | import os 2 | import lkml 3 | import glob 4 | import logging 5 | import coloredlogs 6 | 7 | logger = logging.getLogger(__name__) 8 | coloredlogs.install(level='INFO') 9 | logging.getLogger("looker_sdk").setLevel(logging.WARNING) 10 | 11 | 12 | def get_all_lkml_filepaths(starting_path): 13 | logger.info(starting_path) 14 | lookml_files = [f for f in glob.glob( 15 | starting_path + "/**/*" + 'lkml', recursive=True)] 16 | logger.debug(lookml_files) 17 | return lookml_files 18 | 19 | 20 | def parse_lookml(path): 21 | with open(path, 'r') as lookml_file: 22 | lookml = lkml.load(lookml_file) 23 | return lookml 24 | 25 | 26 | def get_parsed_lookml(lookml_file_paths): 27 | response = [] 28 | for file in lookml_file_paths: 29 | parsed_lookml = parse_lookml(file) 30 | logger.debug(parsed_lookml) 31 | temp = {} 32 | temp[file] = parsed_lookml 33 | response.append(temp) 34 | return response 35 | 36 | 37 | ''' 38 | parse a LookML file from lmanage.LookML to JSON 39 | Authors: 40 | Carl Anderson (carl.anderson@weightwatchers.com) 41 | ''' 42 | 43 | 44 | class LookML(): 45 | 46 | def __init__(self, infilepath): 47 | '''parse the LookML infilepath, convert to JSON, and then read into JSON object 48 | Args: 49 | infilepath (str): path to input LookML file 50 | Returns: 51 | JSON object of LookML 52 | ''' 53 | if not os.path.exists(infilepath): 54 | raise IOError("Filename does not exist: %s" % infilepath) 55 | 56 | self.infilepath = infilepath 57 | if infilepath.endswith(".model.lkml"): 58 | self.filetype = 'model' 59 | elif infilepath.endswith(".view.lkml"): 60 | self.filetype = 'view' 61 | elif infilepath.endswith(".explore.lkml"): 62 | self.filetype = 'explore' 63 | elif infilepath.endswith(".lkml"): 64 | self.filetype = 'generic' 65 | 66 | else: 67 | raise Exception("Unsupported filename " + infilepath) 68 | self.base_filename = os.path.basename(infilepath) 69 | self.base_name = self.base_filename.replace(".model.lkml", "").replace( 70 | ".explore.lkml", "").replace(".view.lkml", "") 71 | 72 | with open(infilepath, 'r') as file: 73 | self.json_data = lkml.load(file) 74 | 75 | def views(self): 76 | """get views (if any) from lmanage.the LookML 77 | Returns: 78 | views (list) if any, None otherwise 79 | """ 80 | if 'views' in self.json_data: 81 | return self.json_data['views'] 82 | return None 83 | 84 | def has_views(self): 85 | """does this have one or more views? 86 | Returns: 87 | bool, whether this has views 88 | """ 89 | vs = self.views() 90 | return (vs and len(vs) > 0) 91 | 92 | def explores(self): 93 | """get explores (if any) from lmanage.the LookML 94 | Returns: 95 | explores (list) if any, None otherwise 96 | """ 97 | if 'explores' in self.json_data: 98 | return self.json_data['explores'] 99 | return None 100 | 101 | def has_explores(self): 102 | """does this have one or more explores? 103 | Returns: 104 | bool, whether this has explores 105 | """ 106 | es = self.explores() 107 | return (es and len(es) > 0) 108 | -------------------------------------------------------------------------------- /lmanage/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /lmanage/utils/errorhandling.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import logging 3 | from looker_sdk import error 4 | from time import sleep 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | def calc_done_percent(iterator: int, total: int) -> str: 10 | percent = (iterator/total) * 100 11 | return f'{int(percent)} %' 12 | 13 | 14 | def return_error_message(input: str): 15 | # err_response = input.args[0] 16 | # err_response = json.loads(err_response) 17 | if len(input.errors) == 0: 18 | err_response = input.message 19 | else: 20 | err_response = input.errors[0].message 21 | 22 | return err_response 23 | 24 | 25 | def test_object_data(object: list) -> bool: 26 | if object: 27 | return True 28 | else: 29 | return False 30 | 31 | 32 | def user_authentication_test(sdk) -> bool: 33 | try: 34 | sdk.me() 35 | return True 36 | except error.SDKError(): 37 | return False 38 | 39 | 40 | def return_sleep_message(call_number=0, quiet=False): 41 | call_number = call_number+1 42 | sleep_number = 2 ** call_number 43 | sleep(sleep_number) 44 | if not quiet: 45 | logger.warn(f'Looker is overwhelmed sleeping for {sleep_number} secs') 46 | 47 | 48 | def dedup_list_of_dicts(input: list): 49 | stringinput = list(set([str(x) for x in input])) 50 | deduped = [ast.literal_eval(x) for x in stringinput] 51 | return deduped 52 | 53 | 54 | def counter(i=0): 55 | i += 1 56 | return i 57 | -------------------------------------------------------------------------------- /lmanage/utils/helpers.py: -------------------------------------------------------------------------------- 1 | def xstr(value): 2 | return value if value is not None else '' 3 | 4 | 5 | def nstr(value): 6 | return None if value == '' else value 7 | -------------------------------------------------------------------------------- /lmanage/utils/logger_creation.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from rich.logging import RichHandler 3 | import datetime 4 | import os 5 | 6 | 7 | def create_timestamp(): 8 | timestamp = datetime.datetime.now() 9 | unix_ts = timestamp.timestamp() 10 | trunc_unix_ts = unix_ts - (unix_ts % 60) 11 | return int(trunc_unix_ts) 12 | 13 | 14 | def init_logger(dunder_name, testing_mode) -> logging.Logger: 15 | log_format = ( 16 | '%(asctime)s - ' 17 | '%(name)s - ' 18 | '%(funcName)s - ' 19 | '%(levelname)s - ' 20 | '%(message)s' 21 | ) 22 | 23 | logging.basicConfig( 24 | format=log_format, 25 | datefmt=".", 26 | handlers=[RichHandler()], 27 | ) 28 | logger = logging.getLogger(dunder_name) 29 | 30 | if testing_mode: 31 | logger.setLevel(logging.DEBUG) 32 | else: 33 | logger.setLevel(logging.INFO) 34 | 35 | unix_ts = create_timestamp() 36 | 37 | # Create logs folder if not exists 38 | os.makedirs('logs', exist_ok=True) 39 | 40 | # Output full log 41 | fh = logging.FileHandler(f'logs/lomt_{unix_ts}.log') 42 | fh.setLevel(logging.DEBUG) 43 | formatter = logging.Formatter(log_format) 44 | fh.setFormatter(formatter) 45 | logger.addHandler(fh) 46 | 47 | return logger 48 | -------------------------------------------------------------------------------- /lmanage/utils/parse_yaml.py: -------------------------------------------------------------------------------- 1 | import json 2 | import ruamel.yaml as yaml 3 | import logging 4 | from lmanage.utils import logger_creation as log_color 5 | 6 | 7 | class Yaml: 8 | def __init__(self, yaml_path): 9 | self.yaml_path = yaml_path 10 | with open(self.yaml_path, 'r') as file: 11 | self.parsed_yaml = yaml.load(file, Loader=yaml.BaseLoader) 12 | 13 | def get_metadata(self, type): 14 | if type == 'folder_metadata': 15 | metadata = self.__get_folder_metadata() 16 | elif type == 'permission_set_metadata': 17 | metadata = self.__get_permission_metadata() 18 | elif type == 'model_set_metadata': 19 | metadata = self.__get_model_set_metadata() 20 | elif type == 'role_metadata': 21 | metadata = self.__get_role_metadata() 22 | elif type == 'user_attribute_metadata': 23 | metadata = self.__get_user_attribute_metadata() 24 | elif type == 'look_metadata': 25 | metadata = self.__get_look_metadata() 26 | elif type == 'dashboard_metadata': 27 | metadata = self.__get_dashboard_metadata() 28 | elif type == 'board_metadata': 29 | metadata = self.__get_board_metadata() 30 | 31 | if not metadata: 32 | print('') 33 | # log f'No {type} specified. Please check your yaml file at {self.yaml_path}.') 34 | 35 | return metadata 36 | 37 | def __get_permission_metadata(self): 38 | response = [] 39 | for objects in self.parsed_yaml: 40 | if 'permissions' in list(objects.keys()): 41 | response.append(objects) 42 | response = [r for r in response if r.get('name') != 'Admin'] 43 | return response 44 | 45 | def __get_model_set_metadata(self): 46 | response = [] 47 | for objects in self.parsed_yaml: 48 | if 'models' in list(objects.keys()): 49 | response.append(objects) 50 | return response 51 | 52 | def __get_role_metadata(self): 53 | response = [] 54 | for objects in self.parsed_yaml: 55 | if 'permission_set' and 'model_set' in list(objects.keys()): 56 | response.append(objects) 57 | response = [r for r in response if r.get('name') != 'Admin'] 58 | return response 59 | 60 | def __get_folder_metadata(self): 61 | response = [] 62 | for objects in self.parsed_yaml: 63 | if 'content_metadata_id' in list(objects.keys()): 64 | response.append(objects) 65 | return response 66 | 67 | def __get_user_attribute_metadata(self): 68 | response = [] 69 | for objects in self.parsed_yaml: 70 | if 'hidden_value' in list(objects.keys()): 71 | response.append(objects) 72 | return response 73 | 74 | def __get_look_metadata(self): 75 | response = [] 76 | for objects in self.parsed_yaml: 77 | if 'query_obj' in list(objects.keys()): 78 | response.append(objects) 79 | return response 80 | 81 | def __get_dashboard_metadata(self): 82 | response = [] 83 | for objects in self.parsed_yaml: 84 | if 'lookml' in list(objects.keys()): 85 | response.append(objects) 86 | return response 87 | 88 | def __get_board_metadata(self): 89 | response = [] 90 | for objects in self.parsed_yaml: 91 | if 'board_sections' in list(objects.keys()): 92 | response.append(objects) 93 | return response 94 | -------------------------------------------------------------------------------- /output/hugo.csv: -------------------------------------------------------------------------------- 1 | ,dashboard_id,element_id,sql_joins,fields_used,sql_table_name,potential_join,used_joins,used_view_names,unused_joins 2 | 0,1,1,['`looker-private-demo.ecomm.order_items`'],"[""order_items.created_month"", ""order_items.count""]","['`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items', 'order_facts', 'inventory_items', 'users', 'user_order_facts', 'products', 'repeat_purchase_facts', 'distribution_centers', 'test_ndt']",['`looker-private-demo.ecomm.order_items`'],['order_items'],"['distribution_centers', 'inventory_items', 'order_facts', 'products', 'repeat_purchase_facts', 'test_ndt', 'user_order_facts', 'users']" 3 | 1,1,2,"['`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.products`']","[""inventory_items.product_distribution_center_id"", ""products.department"", ""products.count""]","['`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items', 'order_facts', 'inventory_items', 'users', 'user_order_facts', 'products', 'repeat_purchase_facts', 'distribution_centers', 'test_ndt']","['`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.products`']","['products', 'order_items', 'inventory_items']","['distribution_centers', 'order_facts', 'repeat_purchase_facts', 'test_ndt', 'user_order_facts', 'users']" 4 | 2,1,3,"['`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.products`']","[""products.department"", ""products.count"", ""inventory_items.count"", ""users.city"", ""users.age""]","['`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items', 'order_facts', 'inventory_items', 'users', 'user_order_facts', 'products', 'repeat_purchase_facts', 'distribution_centers', 'test_ndt']","['`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.products`']","['products', 'users', 'order_items', 'inventory_items']","['distribution_centers', 'order_facts', 'repeat_purchase_facts', 'test_ndt', 'user_order_facts']" 5 | 3,1,5,"['`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', 'test_ndt', '`looker-private-demo.ecomm.order_items`']","[""order_items.total_revenue"", ""order_items.created_month"", ""order_items.user_id"", ""test_ndt.total_revenue"", ""test_ndt.user_id"", ""products.brand""]","['`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items', 'order_facts', 'inventory_items', 'users', 'user_order_facts', 'products', 'repeat_purchase_facts', 'distribution_centers', 'test_ndt']","['`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', 'test_ndt', '`looker-private-demo.ecomm.order_items`']","['test_ndt', 'products', 'users', 'order_items', 'inventory_items']","['distribution_centers', 'order_facts', 'repeat_purchase_facts', 'user_order_facts']" 6 | -------------------------------------------------------------------------------- /output/instance_mapview.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/output/instance_mapview.xlsx -------------------------------------------------------------------------------- /output/l233_output.yaml: -------------------------------------------------------------------------------- 1 | # FOLDER_PERMISSIONS 2 | # Opening Session Welcome to the Capturator, this is the Folder place 3 | # ----------------------------------------------------- 4 | 5 | - !LookerFolder 6 | parent_id: 7 | id: '1' 8 | name: Shared 9 | subfolder: 10 | - !LookerFolder 11 | parent_id: '1' 12 | id: '83' 13 | name: monkey 14 | subfolder: 15 | - !LookerFolder 16 | parent_id: '83' 17 | id: '84' 18 | name: monkey_sub 19 | subfolder: 20 | - !LookerFolder 21 | parent_id: '84' 22 | id: '85' 23 | name: monkey_sub_sub 24 | subfolder: [] 25 | content_metadata_id: '100' 26 | team_edit: [] 27 | team_view: [] 28 | content_metadata_id: '99' 29 | team_edit: [] 30 | team_view: [] 31 | content_metadata_id: '98' 32 | team_edit: [] 33 | team_view: [] 34 | - !LookerFolder 35 | parent_id: '1' 36 | id: '81' 37 | name: test 38 | subfolder: 39 | - !LookerFolder 40 | parent_id: '81' 41 | id: '82' 42 | name: blok 43 | subfolder: [] 44 | content_metadata_id: '97' 45 | team_edit: [] 46 | team_view: [] 47 | content_metadata_id: '96' 48 | team_edit: [] 49 | team_view: [] 50 | content_metadata_id: '1' 51 | team_edit: 52 | - All Users 53 | team_view: [] 54 | # Looker Role 55 | # Opening Session Welcome to the Capturator, this is the Role place 56 | # ----------------------------------------------------- 57 | 58 | 59 | 60 | # PERMISSION SETS 61 | - !LookerPermissionSet 62 | permissions: 63 | - access_data 64 | - clear_cache_refresh 65 | - download_without_limit 66 | - mobile_app_access 67 | - schedule_look_emails 68 | - see_drill_overlay 69 | - see_lookml_dashboards 70 | - see_looks 71 | - see_user_dashboards 72 | - send_to_integration 73 | name: Viewer 74 | - !LookerPermissionSet 75 | permissions: 76 | - access_data 77 | - can_create_forecast 78 | - clear_cache_refresh 79 | - create_custom_fields 80 | - create_table_calculations 81 | - download_without_limit 82 | - explore 83 | - manage_spaces 84 | - mobile_app_access 85 | - save_content 86 | - schedule_look_emails 87 | - see_drill_overlay 88 | - see_lookml 89 | - see_lookml_dashboards 90 | - see_looks 91 | - see_sql 92 | - see_user_dashboards 93 | - send_to_integration 94 | name: User 95 | - !LookerPermissionSet 96 | permissions: 97 | - access_data 98 | - can_create_forecast 99 | - clear_cache_refresh 100 | - create_custom_fields 101 | - create_table_calculations 102 | - deploy 103 | - develop 104 | - download_without_limit 105 | - explore 106 | - manage_spaces 107 | - mobile_app_access 108 | - save_content 109 | - schedule_look_emails 110 | - see_drill_overlay 111 | - see_lookml 112 | - see_lookml_dashboards 113 | - see_looks 114 | - see_pdts 115 | - see_sql 116 | - see_user_dashboards 117 | - send_to_integration 118 | - use_sql_runner 119 | name: Developer 120 | 121 | 122 | # MODEL SETS 123 | - !LookerModelSet 124 | models: 125 | - naga_234_test 126 | - lookml-diagram 127 | - large_project 128 | - data_block_acs_bigquery 129 | - extension-api-explorer 130 | - cloud_logging 131 | name: All 132 | 133 | 134 | # LOOKER ROLES 135 | - !LookerRoles 136 | permission_set: User 137 | model_set: All 138 | teams: 139 | - User 140 | name: User 141 | - !LookerRoles 142 | permission_set: Developer 143 | model_set: All 144 | teams: [] 145 | name: Developer 146 | - !LookerRoles 147 | permission_set: Viewer 148 | model_set: All 149 | teams: [] 150 | name: Viewer 151 | - !LookerRoles 152 | permission_set: Developer 153 | model_set: All 154 | teams: [] 155 | name: haylen_mtr 156 | 157 | 158 | # USER_ATTRIBUTES 159 | - !LookerUserAttribute 160 | name: first_name 161 | uatype: string 162 | hidden_value: false 163 | user_view: 'True' 164 | user_edit: 'True' 165 | default_value: '' 166 | teams: [] 167 | - !LookerUserAttribute 168 | name: id 169 | uatype: number 170 | hidden_value: false 171 | user_view: 'True' 172 | user_edit: 'False' 173 | default_value: '' 174 | teams: [] 175 | - !LookerUserAttribute 176 | name: landing_page 177 | uatype: relative_url 178 | hidden_value: false 179 | user_view: 'True' 180 | user_edit: 'True' 181 | default_value: /browse 182 | teams: [] 183 | - !LookerUserAttribute 184 | name: locale 185 | uatype: string 186 | hidden_value: false 187 | user_view: 'True' 188 | user_edit: 'False' 189 | default_value: '' 190 | teams: [] 191 | - !LookerUserAttribute 192 | name: number_format 193 | uatype: string 194 | hidden_value: false 195 | user_view: 'True' 196 | user_edit: 'False' 197 | default_value: '' 198 | teams: [] 199 | - !LookerUserAttribute 200 | name: rajk 201 | uatype: string 202 | hidden_value: false 203 | user_view: 'True' 204 | user_edit: 'True' 205 | default_value: '' 206 | teams: [] 207 | -------------------------------------------------------------------------------- /output/l233_output_content.yaml: -------------------------------------------------------------------------------- 1 | 2 | 3 | # LookData 4 | - !LookObject 5 | legacy_folder_id: '81' 6 | look_id: '6' 7 | title: Total Test 8 | query_obj: 9 | model: system__activity 10 | view: event 11 | fields: 12 | - event.created_date 13 | - event.count 14 | pivots: 15 | fill_fields: 16 | - event.created_date 17 | filters: 18 | filter_expression: 19 | sorts: 20 | - event.created_date desc 21 | limit: '500' 22 | column_limit: '50' 23 | total: true 24 | row_total: 25 | subtotals: 26 | vis_config: 27 | show_view_names: false 28 | show_row_numbers: true 29 | transpose: false 30 | truncate_text: true 31 | hide_totals: false 32 | hide_row_totals: false 33 | size_to_fit: true 34 | table_theme: white 35 | limit_displayed_rows: false 36 | enable_conditional_formatting: false 37 | header_text_alignment: left 38 | header_font_size: 12 39 | rows_font_size: 12 40 | conditional_formatting_include_totals: false 41 | conditional_formatting_include_nulls: false 42 | hidden_pivots: {} 43 | type: looker_grid 44 | defaults_version: 1 45 | filter_config: {} 46 | visible_ui_sections: 47 | dynamic_fields: '[{"category":"table_calculation","expression":" sum(${event.count})\n","label":"event 48 | count","value_format":null,"value_format_name":null,"_kind_hint":"measure","table_calculation":"event_count","_type_hint":"number"}]' 49 | query_timezone: 50 | description: '' 51 | - !LookObject 52 | legacy_folder_id: '83' 53 | look_id: '7' 54 | title: look 55 | query_obj: 56 | model: system__activity 57 | view: look 58 | fields: 59 | - look.id 60 | - look.moved_to_trash 61 | - look.link 62 | - history.id 63 | - look.title 64 | - look.count 65 | pivots: 66 | fill_fields: 67 | filters: 68 | filter_expression: 69 | sorts: 70 | - look.count desc 0 71 | limit: '500' 72 | column_limit: 73 | total: 74 | row_total: 75 | subtotals: 76 | vis_config: 77 | show_view_names: false 78 | show_row_numbers: true 79 | transpose: false 80 | truncate_text: true 81 | hide_totals: false 82 | hide_row_totals: false 83 | size_to_fit: true 84 | table_theme: white 85 | limit_displayed_rows: false 86 | enable_conditional_formatting: false 87 | header_text_alignment: left 88 | header_font_size: 12 89 | rows_font_size: 12 90 | conditional_formatting_include_totals: false 91 | conditional_formatting_include_nulls: false 92 | type: looker_grid 93 | defaults_version: 1 94 | filter_config: {} 95 | visible_ui_sections: 96 | dynamic_fields: 97 | query_timezone: 98 | description: '' 99 | - !LookObject 100 | legacy_folder_id: '85' 101 | look_id: '8' 102 | title: look 103 | query_obj: 104 | model: system__activity 105 | view: look 106 | fields: 107 | - look.id 108 | - look.moved_to_trash 109 | - look.link 110 | - history.id 111 | - look.title 112 | - look.count 113 | pivots: 114 | fill_fields: 115 | filters: 116 | filter_expression: 117 | sorts: 118 | - look.count desc 0 119 | limit: '500' 120 | column_limit: 121 | total: 122 | row_total: 123 | subtotals: 124 | vis_config: 125 | show_view_names: false 126 | show_row_numbers: true 127 | transpose: false 128 | truncate_text: true 129 | hide_totals: false 130 | hide_row_totals: false 131 | size_to_fit: true 132 | table_theme: white 133 | limit_displayed_rows: false 134 | enable_conditional_formatting: false 135 | header_text_alignment: left 136 | header_font_size: 12 137 | rows_font_size: 12 138 | conditional_formatting_include_totals: false 139 | conditional_formatting_include_nulls: false 140 | type: looker_grid 141 | defaults_version: 1 142 | filter_config: {} 143 | visible_ui_sections: 144 | dynamic_fields: 145 | query_timezone: 146 | description: '' 147 | 148 | 149 | # Dashboard Content 150 | - !DashboardObject 151 | legacy_folder_id: '81' 152 | lookml: "- dashboard: new_dashboard\n title: New Dashboard\n layout: newspaper\n\ 153 | \ preferred_viewer: dashboards-next\n description: ''\n preferred_slug: SkRfx99hbJYDPBuNXZRsdi\n\ 154 | \ elements:\n - title: Untitled\n name: Untitled\n model: system__activity\n\ 155 | \ explore: field_usage\n type: table\n fields: [field_usage.field]\n\ 156 | \ limit: 500\n row:\n col:\n width:\n height:\n" 157 | dashboard_id: '7' 158 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "lmanage" 3 | version = "0.3.36" 4 | description = "LManage is a collection of useful tools for Looker admins to help curate and cleanup content and it's associated source LookML." 5 | keywords = ["keyword", "another_keyword"] 6 | authors = ["hselbie "] 7 | readme = "README.md" 8 | homepage = "https://github.com/looker-open-source/lmanage" 9 | repository = "https://github.com/looker-open-source/lmanage" 10 | 11 | [tool.poetry.dependencies] 12 | python = ">=3.8,<3.12" 13 | looker-sdk = "^23.8.1" 14 | poetry = "^1.4.2" 15 | rich = "^13.4.2" 16 | click = "^8.1.3" 17 | ruamel-yaml = "^0.17.32" 18 | tqdm = "^4.65.0" 19 | tenacity = "^8.2.2" 20 | yaspin = "^2.3.0" 21 | 22 | [tool.poetry.dev-dependencies] 23 | 24 | [build-system] 25 | requires = ["poetry-core>=1.0.0", "setuptools", "wheel", "Cython"] 26 | build-backend = "poetry.core.masonry.api" 27 | 28 | [tool.poetry.scripts] 29 | lmanage = 'lmanage.cli:lmanage' 30 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __version__ = '0.1.4' 16 | -------------------------------------------------------------------------------- /tests/example_yamls/dev_output.yaml: -------------------------------------------------------------------------------- 1 | # FOLDER_PERMISSIONS 2 | # Opening Session Welcome to the Capturator, this is the Folder place 3 | # ----------------------------------------------------- 4 | 5 | - !LookerFolder 6 | parent_id: 7 | id: '1' 8 | name: Shared 9 | subfolder: 10 | - !LookerFolder 11 | parent_id: '1' 12 | id: '202' 13 | name: test 14 | subfolder: 15 | - !LookerFolder 16 | parent_id: '202' 17 | id: '203' 18 | name: emea_cosmo 19 | subfolder: [] 20 | content_metadata_id: '205' 21 | team_edit: [] 22 | team_view: 23 | - new york team 24 | - Freddy 25 | - HugoTesty 26 | content_metadata_id: '204' 27 | team_edit: [] 28 | team_view: [] 29 | - !LookerFolder 30 | parent_id: '1' 31 | id: '201' 32 | name: emea_cosmo 33 | subfolder: 34 | - !LookerFolder 35 | name: hugo_test 36 | subfolder: [] 37 | content_metadata_id: '205' 38 | team_edit: [] 39 | team_view: 40 | - new york team 41 | - Freddy 42 | - HugoTesty 43 | content_metadata_id: '204' 44 | team_edit: [] 45 | team_view: [] 46 | content_metadata_id: '203' 47 | team_edit: [] 48 | team_view: 49 | - BusinessOperations 50 | - emea 51 | content_metadata_id: '1' 52 | team_edit: [] 53 | team_view: 54 | - All Users 55 | # Looker Role 56 | # Opening Session Welcome to the Capturator, this is the Role place 57 | # ----------------------------------------------------- 58 | 59 | 60 | 61 | # PERMISSION SETS 62 | - !LookerPermissionSet 63 | permissions: 64 | - access_data 65 | - download_with_limit 66 | - explore 67 | - schedule_look_emails 68 | - see_looks 69 | name: gghhhh 70 | - !LookerPermissionSet 71 | permissions: 72 | - access_data 73 | - see_looks 74 | - see_user_dashboards 75 | name: test 76 | - !LookerPermissionSet 77 | permissions: 78 | - access_data 79 | - clear_cache_refresh 80 | - create_alerts 81 | - create_prefetches 82 | - create_public_looks 83 | - create_table_calculations 84 | - deploy 85 | - develop 86 | - download_with_limit 87 | - download_without_limit 88 | - explore 89 | - follow_alerts 90 | - manage_homepage 91 | - manage_models 92 | - manage_spaces 93 | - manage_stereo 94 | - save_content 95 | - schedule_external_look_emails 96 | - schedule_look_emails 97 | - see_drill_overlay 98 | - see_lookml 99 | - see_lookml_dashboards 100 | - see_looks 101 | - see_sql 102 | - see_user_dashboards 103 | - send_outgoing_webhook 104 | - send_to_integration 105 | - send_to_s3 106 | - send_to_sftp 107 | - support_access_toggle 108 | - use_sql_runner 109 | name: addfa 110 | 111 | 112 | # MODEL SETS 113 | - !LookerModelSet 114 | models: 115 | - extension-api-explorer 116 | name: All 117 | 118 | 119 | # LOOKER ROLES 120 | - !LookerRoles 121 | permission_set: test 122 | model_set: All 123 | teams: 124 | - BusinessOperations 125 | - Freddy 126 | - HugoTesty 127 | name: Operations_User 128 | - !LookerRoles 129 | permission_set: addfa 130 | model_set: All 131 | teams: 132 | - westlake 133 | - DrewEdit 134 | - BusinessOperations_BO_Dev 135 | name: BusinessOperations_Developer 136 | - !LookerRoles 137 | permission_set: gghhhh 138 | model_set: All 139 | teams: 140 | - BusinessOperations 141 | name: BusinessOperations_User 142 | 143 | 144 | # USER_ATTRIBUTES 145 | - !LookerUserAttribute 146 | name: city 147 | uatype: string 148 | hidden_value: false 149 | user_view: 'True' 150 | user_edit: 'False' 151 | default_value: '%, NULL' 152 | teams: 153 | - london team: London 154 | - new york team: New York 155 | - safetyfirst: least_privilege 156 | - !LookerUserAttribute 157 | name: first_name 158 | uatype: string 159 | hidden_value: false 160 | user_view: 'True' 161 | user_edit: 'True' 162 | default_value: '' 163 | teams: [] 164 | - !LookerUserAttribute 165 | name: landing_page 166 | uatype: relative_url 167 | hidden_value: false 168 | user_view: 'True' 169 | user_edit: 'True' 170 | default_value: /browse 171 | teams: [] 172 | - !LookerUserAttribute 173 | name: locale 174 | uatype: string 175 | hidden_value: false 176 | user_view: 'True' 177 | user_edit: 'False' 178 | default_value: en 179 | teams: [] 180 | - !LookerUserAttribute 181 | name: number_format 182 | uatype: string 183 | hidden_value: false 184 | user_view: 'True' 185 | user_edit: 'False' 186 | default_value: 1,234.56 187 | teams: [] 188 | -------------------------------------------------------------------------------- /tests/example_yamls/folder_permission_input.yaml: -------------------------------------------------------------------------------- 1 | BusinessOperations_folder_permissions: 2 | folder: 3 | name: BusinessOperations 4 | team_edit: BusinessOperations 5 | team_view: 6 | - Product 7 | - EngineeringSecurity 8 | - Friends_Maps 9 | Cameos_folder_permissions: 10 | folder: 11 | name: Cameos 12 | team_edit: Cameos 13 | team_view: 14 | - Product 15 | - InformationTechnology 16 | - DataScience 17 | - ProductManagement 18 | lk2bitmoji_folder_permissions: 19 | folder: 20 | name: lk2bitmoji 21 | team_edit: lk2bitmoji 22 | team_view: 23 | - Product 24 | - InformationTechnology 25 | - Strategy 26 | 27 | -------------------------------------------------------------------------------- /tests/example_yamls/folder_yaml.yaml: -------------------------------------------------------------------------------- 1 | # We implement three categories of Looker user groups: 2 | # 3 | # 1. Roles: these groups are assigned Looker Roles, which control what 4 | # functionality a user has access to in Looker. 5 | # 2. Content: these groups control access to Folders, which organize 6 | # content (dashboards and Looks) in Looker. 7 | # 3. Attributes: these groups are assigned attributes, which are used to 8 | # control access to rows and columns of data via access filters and 9 | # access frants. 10 | 11 | ######### 12 | # MODEL_SET_ROLES # 13 | ######### 14 | role_admin: 15 | role: Admin 16 | team: 17 | - Cameos 18 | 19 | role_BusinessOperations_Developer: 20 | role: BODevelopers 21 | permissions: 22 | - access_data 23 | - can_see_system_activity 24 | - see_dashboards 25 | - clear_cache_refresh 26 | - create_table_calculations 27 | - deploy 28 | - develop 29 | - download_without_limit 30 | - explore 31 | - manage_spaces 32 | - mobile_app_access 33 | - save_content 34 | - schedule_look_emails 35 | - see_drill_overlay 36 | - see_lookml 37 | - see_sql 38 | - see_user_dashboards 39 | - send_to_integration 40 | - use_sql_runner 41 | model_set: 42 | - lk2_business_operations 43 | - DrewsModel 44 | team: 45 | - BusinessOperations_BO_Dev 46 | - DrewEdit 47 | 48 | role_BusinessOperations_User: 49 | role: BOUser 50 | permissions: 51 | - access_data 52 | - clear_cache_refresh 53 | - create_table_calculations 54 | - download_without_limit 55 | - explore 56 | - manage_spaces 57 | - mobile_app_access 58 | - save_content 59 | - schedule_look_emails 60 | - see_drill_overlay 61 | - see_lookml 62 | - see_lookml_dashboards 63 | - see_looks 64 | - see_sql 65 | - see_user_dashboards 66 | - send_to_integration 67 | model_set: 68 | - lk4_huggy 69 | models: 70 | - testy 71 | - buggy 72 | - lk2_jungy 73 | team: 74 | - BusinessOperations 75 | - Freddy 76 | 77 | 78 | 79 | ##################### 80 | # FOLDER PERMISSONS # 81 | ##################### 82 | BusinessOperations_folder_permissions: 83 | folder: 84 | name: BusinessOperations 85 | team_edit: 86 | - BusinessOperations 87 | team_view: 88 | - Snaptest 89 | - Cameos 90 | path: 91 | - Shared/ 92 | 93 | Cameos_folder_permissions: 94 | folder: 95 | name: Cameos 96 | team_edit: 97 | - Cameos 98 | team_view: 99 | - BusinessOperations 100 | - HugoGroup 101 | Hugo_folder_permissions: 102 | folder: 103 | name: Hugo 104 | team_edit: 105 | - Hugo 106 | - HugoEdit 107 | team_view: 108 | - BusinessOperations 109 | - NotHugoGroup 110 | 111 | Drews_folder_permissions: 112 | folder: 113 | name: Drew 114 | team_edit: 115 | - DrewEdit 116 | team_view: 117 | - BusinessOperations 118 | 119 | 120 | 121 | 122 | ################### 123 | # User Attributes # 124 | ################### 125 | # attr_region: 126 | ua_region_all: 127 | name: region_all 128 | type: string 129 | hidden_value: false 130 | user_view: true 131 | user_edit: false 132 | value: 133 | - us 134 | - ag 135 | - bb 136 | - dd 137 | team: 138 | - Cameos 139 | - Freddy 140 | - AudreyGroup 141 | 142 | ua_region_testy: 143 | name: region_testy 144 | type: string 145 | hidden_value: false 146 | user_view: true 147 | user_edit: false 148 | value: 149 | - us 150 | - ag 151 | - bb 152 | team: 153 | - Cameos 154 | - AudreyGroup 155 | 156 | 157 | ua_attr_can_see_hugo: 158 | name: can_see_hugo 159 | type: string 160 | hidden_value: false 161 | user_view: true 162 | user_edit: false 163 | value: 164 | - 'No' 165 | team: 166 | - Cameos 167 | - CanSeeDAUGroup 168 | 169 | ua_attr_can_see_dau: 170 | name: can_see_dau 171 | type: string 172 | hidden_value: false 173 | user_view: true 174 | user_edit: false 175 | value: 176 | - yes 177 | team: 178 | - Cameos 179 | - CanSeeDAUGroup 180 | -------------------------------------------------------------------------------- /tests/example_yamls/fullinstance.yaml: -------------------------------------------------------------------------------- 1 | ######### 2 | # MODEL_SET_ROLES # 3 | ######### 4 | permission_sets: 5 | developer: 6 | permissions: 7 | - access_data 8 | - can_see_system_activity 9 | - see_dashboards 10 | - clear_cache_refresh 11 | - create_table_calculations 12 | - deploy 13 | - develop 14 | - download_without_limit 15 | - explore 16 | - manage_spaces 17 | - mobile_app_access 18 | - save_content 19 | - schedule_look_emails 20 | - see_drill_overlay 21 | - see_lookml 22 | - see_sql 23 | - see_user_dashboards 24 | - send_to_integration 25 | - use_sql_runner 26 | 27 | nodeploy_developer: 28 | permissions: 29 | - access_data 30 | - can_see_system_activity 31 | - see_dashboards 32 | - clear_cache_refresh 33 | - create_table_calculations 34 | - develop 35 | - download_without_limit 36 | - explore 37 | - manage_spaces 38 | - mobile_app_access 39 | - save_content 40 | - schedule_look_emails 41 | - see_drill_overlay 42 | - see_lookml 43 | - see_sql 44 | - see_user_dashboards 45 | - send_to_integration 46 | - use_sql_runner 47 | 48 | user: 49 | permissions: 50 | - access_data 51 | - clear_cache_refresh 52 | - create_table_calculations 53 | - download_without_limit 54 | - explore 55 | - manage_spaces 56 | - mobile_app_access 57 | - save_content 58 | - schedule_look_emails 59 | - see_drill_overlay 60 | - see_lookml 61 | - see_lookml_dashboards 62 | - see_looks 63 | - see_sql 64 | - see_user_dashboards 65 | - send_to_integration 66 | 67 | model_sets: 68 | lk1_test_set: 69 | models: 70 | - test 71 | - test2 72 | lk4_huggy: 73 | models: 74 | - test 75 | - test1 76 | - test2 77 | 78 | roles: 79 | BusinessOperations_Developer: 80 | permission_set: developer 81 | model_set: lk1_test_set 82 | team: 83 | - BusinessOperations_BO_Dev 84 | - DrewEdit 85 | - westlake 86 | 87 | BusinessOperations_User: 88 | permission_set: user 89 | model_set: lk4_huggy 90 | team: 91 | - BusinessOperations 92 | - Freddy 93 | 94 | Operations_User: 95 | permission_set: user 96 | model_set: lk4_huggy 97 | team: 98 | - BusinessOperations 99 | - Freddy 100 | - HugoTesty 101 | 102 | ##################### 103 | # FOLDER PERMISSONS # 104 | ##################### 105 | folder_permissions: 106 | business_operations_folder: 107 | - name: 'Business Operations' 108 | team_view: 109 | - BusinessOperations 110 | - Snaptest 111 | subfolder: 112 | - name: test_sub 113 | team_edit: 114 | - Freddy 115 | team_view: 116 | - hugo 117 | - name: test_sub2 118 | subfolder: 119 | - name: test_sub_sub 120 | team_edit: 121 | - Famke 122 | team_view: 123 | - hugle 124 | - name: subdiddy 125 | subfolder: 126 | - name: hugle_testy 127 | team_edit: 128 | - Freddy 129 | team_view: 130 | - hugle 131 | 132 | sexy_time_folder: 133 | - name: 'sexy time' 134 | team_edit: 135 | - sexy_group1 136 | team_view: 137 | - sexy_group2 138 | subfolder: 139 | - name: sexy_sub 140 | 141 | suffering_succotash_folder: 142 | - name: 'suffering succotash' 143 | team_edit: 144 | - sexy_group1 145 | team_view: 146 | - sexy_group2 147 | subfolder: 148 | - name: another_one 149 | team_edit: 150 | - HugoTesty 151 | - new_group 152 | team_view: 153 | - newer_group 154 | subfolder: 155 | - name: sharkytesttest 156 | team_edit: 157 | - colin 158 | - sharon 159 | team_view: 160 | - nick 161 | - name: ps_allhands 162 | team_view: 163 | - ps_allhand1 164 | team_edit: 165 | - ps_allhand2 166 | 167 | 168 | 169 | ################### 170 | # User Attributes # 171 | ################### 172 | # attr_region: 173 | user_attributes: 174 | region_all: 175 | type: string 176 | hidden_value: false 177 | user_view: true 178 | user_edit: false 179 | value: 180 | - us 181 | - ag 182 | - bb 183 | - dd 184 | team: 185 | - Cameos 186 | - Freddy 187 | - AudreyGroup 188 | 189 | can_see_sharky: 190 | type: string 191 | hidden_value: false 192 | user_view: true 193 | user_edit: false 194 | value: 195 | - 'Sharky' 196 | team: 197 | - HugoTesty 198 | - CanSeeDAUGroup 199 | 200 | can_see_hugo: 201 | type: string 202 | hidden_value: false 203 | user_view: true 204 | user_edit: false 205 | value: 206 | - 'No' 207 | team: 208 | - Cameos 209 | - CanSeeDAUGroup 210 | 211 | 212 | -------------------------------------------------------------------------------- /tests/migrate_schedule.py: -------------------------------------------------------------------------------- 1 | import looker_sdk 2 | import os 3 | import json 4 | from lmanage.datetime import datetime 5 | from lmanage.typing import Callable, List 6 | 7 | ini = '/usr/local/google/home/hugoselbie/code_sample/py/ini/k8.ini' 8 | # os.environ["LOOKERSDK_BASE_URL"] = "https://profservices.dev.looker.com:19999" #If your looker URL has .cloud in it (hosted on GCP), do not include :19999 (ie: https://your.cloud.looker.com). 9 | # os.environ["LOOKERSDK_API_VERSION"] = "3.1" #3.1 is the default version. You can change this to 4.0 if you want. 10 | # os.environ["LOOKERSDK_VERIFY_SSL"] = "true" #Defaults to true if not set. SSL verification should generally be on unless you have a real good reason not to use it. Valid options: true, y, t, yes, 1. 11 | # os.environ["LOOKERSDK_TIMEOUT"] = "120" #Seconds till request timeout. Standard default is 120. 12 | 13 | # #Get the following values from lmanage.your Users page in the Admin panel of your Looker instance > Users > Your user > Edit API keys. If you know your user id, you can visit https://your.looker.com/admin/users//edit. 14 | # os.environ["LOOKERSDK_CLIENT_ID"] = "xxxxx" #No defaults. 15 | # os.environ["LOOKERSDK_CLIENT_SECRET"] = "xxxxx" #No defaults. This should be protected at all costs. Please do not leave it sitting here, even if you don't share this document. 16 | 17 | sdk = looker_sdk.init31(config_file=ini) 18 | print('Looker SDK 3.1 initialized successfully.') 19 | 20 | # How to define old user.. 21 | # User hasn't logged in for n_days based on credentials_email or credentials_saml 22 | n_days = 60 23 | 24 | # function to calculate number of days between last log in date and current date 25 | 26 | 27 | def days_between(d1: datetime, d2: datetime) -> int: 28 | """Return abs difference in days between two timestamps""" 29 | d1 = datetime.strptime(d1, "%Y-%m-%d") 30 | d2 = datetime.strptime(d2, "%Y-%m-%d") 31 | return abs((d2 - d1).days) 32 | 33 | # function to get a list of the old users 34 | 35 | 36 | def fetch_old_users(n_days: int, exclude_email: bool = True, exclude_saml: bool = True): 37 | """Fetch all users who haven't logged in for more than n_days""" 38 | users = sdk.all_users( 39 | fields='id,email,credentials_email(logged_in_at),credentials_saml(logged_in_at)') 40 | if exclude_email: 41 | users = [u for u in users 42 | if u.get('credentials_email') is not None 43 | ] 44 | inactive_email = [u for u in users 45 | if u['credentials_email']['logged_in_at'] != None 46 | and days_between( 47 | u['credentials_email']['logged_in_at'].split('T')[0], 48 | str(datetime.today()).split()[0] 49 | ) > n_days 50 | ] 51 | if exclude_saml: 52 | users = [u for u in users 53 | if u.get('credentials_saml') is not None 54 | ] 55 | inactive_saml = [u for u in users 56 | if u['credentials_saml']['logged_in_at'] != None 57 | and days_between( 58 | u['credentials_saml']['logged_in_at'].split('T')[0], 59 | str(datetime.today()).split()[0] 60 | ) > n_days 61 | ] 62 | user_id_inactive_email = [u.id for u in inactive_email] 63 | user_id_inactive_saml = [u.id for u in inactive_saml] 64 | # Check if user id is in email list and if not then merge saml list user ids into email list 65 | for x in user_id_inactive_saml: 66 | if x not in user_id_inactive_email: 67 | user_id_inactive_email.append(x) 68 | return user_id_inactive_email 69 | 70 | 71 | user_list = fetch_old_users(n_days) 72 | print(user_list) 73 | 74 | # For testing purposes 75 | # ids 236,18,758 76 | # names - alejandro@looker.com, jon.bale@looker.com, jcarnes+profservices@google.com 77 | user_list = [236, 18, 758] 78 | new_owner_id = 11 79 | 80 | # Find all schedules of a particular user id 81 | 82 | 83 | def find_schedules(current_owner_id: int): 84 | result = {} 85 | scheduled_plans = sdk.all_scheduled_plans(user_id=current_owner_id) 86 | for i in scheduled_plans: 87 | result[i['name']] = i['id'] 88 | return result 89 | 90 | # Transfer all schedules of a user to a new user. 91 | 92 | 93 | def update_owner(current_owner_id: int, new_owner_id: int): 94 | body = {} 95 | body['user_id'] = new_owner_id 96 | find = find_schedules(current_owner_id) 97 | for i in find.values(): 98 | sdk.update_scheduled_plan(i, body) 99 | return body 100 | 101 | # Loop through list of users to find schedules and transfer schedules 102 | 103 | 104 | def transfer_schedule(user_list: list, create_user: False): 105 | # find old users 106 | old_user_list = fetch_old_users(n_days=1) 107 | # find their schedules 108 | for user in old_user_list: 109 | x = find_schedules(user) 110 | print(x) 111 | update_owner(user, 2) 112 | # find reassign them to meta user 113 | # do i need to create meta user 114 | # if create_user: 115 | # sd 116 | # update owner 117 | # 118 | # update_owner(u, new_owner_id) 119 | 120 | # return f'Schedule transferred for {user_list}' 121 | 122 | 123 | if __name__ == "__main__": 124 | reassign_schedules = transfer_schedule(user_list) 125 | print(reassign_schedules) 126 | 127 | # from lmanage.pprint import pprint 128 | # import snoop - then @snoop 129 | -------------------------------------------------------------------------------- /tests/test_capture_looks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from os import name 3 | import pytest 4 | from lmanage.capturator.content_capturation import look_capture as lc 5 | from tests import fake_methods_data 6 | 7 | input_data = fake_methods_data.input_data 8 | 9 | 10 | def test_get_all_looks_metadata(mocker): 11 | sdk = fake_methods_data.MockSDK() 12 | look_obj = lc.LookCapture(sdk=sdk, content_folders=123) 13 | all_look_data = fake_methods_data.MockAll_look_Response( 14 | id='123', folder='123') 15 | mocker.patch.object(sdk, "all_looks") 16 | sdk.all_looks.return_value = all_look_data 17 | response = [look_obj.get_all_looks_metadata()] 18 | 19 | assert isinstance(response, list) 20 | assert 'id' in response[0] 21 | assert 'folder' in response[0] 22 | -------------------------------------------------------------------------------- /tests/test_create_folders.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from looker_sdk.sdk.api40 import models 3 | from os import name 4 | import pytest 5 | from lmanage.configurator.folder_configuration import create_folders as cf 6 | from tests import fake_methods_data 7 | 8 | input_data = fake_methods_data.input_data 9 | LOGGER = logging.getLogger(__name__) 10 | 11 | 12 | def test_create_folder(mocker): 13 | # Tests if there is a folder exists we raise an exception 14 | sdk = fake_methods_data.MockSDK() 15 | sf_data = fake_methods_data.MockCreateFolder( 16 | name='frankie_fish', parent_id='123') 17 | mocker.patch.object(sdk, "create_folder") 18 | sdk.create_folder.return_value = sf_data 19 | 20 | test = cf.CreateInstanceFolders(sdk=sdk, folder_metadata='5555').create_folder( 21 | folder_name='test', parent_id='123') 22 | assert isinstance(test, fake_methods_data.MockCreateFolder) 23 | assert test.name == 'frankie_fish' 24 | sdk.create_folder.assert_called_with( 25 | body=models.CreateFolder(name='test', parent_id='123')) 26 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/dashboards/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/tests/test_lookml_files/the_look/dashboards/.gitkeep -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/models/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/tests/test_lookml_files/the_look/models/.gitkeep -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/models_aws/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/tests/test_lookml_files/the_look/models_aws/.gitkeep -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/tests/test_lookml_files/the_look/views/.gitkeep -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/03_inventory_items.view.lkml: -------------------------------------------------------------------------------- 1 | view: inventory_items { 2 | sql_table_name: `looker-private-demo.ecomm.inventory_items` ;; 3 | 4 | ## DIMENSIONS ## 5 | 6 | dimension: id { 7 | primary_key: yes 8 | type: number 9 | sql: ${TABLE}.id ;; 10 | } 11 | 12 | dimension: cost { 13 | type: number 14 | value_format_name: usd 15 | sql: ${TABLE}.cost ;; 16 | } 17 | 18 | dimension_group: created { 19 | type: time 20 | timeframes: [time, date, week, month, raw] 21 | #sql: cast(CASE WHEN ${TABLE}.created_at = "\\N" THEN NULL ELSE ${TABLE}.created_at END as timestamp) ;; 22 | sql: CAST(${TABLE}.created_at AS TIMESTAMP) ;; 23 | } 24 | 25 | dimension: product_id { 26 | type: number 27 | hidden: yes 28 | sql: ${TABLE}.product_id ;; 29 | } 30 | 31 | dimension_group: sold { 32 | type: time 33 | timeframes: [time, date, week, month, raw] 34 | # sql: cast(CASE WHEN ${TABLE}.sold_at = "\\N" THEN NULL ELSE ${TABLE}.sold_at END as timestamp) ;; 35 | sql: ${TABLE}.sold_at ;; 36 | } 37 | 38 | dimension: is_sold { 39 | type: yesno 40 | sql: ${sold_raw} is not null ;; 41 | } 42 | 43 | dimension: days_in_inventory { 44 | description: "days between created and sold date" 45 | type: number 46 | sql: TIMESTAMP_DIFF(coalesce(${sold_raw}, CURRENT_TIMESTAMP()), ${created_raw}, DAY) ;; 47 | } 48 | 49 | dimension: days_in_inventory_tier { 50 | type: tier 51 | sql: ${days_in_inventory} ;; 52 | style: integer 53 | tiers: [0, 5, 10, 20, 40, 80, 160, 360] 54 | } 55 | 56 | dimension: days_since_arrival { 57 | description: "days since created - useful when filtering on sold yesno for items still in inventory" 58 | type: number 59 | sql: TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), ${created_raw}, DAY) ;; 60 | } 61 | 62 | dimension: days_since_arrival_tier { 63 | type: tier 64 | sql: ${days_since_arrival} ;; 65 | style: integer 66 | tiers: [0, 5, 10, 20, 40, 80, 160, 360] 67 | } 68 | 69 | dimension: product_distribution_center_id { 70 | hidden: yes 71 | sql: ${TABLE}.product_distribution_center_id ;; 72 | } 73 | 74 | ## MEASURES ## 75 | 76 | measure: sold_count { 77 | type: count 78 | drill_fields: [detail*] 79 | 80 | filters: { 81 | field: is_sold 82 | value: "Yes" 83 | } 84 | } 85 | 86 | measure: sold_percent { 87 | type: number 88 | value_format_name: percent_2 89 | sql: 1.0 * ${sold_count}/(CASE WHEN ${count} = 0 THEN NULL ELSE ${count} END) ;; 90 | } 91 | 92 | measure: total_cost { 93 | type: sum 94 | value_format_name: usd 95 | sql: ${cost} ;; 96 | } 97 | 98 | measure: average_cost { 99 | type: average 100 | value_format_name: usd 101 | sql: ${cost} ;; 102 | } 103 | 104 | measure: count { 105 | type: count 106 | drill_fields: [detail*] 107 | } 108 | 109 | measure: number_on_hand { 110 | type: count 111 | drill_fields: [detail*] 112 | 113 | filters: { 114 | field: is_sold 115 | value: "No" 116 | } 117 | } 118 | 119 | measure: stock_coverage_ratio { 120 | type: number 121 | description: "Stock on Hand vs Trailing 28d Sales Ratio" 122 | sql: 1.0 * ${number_on_hand} / nullif(${order_items.count_last_28d},0) ;; 123 | value_format_name: decimal_2 124 | html:

{{ rendered_value }}

;; 125 | } 126 | 127 | set: detail { 128 | fields: [id, products.item_name, products.category, products.brand, products.department, cost, created_time, sold_time] 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/04_products.view.lkml: -------------------------------------------------------------------------------- 1 | view: products { 2 | sql_table_name: `looker-private-demo.ecomm.products` ;; 3 | 4 | ### DIMENSIONS ### 5 | 6 | dimension: id { 7 | primary_key: yes 8 | type: number 9 | sql: ${TABLE}.id ;; 10 | } 11 | 12 | dimension: category { 13 | sql: TRIM(${TABLE}.category) ;; 14 | drill_fields: [item_name] 15 | } 16 | 17 | dimension: item_name { 18 | sql: TRIM(${TABLE}.name) ;; 19 | } 20 | 21 | dimension: brand { 22 | sql: TRIM(${TABLE}.brand) ;; 23 | link: { 24 | label: "Website" 25 | url: "http://www.google.com/search?q={{ value | encode_uri }}+clothes&btnI" 26 | icon_url: "http://www.google.com/s2/favicons?domain=www.{{ value | encode_uri }}.com" 27 | } 28 | link: { 29 | label: "Facebook" 30 | url: "http://www.google.com/search?q=site:facebook.com+{{ value | encode_uri }}+clothes&btnI" 31 | icon_url: "https://upload.wikimedia.org/wikipedia/commons/c/c2/F_icon.svg" 32 | } 33 | link: { 34 | label: "{{value}} Analytics Dashboard" 35 | url: "/dashboards/CRMxoGiGJUv4eGALMHiAb0?Brand%20Name={{ value | encode_uri }}" 36 | icon_url: "http://www.looker.com/favicon.ico" 37 | } 38 | drill_fields: [category, item_name] 39 | } 40 | 41 | dimension: retail_price { 42 | type: number 43 | sql: ${TABLE}.retail_price ;; 44 | } 45 | 46 | dimension: department { 47 | sql: TRIM(${TABLE}.department) ;; 48 | } 49 | 50 | dimension: sku { 51 | sql: ${TABLE}.sku ;; 52 | } 53 | 54 | dimension: distribution_center_id { 55 | type: number 56 | sql: CAST(${TABLE}.distribution_center_id AS INT64) ;; 57 | } 58 | 59 | ## MEASURES ## 60 | 61 | measure: count { 62 | type: count 63 | drill_fields: [detail*] 64 | } 65 | 66 | measure: brand_count { 67 | type: count_distinct 68 | sql: ${brand} ;; 69 | drill_fields: [brand, detail2*, -brand_count] # show the brand, a bunch of counts (see the set below), don't show the brand count, because it will always be 1 70 | } 71 | 72 | measure: category_count { 73 | alias: [category.count] 74 | type: count_distinct 75 | sql: ${category} ;; 76 | drill_fields: [category, detail2*, -category_count] # don't show because it will always be 1 77 | } 78 | 79 | measure: department_count { 80 | alias: [department.count] 81 | type: count_distinct 82 | sql: ${department} ;; 83 | drill_fields: [department, detail2*, -department_count] # don't show because it will always be 1 84 | } 85 | 86 | set: detail { 87 | fields: [id, item_name, brand, category, department, retail_price, customers.count, orders.count, order_items.count, inventory_items.count] 88 | } 89 | 90 | set: detail2 { 91 | fields: [category_count, brand_count, department_count, count, customers.count, orders.count, order_items.count, inventory_items.count, products.count] 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/05_distribution_centers.view.lkml: -------------------------------------------------------------------------------- 1 | view: distribution_centers { 2 | sql_table_name: `looker-private-demo.ecomm.distribution_centers` ;; 3 | dimension: location { 4 | type: location 5 | sql_latitude: ${TABLE}.latitude ;; 6 | sql_longitude: ${TABLE}.longitude ;; 7 | } 8 | 9 | dimension: latitude { 10 | sql: ${TABLE}.latitude ;; 11 | hidden: yes 12 | } 13 | 14 | dimension: longitude { 15 | sql: ${TABLE}.longitude ;; 16 | hidden: yes 17 | } 18 | 19 | dimension: id { 20 | type: number 21 | primary_key: yes 22 | sql: ${TABLE}.id ;; 23 | } 24 | 25 | dimension: name { 26 | sql: ${TABLE}.name ;; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/11_order_facts.view.lkml: -------------------------------------------------------------------------------- 1 | include: "/models/**/thelook.model.lkml" 2 | view: order_facts { 3 | derived_table: { 4 | explore_source: order_items { 5 | column: order_id {field: order_items.order_id_no_actions } 6 | column: items_in_order { field: order_items.count } 7 | column: order_amount { field: order_items.total_sale_price } 8 | column: order_cost { field: inventory_items.total_cost } 9 | column: user_id {field: order_items.user_id } 10 | column: created_at {field: order_items.created_raw} 11 | column: order_gross_margin {field: order_items.total_gross_margin} 12 | derived_column: order_sequence_number { 13 | sql: RANK() OVER (PARTITION BY user_id ORDER BY created_at) ;; 14 | } 15 | } 16 | datagroup_trigger: ecommerce_etl 17 | } 18 | 19 | dimension: order_id { 20 | type: number 21 | hidden: yes 22 | primary_key: yes 23 | sql: ${TABLE}.order_id ;; 24 | } 25 | 26 | dimension: items_in_order { 27 | type: number 28 | sql: ${TABLE}.items_in_order ;; 29 | } 30 | 31 | dimension: order_amount { 32 | type: number 33 | value_format_name: usd 34 | sql: ${TABLE}.order_amount ;; 35 | } 36 | 37 | dimension: order_cost { 38 | type: number 39 | value_format_name: usd 40 | sql: ${TABLE}.order_cost ;; 41 | } 42 | 43 | dimension: order_gross_margin { 44 | type: number 45 | value_format_name: usd 46 | } 47 | 48 | dimension: order_sequence_number { 49 | type: number 50 | sql: ${TABLE}.order_sequence_number ;; 51 | } 52 | 53 | dimension: is_first_purchase { 54 | type: yesno 55 | sql: ${order_sequence_number} = 1 ;; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/12_user_order_facts.view.lkml: -------------------------------------------------------------------------------- 1 | view: user_order_facts { 2 | derived_table: { 3 | sql: 4 | SELECT 5 | user_id 6 | , COUNT(DISTINCT order_id) AS lifetime_orders 7 | , SUM(sale_price) AS lifetime_revenue 8 | , CAST(MIN(created_at) AS TIMESTAMP) AS first_order 9 | , CAST(MAX(created_at) AS TIMESTAMP) AS latest_order 10 | , COUNT(DISTINCT FORMAT_TIMESTAMP('%Y%m', created_at)) AS number_of_distinct_months_with_orders 11 | --, FIRST_VALUE(CONCAT(uniform(2, 9, random(1)),uniform(0, 9, random(2)),uniform(0, 9, random(3)),'-',uniform(0, 9, random(4)),uniform(0, 9, random(5)),uniform(0, 9, random(6)),'-',uniform(0, 9, random(7)),uniform(0, 9, random(8)),uniform(0, 9, random(9)),uniform(0, 9, random(10)))) OVER (PARTITION BY user_id ORDER BY user_id) AS phone_number 12 | FROM looker-private-demo.ecomm.order_items 13 | GROUP BY user_id 14 | ;; 15 | datagroup_trigger: ecommerce_etl 16 | } 17 | 18 | dimension: user_id { 19 | primary_key: yes 20 | hidden: yes 21 | sql: ${TABLE}.user_id ;; 22 | } 23 | 24 | # dimension: phone_number { 25 | # type: string 26 | # tags: ["phone"] 27 | # sql: ${TABLE}.phone_number ;; 28 | # } 29 | 30 | 31 | ##### Time and Cohort Fields ###### 32 | 33 | dimension_group: first_order { 34 | type: time 35 | timeframes: [date, week, month, year] 36 | sql: ${TABLE}.first_order ;; 37 | } 38 | 39 | dimension_group: latest_order { 40 | type: time 41 | timeframes: [date, week, month, year] 42 | sql: ${TABLE}.latest_order ;; 43 | } 44 | 45 | dimension: days_as_customer { 46 | description: "Days between first and latest order" 47 | type: number 48 | sql: TIMESTAMP_DIFF(${TABLE}.latest_order, ${TABLE}.first_order, DAY)+1 ;; 49 | } 50 | 51 | dimension: days_as_customer_tiered { 52 | type: tier 53 | tiers: [0, 1, 7, 14, 21, 28, 30, 60, 90, 120] 54 | sql: ${days_as_customer} ;; 55 | style: integer 56 | } 57 | 58 | ##### Lifetime Behavior - Order Counts ###### 59 | 60 | dimension: lifetime_orders { 61 | type: number 62 | sql: ${TABLE}.lifetime_orders ;; 63 | } 64 | 65 | dimension: repeat_customer { 66 | description: "Lifetime Count of Orders > 1" 67 | type: yesno 68 | sql: ${lifetime_orders} > 1 ;; 69 | } 70 | 71 | dimension: lifetime_orders_tier { 72 | type: tier 73 | tiers: [0, 1, 2, 3, 5, 10] 74 | sql: ${lifetime_orders} ;; 75 | style: integer 76 | } 77 | 78 | measure: average_lifetime_orders { 79 | type: average 80 | value_format_name: decimal_2 81 | sql: ${lifetime_orders} ;; 82 | } 83 | 84 | dimension: distinct_months_with_orders { 85 | type: number 86 | sql: ${TABLE}.number_of_distinct_months_with_orders ;; 87 | } 88 | 89 | ##### Lifetime Behavior - Revenue ###### 90 | 91 | dimension: lifetime_revenue { 92 | type: number 93 | value_format_name: usd 94 | sql: ${TABLE}.lifetime_revenue ;; 95 | } 96 | 97 | dimension: lifetime_revenue_tier { 98 | type: tier 99 | tiers: [0, 25, 50, 100, 200, 500, 1000] 100 | sql: ${lifetime_revenue} ;; 101 | style: integer 102 | } 103 | 104 | measure: average_lifetime_revenue { 105 | type: average 106 | value_format_name: usd 107 | sql: ${lifetime_revenue} ;; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/13_repeat_purchase_facts.view.lkml: -------------------------------------------------------------------------------- 1 | view: repeat_purchase_facts { 2 | derived_table: { 3 | datagroup_trigger: ecommerce_etl 4 | sql: SELECT 5 | order_items.order_id as order_id 6 | , order_items.created_at 7 | , COUNT(DISTINCT repeat_order_items.id) AS number_subsequent_orders 8 | , MIN(repeat_order_items.created_at) AS next_order_date 9 | , MIN(repeat_order_items.order_id) AS next_order_id 10 | FROM looker-private-demo.ecomm.order_items as order_items 11 | LEFT JOIN looker-private-demo.ecomm.order_items repeat_order_items 12 | ON order_items.user_id = repeat_order_items.user_id 13 | AND order_items.created_at < repeat_order_items.created_at 14 | GROUP BY 1, 2 15 | ;; 16 | } 17 | 18 | dimension: order_id { 19 | type: number 20 | hidden: yes 21 | primary_key: yes 22 | sql: ${TABLE}.order_id ;; 23 | } 24 | 25 | dimension: next_order_id { 26 | type: number 27 | hidden: yes 28 | sql: ${TABLE}.next_order_id ;; 29 | } 30 | 31 | dimension: has_subsequent_order { 32 | type: yesno 33 | sql: ${next_order_id} > 0 ;; 34 | } 35 | 36 | dimension: number_subsequent_orders { 37 | type: number 38 | sql: ${TABLE}.number_subsequent_orders ;; 39 | } 40 | 41 | dimension_group: next_order { 42 | type: time 43 | timeframes: [raw, date] 44 | hidden: yes 45 | sql: CAST(${TABLE}.next_order_date AS TIMESTAMP) ;; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/25_trailing_sales_snapshot.view.lkml: -------------------------------------------------------------------------------- 1 | view: trailing_sales_snapshot { 2 | derived_table: { 3 | datagroup_trigger: ecommerce_etl 4 | sql: with calendar as 5 | (select distinct created_at as snapshot_date 6 | from looker-private-demo.ecomm.inventory_items 7 | -- where dateadd('day',90,created_at)>=current_date 8 | ) 9 | 10 | select 11 | inventory_items.product_id 12 | ,date(order_items.created_at) as snapshot_date 13 | ,count(*) as trailing_28d_sales 14 | from looker-private-demo.ecomm.order_items 15 | join looker-private-demo.ecomm.inventory_items 16 | on order_items.inventory_item_id = inventory_items.id 17 | join calendar 18 | on date(order_items.created_at) <= date_add(calendar.snapshot_date, interval 28 day) 19 | and date(order_items.created_at) >= calendar.snapshot_date 20 | -- where dateadd('day',90,calendar.snapshot_date)>=current_date 21 | group by 1,2 22 | ;; 23 | } 24 | 25 | # measure: count { 26 | # type: count 27 | # drill_fields: [detail*] 28 | # } 29 | 30 | dimension: product_id { 31 | type: number 32 | sql: ${TABLE}.product_id ;; 33 | } 34 | 35 | dimension: snapshot_date { 36 | type: date 37 | sql: cast(${TABLE}.snapshot_date as timestamp) ;; 38 | } 39 | 40 | dimension: trailing_28d_sales { 41 | type: number 42 | hidden: yes 43 | sql: ${TABLE}.trailing_28d_sales ;; 44 | } 45 | 46 | measure: sum_trailing_28d_sales { 47 | type: sum 48 | sql: ${trailing_28d_sales} ;; 49 | } 50 | 51 | measure: sum_trailing_28d_sales_yesterday { 52 | type: sum 53 | hidden: yes 54 | sql: ${trailing_28d_sales} ;; 55 | filters: { 56 | field: snapshot_date 57 | value: "yesterday" 58 | } 59 | } 60 | 61 | measure: sum_trailing_28d_sales_last_wk { 62 | type: sum 63 | hidden: yes 64 | sql: ${trailing_28d_sales} ;; 65 | filters: { 66 | field: snapshot_date 67 | value: "8 days ago for 1 day" 68 | } 69 | } 70 | 71 | set: detail { 72 | fields: [product_id, snapshot_date, trailing_28d_sales] 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/51_events.view.lkml: -------------------------------------------------------------------------------- 1 | view: events { 2 | sql_table_name: `looker-private-demo.ecomm.events` ;; 3 | 4 | dimension: event_id { 5 | type: number 6 | primary_key: yes 7 | tags: ["mp_event_id"] 8 | sql: ${TABLE}.id ;; 9 | } 10 | 11 | dimension: session_id { 12 | type: number 13 | hidden: yes 14 | sql: ${TABLE}.session_id ;; 15 | } 16 | 17 | dimension: ip { 18 | label: "IP Address" 19 | view_label: "Visitors" 20 | sql: ${TABLE}.ip_address ;; 21 | } 22 | 23 | dimension: user_id { 24 | sql: ${TABLE}.user_id ;; 25 | } 26 | 27 | dimension_group: event { 28 | type: time 29 | # timeframes: [time, date, hour, time_of_day, hour_of_day, week, day_of_week_index, day_of_week] 30 | sql: ${TABLE}.created_at ;; 31 | } 32 | 33 | dimension: sequence_number { 34 | type: number 35 | description: "Within a given session, what order did the events take place in? 1=First, 2=Second, etc" 36 | sql: ${TABLE}.sequence_number ;; 37 | } 38 | 39 | dimension: is_entry_event { 40 | type: yesno 41 | description: "Yes indicates this was the entry point / landing page of the session" 42 | sql: ${sequence_number} = 1 ;; 43 | } 44 | 45 | dimension: is_exit_event { 46 | type: yesno 47 | label: "UTM Source" 48 | sql: ${sequence_number} = ${sessions.number_of_events_in_session} ;; 49 | description: "Yes indicates this was the exit point / bounce page of the session" 50 | } 51 | 52 | measure: count_bounces { 53 | type: count 54 | description: "Count of events where those events were the bounce page for the session" 55 | 56 | filters: { 57 | field: is_exit_event 58 | value: "Yes" 59 | } 60 | } 61 | 62 | measure: bounce_rate { 63 | type: number 64 | value_format_name: percent_2 65 | description: "Percent of events where those events were the bounce page for the session, out of all events" 66 | sql: ${count_bounces}*1.0 / nullif(${count}*1.0,0) ;; 67 | } 68 | 69 | dimension: full_page_url { 70 | sql: ${TABLE}.uri ;; 71 | } 72 | 73 | dimension: viewed_product_id { 74 | type: number 75 | sql: CASE WHEN ${event_type} = 'Product' THEN 76 | CAST(SPLIT(${full_page_url}, '/')[OFFSET(ARRAY_LENGTH(SPLIT(${full_page_url}, '/'))-1)] AS INT64) 77 | END 78 | ;; 79 | } 80 | 81 | dimension: event_type { 82 | sql: ${TABLE}.event_type ;; 83 | tags: ["mp_event_name"] 84 | } 85 | 86 | dimension: funnel_step { 87 | description: "Login -> Browse -> Add to Cart -> Checkout" 88 | sql: CASE 89 | WHEN ${event_type} IN ('Login', 'Home') THEN '(1) Land' 90 | WHEN ${event_type} IN ('Category', 'Brand') THEN '(2) Browse Inventory' 91 | WHEN ${event_type} = 'Product' THEN '(3) View Product' 92 | WHEN ${event_type} = 'Cart' THEN '(4) Add Item to Cart' 93 | WHEN ${event_type} = 'Purchase' THEN '(5) Purchase' 94 | END 95 | ;; 96 | } 97 | 98 | measure: unique_visitors { 99 | type: count_distinct 100 | description: "Uniqueness determined by IP Address and User Login" 101 | view_label: "Visitors" 102 | sql: ${ip} ;; 103 | drill_fields: [visitors*] 104 | } 105 | 106 | dimension: location { 107 | type: location 108 | view_label: "Visitors" 109 | sql_latitude: ${TABLE}.latitude ;; 110 | sql_longitude: ${TABLE}.longitude ;; 111 | } 112 | 113 | dimension: approx_location { 114 | type: location 115 | view_label: "Visitors" 116 | sql_latitude: round(${TABLE}.latitude,1) ;; 117 | sql_longitude: round(${TABLE}.longitude,1) ;; 118 | } 119 | 120 | dimension: has_user_id { 121 | type: yesno 122 | view_label: "Visitors" 123 | description: "Did the visitor sign in as a website user?" 124 | sql: ${users.id} > 0 ;; 125 | } 126 | 127 | dimension: browser { 128 | view_label: "Visitors" 129 | sql: ${TABLE}.browser ;; 130 | } 131 | 132 | dimension: os { 133 | label: "Operating System" 134 | view_label: "Visitors" 135 | sql: ${TABLE}.os ;; 136 | } 137 | 138 | measure: count { 139 | type: count 140 | drill_fields: [simple_page_info*] 141 | } 142 | 143 | measure: sessions_count { 144 | type: count_distinct 145 | sql: ${session_id} ;; 146 | } 147 | 148 | measure: count_m { 149 | label: "Count (MM)" 150 | type: number 151 | hidden: yes 152 | sql: ${count}/1000000.0 ;; 153 | drill_fields: [simple_page_info*] 154 | value_format: "#.### \"M\"" 155 | } 156 | 157 | measure: unique_visitors_m { 158 | label: "Unique Visitors (MM)" 159 | view_label: "Visitors" 160 | type: number 161 | sql: count (distinct ${ip}) / 1000000.0 ;; 162 | description: "Uniqueness determined by IP Address and User Login" 163 | value_format: "#.### \"M\"" 164 | hidden: yes 165 | drill_fields: [visitors*] 166 | } 167 | 168 | measure: unique_visitors_k { 169 | label: "Unique Visitors (k)" 170 | view_label: "Visitors" 171 | type: number 172 | hidden: yes 173 | description: "Uniqueness determined by IP Address and User Login" 174 | sql: count (distinct ${ip}) / 1000.0 ;; 175 | value_format: "#.### \"k\"" 176 | drill_fields: [visitors*] 177 | } 178 | 179 | set: simple_page_info { 180 | fields: [event_id, event_time, event_type, full_page_url, user_id, funnel_step] 181 | } 182 | 183 | set: visitors { 184 | fields: [ip, os, browser, user_id, count] 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/explores.lkml: -------------------------------------------------------------------------------- 1 | explore: distribution_centers { 2 | join: products { 3 | type: left_outer 4 | sql_on: ${products.distribution_center_id} = ${distribution_centers.id} ;; 5 | relationship: many_to_one 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /tests/test_lookml_files/the_look/views/test_ndt.view.lkml: -------------------------------------------------------------------------------- 1 | view: test_ndt { 2 | derived_table: { 3 | explore_source: order_items { 4 | column: total_revenue {} 5 | column: created_month {} 6 | column: user_id {} 7 | } 8 | } 9 | dimension: total_revenue { 10 | type: number 11 | } 12 | dimension: created_month { 13 | type: date_month 14 | } 15 | dimension: user_id { 16 | type: number 17 | } 18 | } 19 | 20 | --------------------------------------------------------------------------------