├── .github
├── CODEOWNERS
└── workflows
│ ├── configurator_deploy.yaml
│ └── pythonapp.yml
├── .gitignore
├── .vim
└── coc-settings.json
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── __main__.py
├── dist
├── lmanage-0.1.0-py3-none-any.whl
├── lmanage-0.1.0.tar.gz
├── lmanage-0.1.1-py3-none-any.whl
├── lmanage-0.1.1.tar.gz
├── lmanage-0.1.2-py3-none-any.whl
├── lmanage-0.1.2.tar.gz
├── lmanage-0.1.3-py3-none-any.whl
├── lmanage-0.1.3.tar.gz
├── lmanage-0.1.4-py3-none-any.whl
├── lmanage-0.1.4.tar.gz
├── lmanage-0.1.5-py3-none-any.whl
├── lmanage-0.1.5.tar.gz
├── lmanage-0.1.6-py3-none-any.whl
├── lmanage-0.1.6.tar.gz
├── lmanage-0.1.7-py3-none-any.whl
├── lmanage-0.1.7.tar.gz
├── lmanage-0.1.8-py3-none-any.whl
├── lmanage-0.1.8.tar.gz
├── lmanage-0.1.9-py3-none-any.whl
├── lmanage-0.1.9.tar.gz
├── lmanage-0.2.0-py3-none-any.whl
├── lmanage-0.2.0.tar.gz
├── lmanage-0.2.1-py3-none-any.whl
├── lmanage-0.2.1.tar.gz
├── lmanage-0.2.2-py3-none-any.whl
├── lmanage-0.2.2.tar.gz
├── lmanage-0.2.3-py3-none-any.whl
├── lmanage-0.2.3.tar.gz
├── lmanage-0.2.4-py3-none-any.whl
├── lmanage-0.2.4.tar.gz
├── lmanage-0.2.5-py3-none-any.whl
├── lmanage-0.2.5.tar.gz
├── lmanage-0.2.6.1-py3-none-any.whl
├── lmanage-0.2.6.1.tar.gz
├── lmanage-0.2.6.2-py3-none-any.whl
├── lmanage-0.2.6.2.tar.gz
├── lmanage-0.2.6.3-py3-none-any.whl
├── lmanage-0.2.6.3.tar.gz
├── lmanage-0.2.6.4-py3-none-any.whl
├── lmanage-0.2.6.4.tar.gz
├── lmanage-0.2.7-py3-none-any.whl
├── lmanage-0.2.7.tar.gz
├── lmanage-0.2.8-py3-none-any.whl
├── lmanage-0.2.8.tar.gz
├── lmanage-0.2.81-py3-none-any.whl
├── lmanage-0.2.81.tar.gz
├── lmanage-0.2.82-py3-none-any.whl
├── lmanage-0.2.82.tar.gz
├── lmanage-0.2.83-py3-none-any.whl
├── lmanage-0.2.83.tar.gz
├── lmanage-0.2.9-py3-none-any.whl
├── lmanage-0.2.9.tar.gz
├── lmanage-0.2.91-py3-none-any.whl
├── lmanage-0.2.91.tar.gz
├── lmanage-0.2.92-py3-none-any.whl
├── lmanage-0.2.92.tar.gz
├── lmanage-0.3.0-py3-none-any.whl
└── lmanage-0.3.0.tar.gz
├── file.log
├── images
├── hermes.png
├── lmanage_folder_diagram.png
├── lmanage_mapview_chart.pdf
├── mapview_walkthru.jpeg
├── my_diagram.png
└── web_service.png
├── instructions
├── capturator_README.md
├── configurator_README.md
├── looker_settings_capture.md
└── mapview_README.md
├── lmanage
├── .vimspector.json
├── __init__.py
├── capturator
│ ├── __init__.py
│ ├── content_capturation
│ │ ├── __init__.py
│ │ ├── board_capture.py
│ │ ├── dashboard_capture.py
│ │ └── look_capture.py
│ ├── folder_capturation
│ │ ├── __init__.py
│ │ ├── create_folder_yaml_structure.py
│ │ └── folder_config.py
│ ├── looker_api_reader.py
│ ├── looker_config_saver.py
│ ├── user_attribute_capturation
│ │ ├── __init__.py
│ │ └── capture_ua_permissions.py
│ └── user_group_capturation
│ │ ├── __init__.py
│ │ ├── group_config.py
│ │ ├── role_config.py
│ │ └── user_permission.py
├── cli.py
├── configurator
│ ├── .vimspector.json
│ ├── __init__.py
│ ├── content_configuration
│ │ ├── __init__.py
│ │ ├── clean_instance_content.py
│ │ ├── create_boards.py
│ │ ├── create_dashboards.py
│ │ └── create_looks.py
│ ├── create_object.py
│ ├── folder_configuration
│ │ ├── __init__.py
│ │ ├── create_and_provision_instance_folders.py
│ │ ├── create_instance_folders.py
│ │ └── folder_config.py
│ ├── instance_configuration_settings
│ │ └── fullinstance.yaml
│ ├── looker_config_reader.py
│ ├── looker_provisioner.py
│ ├── user_attribute_configuration
│ │ ├── __init__.py
│ │ └── create_and_assign_user_attributes.py
│ └── user_group_configuration
│ │ ├── __init__.py
│ │ ├── create_instance_groups.py
│ │ ├── create_instance_roles.py
│ │ └── create_role_base.py
├── find_replace.sh
├── logger_config.py
├── looker_auth.py
├── looker_config.py
├── mapview
│ ├── .vimspector.json
│ ├── __init__.py
│ ├── compare_content_to_lookml.py
│ ├── hugo.csv
│ ├── instancedata.py
│ ├── legacy_get_content_with_views.py
│ ├── main.py
│ ├── mapexplores.py
│ ├── mapview_execute.py
│ ├── matched_field_analysis.py
│ ├── pandas_multiple.xlsx
│ ├── parse_sql_tables.py
│ └── utils
│ │ ├── .vimspector.json
│ │ ├── __init__.py
│ │ ├── create_df.py
│ │ ├── parse_lookml.py
│ │ └── parsing_sql.py
└── utils
│ ├── __init__.py
│ ├── errorhandling.py
│ ├── helpers.py
│ ├── logger_creation.py
│ ├── looker_object_constructors.py
│ └── parse_yaml.py
├── output
├── demo_output.yaml
├── demo_output_content.yaml
├── hugo.csv
├── instance_mapview.xlsx
├── l233_output.yaml
├── l233_output_content.yaml
├── profserv_output.yaml
└── profserv_output_content.yaml
├── poetry.lock
├── pyproject.toml
└── tests
├── __init__.py
├── example_yamls
├── Looker22_12_output.yaml
├── Looker22_18_output.yaml
├── clustered_output.yaml
├── demo_output.yaml
├── dev_output.yaml
├── folder_permission_input.yaml
├── folder_yaml.yaml
├── fullinstance.yaml
├── k8_output.yaml
├── profserv_output.yaml
├── test_demo_output.yaml
├── test_output.yaml
├── test_test_output.yaml
└── user_permission_output.yaml
├── fake_methods_data.py
├── migrate_schedule.py
├── test_capture_looks.py
├── test_create_folders.py
└── test_lookml_files
└── the_look
├── dashboards
├── .gitkeep
├── brand_lookup.dashboard.lookml
├── business_pulse.dashboard.lookml
├── customer_lookup.dashboard.lookml
├── shipping_logistics_overview.dashboard.lookml
└── web_analytics_overview.dashboard.lookml
├── models
├── .gitkeep
├── bq.model.lkml
├── order_items.model.lkml
└── thelook.model.lkml
├── models_aws
├── .gitkeep
├── tests.txt
└── thelook_redshift.model.lkml
└── views
├── .gitkeep
├── 01_order_items.view.lkml
├── 02_users.view.lkml
├── 03_inventory_items.view.lkml
├── 04_products.view.lkml
├── 05_distribution_centers.view.lkml
├── 11_order_facts.view.lkml
├── 12_user_order_facts.view.lkml
├── 13_repeat_purchase_facts.view.lkml
├── 22_affinity.view.lkml
├── 25_trailing_sales_snapshot.view.lkml
├── 51_events.view.lkml
├── explores.lkml
└── test_ndt.view.lkml
/.github/CODEOWNERS:
--------------------------------------------------------------------------------
1 | * @looker-open-source/cloud-looker-devrel @hugoselbie @rbob86 @belvederej
2 |
--------------------------------------------------------------------------------
/.github/workflows/configurator_deploy.yaml:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | # This workflow will install Python dependencies, run tests and lint with a single version of Python
16 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
17 |
18 | name: Looker Validation Suite
19 |
20 | on:
21 | push:
22 | branches: [ main ]
23 | pull_request:
24 | branches: [ main ]
25 |
26 | jobs:
27 | build:
28 | runs-on: ubuntu-latest
29 |
30 | steps:
31 | - uses: actions/checkout@v2
32 | - name: Set up Python 3.8
33 | uses: actions/setup-python@v2
34 | with:
35 | python-version: "3.8"
36 | - name: Install dependencies
37 | run: |
38 | python -m pip install --upgrade pip
39 | pip install lmanage
40 | # Run Looker Instance Configuratory
41 | - name: Run LManage Configurator
42 | env:
43 | LOOKERSDK_API_VERSION: ${{ secrets.LOOKERSDK_API_VERSION }}
44 | LOOKERSDK_BASE_URL: ${{ secrets.LOOKERSDK_BASE_URL }}
45 | LOOKERSDK_CLIENT_ID: ${{ secrets.LOOKERSDK_CLIENT_ID }}
46 | LOOKERSDK_CLIENT_SECRET: ${{ secrets.LOOKERSDK_CLIENT_SECRET }}
47 | run: |
48 | cd .github
49 | echo ${{ github.head_ref }}
50 | echo ${{ github.event.push.ref }}
51 | lmanage configurator -yp /output/dev_output.yaml
52 |
--------------------------------------------------------------------------------
/.github/workflows/pythonapp.yml:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | name: python_application
16 |
17 | on:
18 | push:
19 | branches:
20 | - main
21 | pull_request:
22 | branches:
23 | - main
24 |
25 | jobs:
26 | build_37:
27 |
28 | runs-on: ubuntu-latest
29 |
30 | steps:
31 | - uses: actions/checkout@v1
32 | - name: Set up Python 3.8
33 | uses: actions/setup-python@v1
34 | with:
35 | python-version: 3.8
36 | - name: Install dependencies
37 | run: |
38 | pip install flake8 pytest pytest-mock
39 | pip install .
40 | - name: Lint with flake8
41 | run: |
42 | # stop the build if there are Python syntax errors or undefined names
43 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
44 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
45 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
46 | - name: Test with pytest
47 | run: |
48 | pytest
49 |
50 | build_38:
51 | runs-on: ubuntu-latest
52 |
53 | steps:
54 | - uses: actions/checkout@v1
55 | - name: Set up Python 3.8
56 | uses: actions/setup-python@v1
57 | with:
58 | python-version: 3.8
59 | - name: Install dependencies
60 | run: |
61 | pip install flake8 pytest pytest-mock
62 | pip install .
63 | - name: Lint with flake8
64 | run: |
65 | # stop the build if there are Python syntax errors or undefined names
66 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
67 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
68 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
69 | - name: Test with pytest
70 | run: |
71 | pytest
72 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | look_backup/
2 | looker_backup_content/
3 |
4 | dist/
5 | tests/example_yamls/
6 | output/
7 | .history/
8 | #Ignore all setup files
9 | .venv/
10 | .vscode/
11 | folder_walk.sh
12 | None
13 | .DS_Store
14 |
15 | #Ignore test files
16 | test.py
17 | test.csv
18 | output.csv
19 | delinquentcontent.csv
20 | spam.log
21 | map_view.log
22 | utils/test.py
23 | utils/old_files/
24 |
25 | #Ignore legacy files
26 | lmanage/getContentWithViews.py
27 | lmanage/delinquent_content.py
28 | lmanage/delinquent_user.py
29 | lmanage/validate_content.py
30 | lmanage/find_content_with_lookml_params.py
31 |
32 | #Ignore test files for early content
33 | tests/test_getContentWithViews.py
34 | tests/test_delinquent_content.py
35 | tests/test_delinquent_user.py
36 | tests/test_find_content_with_lookml_params.py
37 | tests/test_validate_content.py
38 | tests/test_lookml_files.py
39 | tests/snap_db_response.json
40 | tests/test_lookml_files/aws_thelook_redshift/
41 | logs/
42 |
43 | #Ignore pycache
44 | __pycache__/
45 | *.pyc
46 |
47 | # Ignore config & capturator files
48 | looker.ini
49 | config/
50 |
--------------------------------------------------------------------------------
/.vim/coc-settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "python.linting.pylintEnabled": false,
3 | "python.linting.pep8Enabled": false,
4 | "python.linting.enabled": true,
5 | "python.linting.flake8Enabled": true
6 | }
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | In the interest of fostering an open and welcoming environment, we as
6 | contributors and maintainers pledge to making participation in our project and
7 | our community a harassment-free experience for everyone, regardless of age, body
8 | size, disability, ethnicity, gender identity and expression, level of
9 | experience, education, socio-economic status, nationality, personal appearance,
10 | race, religion, or sexual identity and orientation.
11 |
12 | ## Our Standards
13 |
14 | Examples of behavior that contributes to creating a positive environment
15 | include:
16 |
17 | * Using welcoming and inclusive language
18 | * Being respectful of differing viewpoints and experiences
19 | * Gracefully accepting constructive criticism
20 | * Focusing on what is best for the community
21 | * Showing empathy towards other community members
22 |
23 | Examples of unacceptable behavior by participants include:
24 |
25 | * The use of sexualized language or imagery and unwelcome sexual attention or
26 | advances
27 | * Trolling, insulting/derogatory comments, and personal or political attacks
28 | * Public or private harassment
29 | * Publishing others' private information, such as a physical or electronic
30 | address, without explicit permission
31 | * Other conduct which could reasonably be considered inappropriate in a
32 | professional setting
33 |
34 | ## Our Responsibilities
35 |
36 | Project maintainers are responsible for clarifying the standards of acceptable
37 | behavior and are expected to take appropriate and fair corrective action in
38 | response to any instances of unacceptable behavior.
39 |
40 | Project maintainers have the right and responsibility to remove, edit, or reject
41 | comments, commits, code, wiki edits, issues, and other contributions that are
42 | not aligned to this Code of Conduct, or to ban temporarily or permanently any
43 | contributor for other behaviors that they deem inappropriate, threatening,
44 | offensive, or harmful.
45 |
46 | ## Scope
47 |
48 | This Code of Conduct applies both within project spaces and in public spaces
49 | when an individual is representing the project or its community. Examples of
50 | representing a project or community include using an official project e-mail
51 | address, posting via an official social media account, or acting as an appointed
52 | representative at an online or offline event. Representation of a project may be
53 | further defined and clarified by project maintainers.
54 |
55 | This Code of Conduct also applies outside the project spaces when the Project
56 | Steward has a reasonable belief that an individual's behavior may have a
57 | negative impact on the project or its community.
58 |
59 | ## Conflict Resolution
60 |
61 | We do not believe that all conflict is bad; healthy debate and disagreement
62 | often yield positive results. However, it is never okay to be disrespectful or
63 | to engage in behavior that violates the project’s code of conduct.
64 |
65 | If you see someone violating the code of conduct, you are encouraged to address
66 | the behavior directly with those involved. Many issues can be resolved quickly
67 | and easily, and this gives people more control over the outcome of their
68 | dispute. If you are unable to resolve the matter for any reason, or if the
69 | behavior is threatening or harassing, report it. We are dedicated to providing
70 | an environment where participants feel welcome and safe.
71 |
72 | Reports should be directed to *Hugo Selbie* hugoselbie@google.com the
73 | Project Steward(s) for *lmanage*. It is the Project Steward’s duty to
74 | receive and address reported violations of the code of conduct. They will then
75 | work with a committee consisting of representatives from the Open Source
76 | Programs Office and the Google Open Source Strategy team. If for any reason you
77 | are uncomfortable reaching out to the Project Steward, please email
78 | opensource@google.com.
79 |
80 | We will investigate every complaint, but you may not receive a direct response.
81 | We will use our discretion in determining when and how to follow up on reported
82 | incidents, which may range from not taking action to permanent expulsion from
83 | the project and project-sponsored spaces. We will notify the accused of the
84 | report and provide them an opportunity to discuss it before any action is taken.
85 | The identity of the reporter will be omitted from the details of the report
86 | supplied to the accused. In potentially harmful situations, such as ongoing
87 | harassment or threats to anyone's safety, we may take action without notice.
88 |
89 | ## Attribution
90 |
91 | This Code of Conduct is adapted from the Contributor Covenant, version 1.4,
92 | available at
93 | https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
94 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement (CLA). You (or your employer) retain the copyright to your
10 | contribution; this simply gives us permission to use and redistribute your
11 | contributions as part of the project. Head over to
12 | to see your current agreements on file or
13 | to sign a new one.
14 |
15 | You generally only need to submit a CLA once, so if you've already submitted one
16 | (even if it was for a different project), you probably don't need to do it
17 | again.
18 |
19 | ## Code Reviews
20 |
21 | All submissions, including submissions by project members, require review. We
22 | use GitHub pull requests for this purpose. Consult
23 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
24 | information on using pull requests.
25 |
26 | ## Community Guidelines
27 |
28 | This project follows
29 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/).
30 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # LManage
2 | ## What is it.
3 | LManage is a collection of useful tools for [Looker](https://looker.com/) admins to help curate and cleanup content and it's associated source [LookML](https://docs.looker.com/data-modeling/learning-lookml/what-is-lookml).
4 |
5 | ## How do i Install it.
6 | Lmanage can be found on [pypi](#).
7 | ```
8 | pip install lmanage
9 | ```
10 |
11 | ## How do I Use it.
12 | ### Commands
13 | LManage will ultimately will have many different commands as development continues
14 | | Status | Command | Rationale |
15 | |---------|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
16 | | Live | Object Migrator Tool | Migrate Looker Objects such as Content, Folders and Permissions, User Groups, Roles and Attributes between a Looker Instance or for Version Control [instructions](https://github.com/looker-open-source/lmanage/tree/main/instructions/looker_settings_capture.md) |
17 | | Planned | scoper | Takes in a model file, elminates the * includes, iterate through the explores and joins and creates a fully scoped model include list for validation performance and best practice code organization |
18 | | Planned | removeuser | Based on last time logged in, prune Looker users to ensure a performant, compliant Looker instance |
19 | | Planned | [mapview](https://github.com/looker-open-source/lmanage/tree/main/instructions/mapview_README.md) | Find the LookML fields and tables that are associated with a piece of Looker content |
20 |
21 | #### help and version
22 | ```
23 | lmanage --help
24 | Usage: lmanage [OPTIONS] COMMAND [ARGS]...
25 |
26 | Options:
27 | --version Show the version and exit.
28 | --help Show this message and exit.
29 |
30 | Commands:
31 | capturator
32 | configurator
33 | ```
34 | #### Looker Object Migrator
35 | The object migrator allows you to preserve a point in time representation of your Looker content (Looks and Dashboards), Folder structure, Content access settings, User groups, User roles, User Attributes and preserve these as a Yaml file. This tool then lets you configure a new instance based on that Yaml file.
36 |
37 | [instructions](https://github.com/looker-open-source/lmanage/tree/main/instructions/looker_settings_capture.md)
38 |
39 |
40 | **This is not an officially supported Google Product.**
41 |
--------------------------------------------------------------------------------
/dist/lmanage-0.1.0-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.0-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.0.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.1-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.1-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.1.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.2-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.2-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.2.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.3-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.3-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.3.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.3.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.4-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.4-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.4.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.4.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.5-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.5-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.5.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.5.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.6-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.6-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.6.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.6.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.7-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.7-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.7.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.7.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.8-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.8-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.8.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.8.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.1.9-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.9-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.1.9.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.1.9.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.0-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.0-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.0.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.1-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.1-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.1.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.2-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.2-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.2.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.3-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.3-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.3.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.3.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.4-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.4-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.4.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.4.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.5-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.5-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.5.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.5.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.6.1-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.1-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.6.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.1.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.6.2-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.2-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.6.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.2.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.6.3-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.3-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.6.3.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.3.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.6.4-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.4-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.6.4.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.6.4.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.7-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.7-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.7.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.7.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.8-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.8-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.8.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.8.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.81-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.81-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.81.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.81.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.82-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.82-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.82.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.82.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.83-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.83-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.83.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.83.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.9-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.9-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.9.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.9.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.91-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.91-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.91.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.91.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.2.92-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.92-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.2.92.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.2.92.tar.gz
--------------------------------------------------------------------------------
/dist/lmanage-0.3.0-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.3.0-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/lmanage-0.3.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/dist/lmanage-0.3.0.tar.gz
--------------------------------------------------------------------------------
/file.log:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/file.log
--------------------------------------------------------------------------------
/images/hermes.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/hermes.png
--------------------------------------------------------------------------------
/images/lmanage_folder_diagram.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/lmanage_folder_diagram.png
--------------------------------------------------------------------------------
/images/lmanage_mapview_chart.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/lmanage_mapview_chart.pdf
--------------------------------------------------------------------------------
/images/mapview_walkthru.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/mapview_walkthru.jpeg
--------------------------------------------------------------------------------
/images/my_diagram.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/my_diagram.png
--------------------------------------------------------------------------------
/images/web_service.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/images/web_service.png
--------------------------------------------------------------------------------
/instructions/looker_settings_capture.md:
--------------------------------------------------------------------------------
1 | # Looker Instance Object Migrator Tool
2 | ## The tool is opinionated in functionality by not migrating content or settings that are established on a per user basis.
3 |
4 | The Looker Object Migrator Tool is an API based CLI tool to capture Looker Objects preserving a point in time snapshot as a text based file. This file can then be version controlled and used to configure or amend existing objects in a Looker instance.
5 |
6 | The advantages of having Looker Objects set out in a text file are numerous, for example:
7 | - version controlling Looker objects such as content or Folder structure and permissions (or even setting up automated processes such as gitops)
8 | - massively reducing number of clicks required to set up an instance from scratch, or revert an instance back to desired state
9 | - defining one source of truth across multiple looker instances, for instance shards
10 | - having a clearly defined security permission doc that can interact with other services such as SAML.
11 |
12 | The Object Migrator Tool is comprised of two commands a capture tool and configure tool. The tools are designed to work in concert, i.e. the configure tool is expecting the output of the capture tool, although it's entirely possible to use them individually.
13 |
14 | ## Impacted Settings
15 | The Looker Instance settings that are impacted by these tools are:
16 | - User Attributes (creation and value setting on groups)
17 | - Nested Folder Structure
18 | - Looker Content Access Settings
19 | - User Groups
20 | - User Roles
21 | - User Permission Sets
22 | - User Model Sets
23 | - Looker Content (looks and dashboards)
24 | - Looker Schedules and Alerts
25 |
26 | ## Typical Workflow
27 |
28 | 1. Run the LManage Capturator commmand to generate a Yaml file [example yaml file](#)
29 | 2. Make sure you understand the existing settings,
30 | 3. Check Yaml file into a version control system of choice
31 | 4. Restore Looker system settings using the LManage Configurator command
32 |
33 | #### [Capture Tool aka Capturator specific documentation](https://github.com/looker-open-source/lmanage/blob/main/instructions/capturator_README.md)
34 | #### [Configure Tool aka Configurator specific documentation](https://github.com/looker-open-source/lmanage/blob/main/instructions/configurator_README.md)
35 |
36 |
37 |
38 |
--------------------------------------------------------------------------------
/instructions/mapview_README.md:
--------------------------------------------------------------------------------
1 | # mapview
2 | # This is currently a legacy tool and has been deliberately broken while undergoing a rewrite.
3 | The mapview command will find the etymology of the content on your dashboard, exporting a CSV that looks like [this](https://docs.google.com/spreadsheets/d/1TzeJW46ml0uzO9RdLOOLxwtvUWjhmZxoa-xq4pbznV0/edit?resourcekey=0-xbWC87hXYFNgy1As06NncA#gid=900312158).
4 |
5 | ##### example usage
6 | `lmanage mapview --path ./output/my_output.csv --ini-file ~/py/projects/ini/k8.ini --project /test_lookml_files/the_look -table "order_items"`
7 | ##### flags
8 | - **lookml_file_path** (`--lookml_file_path`, `-lfp`) This is the path where you would like the outputfile for your returned dataset to be.
9 | - **ini-file** (`--ini-file`, `-i`) This is the file path to the ini file that you've created to use the Looker SDK
10 | - **output_path** (`--output_path`, `-op`) File path to export an xlsx tabbed file of your results input your file path to save a tabbed xls of results
11 | ```
12 | #example Ini file
13 | [Looker_Instance]
14 | base_url=https://looker-dev.company.com:19999 (or 443 if hosted in GCP)
15 | client_id=abc
16 | client_secret=xyz
17 | verify_ssl=True
18 | ```
19 |
20 | 
21 |
22 |
23 | ## Excel Tabbed Returns
24 | ### DashboardToLookMLMapping
25 | - **used_views**: the view_names extracted from the query
26 | - **matched_fields**: the fields matched with inputted lookml
27 | - **dashboard**: the id of the looker dashboard
28 | - **element_id**: the id of the visualization element on the looker dashboard
29 |
30 | ### MostUsedDimensions
31 | - **matched_fields**: the fields matched with inputted lookml
32 | - **dashboard_count**: how many dashboards are used by this field
33 |
34 | ### MostUsedViews
35 | - **used_views**: the views extracted from the query
36 | - **dashboard_count**: how many dashboards are used by this field
37 |
38 | ##### n.b.
39 | **Multi Project Usage**
40 | Dashboards can hold tiles from multiple projects, in this case if you create one local folder of lookml see example below, then pass the value of that one meta folder the the `--project` flag. Doing this will enable the underlying LookML parsing engine driven by [pyLookML](https://github.com/llooker/pylookml) to iterate over all the relevant files and find the appropriate cross project matches.
41 |
42 | ```
43 | ├── test_lookml_files
44 | │ ├── dashboards
45 | │ │ ├── brand_lookup.dashboard.lookml
46 | │ │ ├── business_pulse.dashboard.lookml
47 | │ │ ├── customer_lookup.dashboard.lookml
48 | │ ├── models_proj1
49 | │ │ └── thelook.model.lkml
50 | │ ├── models_proj2
51 | │ │ └── thelook_redshift.model.lkml
52 | │ ├── view_proj1
53 | │ │ ├── 01_order_items.view.lkml
54 | │ │ ├── 02_users.view.lkml
55 | │ │ ├── 03_inventory_items.view.lkml
56 | │ │ ├── 04_products.view.lkml
57 | │ │ ├── 05_distribution_centers.view.lkml
58 | │ └── view_proj2
59 | │ ├── 01_order_items.view.lkml
60 | │ ├── 02_users.view.lkml
61 | │ ├── 03_inventory_items.view.lkml
62 | │ ├── 04_products.view.lkml
63 | │ ├── 05_distribution_centers.view.lkml
64 | │ ├── 11_order_facts.view.lkml
65 | │ ├── 12_user_order_facts.view.lkml
66 | │ ├── 13_repeat_purchase_facts.view.lkml
67 | │ ├── 22_affinity.view.lkml
68 | │ ├── 25_trailing_sales_snapshot.view.lkml
69 | │ ├── 51_events.view.lkml
70 | │ ├── explores.lkml
71 | │ └── test_ndt.view.lkml.
72 | ```
73 |
74 |
75 |
76 | **This is not an officially supported Google Product.**
77 |
--------------------------------------------------------------------------------
/lmanage/.vimspector.json:
--------------------------------------------------------------------------------
1 | {
2 | "configurations": {
3 | "capturator": {
4 | "adapter": "debugpy",
5 | "configuration": {
6 | "python": "/usr/local/google/home/hugoselbie/.cache/pypoetry/virtualenvs/lmanage-UKKoJ9La-py3.8/bin/python",
7 | "request": "launch",
8 | "protocol": "auto",
9 | "stopOnEntry": true,
10 | "console": "integratedTerminal",
11 | "program": "${workspaceRoot}/capture_instance_permission_structure.py",
12 | "cwd": "${workspaceRoot}"
13 | }
14 | },
15 | "configuration": {
16 | "adapter": "debugpy",
17 | "configuration": {
18 | "python": "/usr/local/google/home/hugoselbie/.cache/pypoetry/virtualenvs/lmanage-UKKoJ9La-py3.8/bin/python",
19 | "request": "launch",
20 | "protocol": "auto",
21 | "stopOnEntry": true,
22 | "console": "integratedTerminal",
23 | "program": "${workspaceRoot}/provision_instance_permission_structure.py",
24 | "cwd": "${workspaceRoot}"
25 | }
26 | }
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/lmanage/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | __version__ = '0.1.4'
16 |
--------------------------------------------------------------------------------
/lmanage/capturator/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | __version__ = '0.1.4'
16 |
--------------------------------------------------------------------------------
/lmanage/capturator/content_capturation/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/capturator/content_capturation/__init__.py
--------------------------------------------------------------------------------
/lmanage/capturator/content_capturation/board_capture.py:
--------------------------------------------------------------------------------
1 | from tqdm import tqdm
2 | import logging
3 | from lmanage.utils.errorhandling import return_sleep_message
4 | from lmanage.utils.looker_object_constructors import BoardObject
5 | from lmanage.utils import logger_creation as log_color
6 | from yaspin import yaspin
7 | #logger = log_color.init_logger(__name__, logger_level)
8 |
9 |
10 | class CaptureBoards():
11 | def __init__(self, sdk):
12 | self.sdk = sdk
13 |
14 | def handle_board_structure(self, board_response: dict) -> dict:
15 | board_section_list = board_response.board_sections
16 | bs_list = []
17 |
18 | for board_section in board_section_list:
19 | board_item_list = board_section.board_items
20 | new_bi_list = [board_item.__dict__ for board_item in board_item_list]
21 | board_section['board_items'] = new_bi_list
22 | bs_dict = board_section.__dict__
23 | bs_list.append(bs_dict)
24 |
25 | board_response['board_sections'] = bs_list
26 |
27 | return board_response
28 |
29 |
30 | def get_all_boards(self) -> dict:
31 | response = []
32 | with yaspin().white.bold.shark.on_blue as sp:
33 | sp.text="getting all system board metadata (can take a while)"
34 | all_system_boards = self.sdk.all_boards()
35 |
36 | for board in all_system_boards:
37 | bresponse = self.handle_board_structure(board_response=board)
38 | b = BoardObject(
39 | content_metadata_id=board.content_metadata_id,
40 | section_order=board.section_order,
41 | title=board.title,
42 | primary_homepage=board.primary_homepage,
43 | description=board.description,
44 | board_sections=bresponse.board_sections
45 | )
46 | response.append(b)
47 | return response
48 |
49 |
50 | def execute(self):
51 | all_boards = self.get_all_boards()
52 | return all_boards
--------------------------------------------------------------------------------
/lmanage/capturator/content_capturation/dashboard_capture.py:
--------------------------------------------------------------------------------
1 | from tqdm import tqdm
2 | import logging
3 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color
4 | from tenacity import retry, wait_fixed, wait_random, stop_after_attempt
5 | from yaspin import yaspin
6 | from looker_sdk import error
7 | from collections import Counter
8 |
9 | class CaptureDashboards():
10 | def __init__(self, sdk, content_folders: dict, logger):
11 | self.sdk = sdk
12 | self.content_folders = content_folders
13 | self.logger = logger
14 | self.all_alerts = self.sdk.search_alerts(all_owners=True)
15 | all_dashboard_elements_with_alerts = [alert.dashboard_element_id for alert in self.all_alerts]
16 | self.dashboard_element_alert_counts = dict(Counter(all_dashboard_elements_with_alerts))
17 |
18 | def get_all_dashboards(self) -> dict:
19 | scrub_dashboards = {}
20 | with yaspin().white.bold.shark.on_blue as sp:
21 | sp.text = "getting all system dashboard metadata (can take a while)"
22 | all_dashboards = self.sdk.all_dashboards(fields="id,folder,slug")
23 |
24 | for dash in all_dashboards:
25 | self.logger.debug(self.content_folders)
26 | if dash.folder.id in self.content_folders:
27 | scrub_dashboards[dash.id] = {}
28 | scrub_dashboards[dash.id]['folder_id'] = dash.folder.id
29 | scrub_dashboards[dash.id]['slug'] = dash.slug
30 | # logger.info(scrub_dashboards)
31 | return scrub_dashboards
32 |
33 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
34 | def get_looker_dashboard(self, looker_dashboard_id, fields='dashboard_elements'):
35 | dashboard_elements = self.sdk.dashboard(
36 | looker_dashboard_id, fields)
37 | return dashboard_elements
38 |
39 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
40 | def get_looker_schedule_plan_for_dashboard(self, looker_dashboard_id):
41 | dash_schedule_plans = self.sdk.scheduled_plans_for_dashboard(
42 | looker_dashboard_id, all_users=True)
43 | return dash_schedule_plans
44 |
45 | def get_dashboard_lookml(self, all_dashboards: dict) -> list:
46 | # logging.info("Beginning Dashboard Capture:")
47 | response = []
48 | for dashboard_id in tqdm(all_dashboards, desc="Dashboard Capture", unit="dashboards", colour="#2c8558"):
49 | lookml = None
50 | trys = 0
51 | if "::" in dashboard_id:
52 | continue
53 | else:
54 | dashboard_elements = self.get_looker_dashboard(
55 | looker_dashboard_id=dashboard_id)['dashboard_elements']
56 | dashboard_element_ids = [e['id'] for e in dashboard_elements]
57 | dashboard_element_alert_counts = [self.dashboard_element_alert_counts.get(id, 0) for id in dashboard_element_ids]
58 | scheduled_plans = self.get_looker_schedule_plan_for_dashboard(
59 | looker_dashboard_id=dashboard_id)
60 | alerts = [
61 | loc.AlertObject(alert) for alert in self.all_alerts if alert['dashboard_element_id'] in dashboard_element_ids]
62 |
63 | while lookml is None and trys < 5:
64 | trys += 1
65 | try:
66 | lookml = self.sdk.dashboard_lookml(
67 | dashboard_id=dashboard_id)
68 | except error.SDKError as e:
69 | self.logger.error(f"LookML download error for dashboard {dashboard_id}")
70 | eh.return_sleep_message(call_number=trys, quiet=True)
71 |
72 | if lookml:
73 | self.logger.debug(lookml.lookml)
74 | captured_dashboard = loc.DashboardObject(
75 | legacy_folder_id=all_dashboards.get(dashboard_id),
76 | lookml=lookml.lookml,
77 | dashboard_id=dashboard_id,
78 | dashboard_slug=all_dashboards.get(
79 | dashboard_id).get('slug'),
80 | dashboard_element_alert_counts=dashboard_element_alert_counts,
81 | scheduled_plans=scheduled_plans,
82 | alerts=alerts
83 | )
84 | response.append(captured_dashboard)
85 | return response
86 |
87 | def execute(self):
88 | all_dashboards = self.get_all_dashboards()
89 | captured_dash = self.get_dashboard_lookml(all_dashboards)
90 | return captured_dash
91 |
--------------------------------------------------------------------------------
/lmanage/capturator/content_capturation/look_capture.py:
--------------------------------------------------------------------------------
1 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color
2 | from tqdm import tqdm
3 | from yaspin import yaspin
4 | from tenacity import retry, wait_fixed, wait_random, stop_after_attempt
5 |
6 | # logger = log_color.init_logger(__name__, logger_level)
7 |
8 |
9 | class LookCapture:
10 | def __init__(self, sdk, content_folders, logger):
11 | self.sdk = sdk
12 | self.content_folders = content_folders
13 | self.logger = logger
14 |
15 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
16 | def get_all_looks_metadata(self) -> list:
17 | response = self.sdk.all_looks(fields='id,folder')
18 | return response
19 |
20 | def all_looks(self):
21 | system_folders = ['Users', 'Embed Users', 'Embed Groups']
22 | all_look_meta = None
23 | with yaspin().white.bold.shark.on_blue as sp:
24 | sp.text = "getting all system look metadata (can take a while)"
25 | all_look_meta = self.get_all_looks_metadata()
26 |
27 | scrub_looks = {}
28 | for look in all_look_meta:
29 | if look.folder.id in self.content_folders or look.folder.id == '1':
30 | scrub_looks[look.id] = look.folder.id
31 | else:
32 | continue
33 |
34 | return scrub_looks
35 |
36 | def get_look_metadata(self, look_id: str) -> dict:
37 | look_meta = None
38 | trys = 0
39 | while look_meta is None:
40 | try:
41 | look_meta = self.sdk.look(look_id=look_id)
42 | except:
43 | eh.return_sleep_message
44 | return look_meta
45 |
46 | def clean_query_obj(self, query_metadata: dict) -> dict:
47 | metadata_keep_keys = [
48 | "model",
49 | "view",
50 | "fields",
51 | "pivots",
52 | "fill_fields",
53 | "filters",
54 | "filter_expression",
55 | "sorts",
56 | "limit",
57 | "column_limit",
58 | "total",
59 | "row_total",
60 | "subtotals",
61 | "vis_config",
62 | "filter_config",
63 | "visible_ui_sections",
64 | "dynamic_fields",
65 | "query_timezone"]
66 | restricted_look_metadata = dict(
67 | (k, query_metadata[k]) for k in metadata_keep_keys)
68 | return restricted_look_metadata
69 |
70 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
71 | def get_scheduled_plans_for_look(self, looker_look_id):
72 | return self.sdk.scheduled_plans_for_look(
73 | look_id=looker_look_id, all_users=True)
74 |
75 | def execute(self):
76 | '''
77 | 1. get all the looks and extract the id's
78 | 2. iterate through the looks and extract metadata, only keep necessary fields to make a query for look transference
79 | 3. create custom look object and add to list
80 | '''
81 | all_look_data = self.all_looks()
82 | looks = []
83 | content = 1
84 | for look in tqdm(all_look_data, desc="Look Capture", unit=" looks", colour="#2c8558"):
85 | lmetadata = self.get_look_metadata(look_id=look)
86 | scheduled_plans = self.get_scheduled_plans_for_look(
87 | looker_look_id=lmetadata.id)
88 | query_object = lmetadata.query.__dict__
89 | nq_obj = self.clean_query_obj(query_metadata=query_object)
90 | self.logger.debug(nq_obj)
91 |
92 | legacy_folder = lmetadata.folder_id
93 | look_id = lmetadata.id
94 | title = lmetadata.title
95 | look_obj = loc.LookObject(
96 | query_obj=nq_obj,
97 | description=lmetadata.description,
98 | legacy_folder_id=legacy_folder,
99 | look_id=look_id,
100 | title=title,
101 | scheduled_plans=scheduled_plans)
102 | looks.append(look_obj)
103 | content += 1
104 | return looks
105 |
--------------------------------------------------------------------------------
/lmanage/capturator/folder_capturation/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/capturator/folder_capturation/__init__.py
--------------------------------------------------------------------------------
/lmanage/capturator/folder_capturation/create_folder_yaml_structure.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import time
3 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color
4 |
5 | #logger = log_color.init_logger(__name__, logger_level)
6 |
7 | class CreateFolderOutput():
8 | def __init__(self, folder_list):
9 | self.folder_list = folder_list
10 |
11 | def create_folder_structure(self, folder_obj: object, data_storage: list):
12 | temp = {}
13 | temp['name'] = folder_obj.name
14 | temp['permissions'] = folder_obj.content_access_list
15 | temp['subfolder'] = folder_obj.children
16 | data_storage.append(temp)
17 |
18 | if isinstance(folder_obj.children, list):
19 | for subfolder in folder_obj.children:
20 | self.create_folder_structure(subfolder, data_storage)
21 |
22 | return data_storage
23 |
24 | def make_dict(self):
25 | x = []
26 | f = self.folder_list
27 |
28 | for folders in f:
29 | data_storage = []
30 | x = self.create_folder_structure(
31 | folder_obj=folders, data_storage=data_storage)
32 | return x
33 |
34 | def execute(self):
35 | '''
36 | goal: extract all the folders into the appropriate markdown folder structure
37 | '''
38 | x = self.make_dict()
39 | return x
40 |
--------------------------------------------------------------------------------
/lmanage/capturator/looker_api_reader.py:
--------------------------------------------------------------------------------
1 | from lmanage.capturator.folder_capturation import folder_config as fc
2 | from lmanage.capturator.user_group_capturation import role_config as rc
3 | from lmanage.capturator.user_attribute_capturation import capture_ua_permissions as cup
4 | from lmanage.capturator.content_capturation import dashboard_capture as dc, look_capture as lc, board_capture as bc
5 | from lmanage.looker_config import LookerConfig
6 | from lmanage.logger_config import setup_logger
7 | from lmanage.looker_auth import LookerAuth
8 |
9 | logger = setup_logger()
10 |
11 |
12 | class LookerApiReader():
13 | shared_folder_id = '1'
14 |
15 | def __init__(self, ini_file):
16 | self.sdk = LookerAuth().authenticate(ini_file)
17 |
18 | def get_config(self) -> LookerConfig:
19 | settings = self.__read_settings()
20 | content = self.__read_content()
21 | return LookerConfig(settings, content)
22 |
23 | def __read_settings(self):
24 | # Folder configuration
25 | folder_returns = fc.CaptureFolderConfig(
26 | sdk=self.sdk, logger=logger).execute()
27 | folder_structure_list = folder_returns[1]
28 | self.content_folders = [self.shared_folder_id] + folder_returns[0]
29 |
30 | # Roles, Permission Sets & Model Sets
31 | role_info = rc.ExtractRoleInfo(sdk=self.sdk, logger=logger)
32 | permission_sets = role_info.extract_permission_sets()
33 | model_sets = role_info.extract_model_sets()
34 | roles = role_info.extract_role_info()
35 |
36 | # User Attributes
37 | user_attributes = cup.ExtractUserAttributes(
38 | sdk=self.sdk, logger=logger).create_user_attributes()
39 |
40 | return {
41 | 'folder_permissions': folder_structure_list,
42 | 'permission_sets': permission_sets,
43 | 'model_sets': model_sets,
44 | 'roles': roles,
45 | 'user_attributes': user_attributes
46 | }
47 |
48 | def __read_content(self):
49 | # Boards
50 | # boards = bc.CaptureBoards(sdk=sdk).execute()
51 |
52 | # Looks
53 | looks = lc.LookCapture(
54 | sdk=self.sdk, content_folders=self.content_folders, logger=logger).execute()
55 |
56 | # Dashboards
57 | dashboards = dc.CaptureDashboards(
58 | sdk=self.sdk, content_folders=self.content_folders, logger=logger).execute()
59 |
60 | return {
61 | 'looks': looks,
62 | 'dashboards': dashboards
63 | }
64 |
--------------------------------------------------------------------------------
/lmanage/capturator/looker_config_saver.py:
--------------------------------------------------------------------------------
1 | import looker_sdk
2 | import ruamel.yaml
3 | import os
4 | from lmanage.looker_config import LookerConfig
5 | from lmanage.logger_config import setup_logger
6 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh
7 |
8 | logger = setup_logger()
9 |
10 |
11 | class LookerConfigSaver():
12 | def __init__(self, config_dir):
13 | self.__init_yaml()
14 | self.config_dir = config_dir
15 | if not os.path.exists(config_dir):
16 | os.makedirs(config_dir)
17 |
18 | def save(self, config: LookerConfig):
19 | self.__save_settings(config.settings)
20 | self.__save_content(config.content)
21 |
22 | def __init_yaml(self):
23 | self.yaml = ruamel.yaml.YAML()
24 | self.yaml.register_class(loc.LookerFolder)
25 | self.yaml.register_class(loc.LookerModelSet)
26 | self.yaml.register_class(loc.LookerPermissionSet)
27 | self.yaml.register_class(loc.LookerRoles)
28 | self.yaml.register_class(loc.LookerUserAttribute)
29 | self.yaml.register_class(loc.LookObject)
30 | self.yaml.register_class(loc.DashboardObject)
31 | self.yaml.register_class(loc.AlertObject)
32 | self.yaml.register_class(loc.AlertAppliedDashboardFilterObject)
33 | self.yaml.register_class(loc.AlertDestinationObject)
34 | self.yaml.register_class(loc.AlertFieldObject)
35 | self.yaml.register_class(loc.AlertFieldFilterObject)
36 | self.yaml.register_class(loc.BoardObject)
37 | self.yaml.register_class(looker_sdk.sdk.api40.models.ScheduledPlan)
38 | self.yaml.register_class(
39 | looker_sdk.sdk.api40.models.ScheduledPlanDestination)
40 | self.yaml.register_class(looker_sdk.sdk.api40.models.UserPublic)
41 |
42 | def __save_settings(self, settings):
43 | # Folder Permissions
44 | with open(self.__get_settings_path(), 'w') as file:
45 | fd_yml_txt = '''# FOLDER_PERMISSIONS\n# Opening Session Welcome to the Capturator, this is the Folder place\n# -----------------------------------------------------\n\n'''
46 | file.write(fd_yml_txt)
47 | self.yaml.dump(settings['folder_permissions'], file)
48 |
49 | # Roles, Permission Sets & Model Sets
50 | with open(self.__get_settings_path(), 'a') as file:
51 | r_yml_txt = '''# Looker Role\n# Opening Session Welcome to the Capturator, this is the Role place\n# -----------------------------------------------------\n\n'''
52 | file.write(r_yml_txt)
53 | file.write('\n\n# PERMISSION SETS\n')
54 | self.yaml.dump(settings['permission_sets'], file)
55 | file.write('\n\n# MODEL SETS\n')
56 | self.yaml.dump(settings['model_sets'], file)
57 | file.write('\n\n# LOOKER ROLES\n')
58 | self.yaml.dump(settings['roles'], file)
59 |
60 | # User Attributes
61 | with open(self.__get_settings_path(), 'a') as file:
62 | file.write('\n\n# USER_ATTRIBUTES\n')
63 | self.yaml.dump(settings['user_attributes'], file)
64 |
65 | def __save_content(self, content):
66 | # Boards
67 | # with open(self.__get_content_path(), 'w+') as file:
68 | # file.write('\n\n# BoardData\n')
69 | # self.yaml.dump(self.content.boards, file)
70 |
71 | # Looks
72 | looks = content['looks']
73 | with open(self.__get_content_path(), 'wb') as file:
74 | file.write(bytes('\n\n# LookData\n', 'utf-8'))
75 | if eh.test_object_data(looks):
76 | self.yaml.dump(looks, file)
77 | else:
78 | file.write(bytes('#No Captured Looks', 'utf-8'))
79 |
80 | # Dashboards
81 | dashboards = content['dashboards']
82 | with open(self.__get_content_path(), 'ab') as file:
83 | file.write(bytes('\n\n# Dashboard Content\n', 'utf-8'))
84 | if eh.test_object_data(dashboards):
85 | self.yaml.dump(dashboards, file)
86 | else:
87 | file.write(bytes('#No Captured Dashboards', 'utf-8'))
88 |
89 | def __get_settings_path(self):
90 | return f'{self.config_dir}/settings.yaml'
91 |
92 | def __get_content_path(self):
93 | return f'{self.config_dir}/content.yaml'
94 |
--------------------------------------------------------------------------------
/lmanage/capturator/user_attribute_capturation/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/capturator/user_attribute_capturation/__init__.py
--------------------------------------------------------------------------------
/lmanage/capturator/user_attribute_capturation/capture_ua_permissions.py:
--------------------------------------------------------------------------------
1 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color
2 | from tqdm import tqdm
3 | from tenacity import retry, wait_fixed, wait_random, stop_after_attempt
4 |
5 | class ExtractUserAttributes():
6 | def __init__(self, sdk, logger):
7 | self.sdk = sdk
8 | self.logger = logger
9 | self.user_attribute_metadata = self.existing_user_attributes()
10 | self.all_group_metadata = self.all_group_metadatas()
11 |
12 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
13 | def get_all_user_attributes(self) -> dict:
14 | response = self.sdk.all_user_attributes()
15 | return response
16 |
17 | def existing_user_attributes(self) -> dict:
18 | ex_ua = self.get_all_user_attributes()
19 | resp = [ua for ua in ex_ua if not ua.is_system]
20 | return resp
21 |
22 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
23 | def all_group_metadatas(self):
24 | response=self.sdk.all_groups()
25 | return response
26 |
27 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
28 | def all_user_attribute_group_value_metadata(self, uaid: int):
29 | response = self.sdk.all_user_attribute_group_values(uaid)
30 | return response
31 |
32 |
33 | def create_user_attributes(self):
34 | group_metadata = {
35 | group.id: group.name for group in self.all_group_metadata}
36 | response = []
37 | for ua in tqdm(self.user_attribute_metadata, desc = "User Attribute Capture", unit=" user att", colour="#2c8558"):
38 | group_assign = self.all_user_attribute_group_value_metadata(uaid=ua.id)
39 | team_values = []
40 |
41 | for group in group_assign:
42 | group_name = group_metadata.get(group.group_id)
43 | teams = {}
44 | teams[group_name] = group.value
45 | team_values.append(teams)
46 | looker_ua = loc.LookerUserAttribute(
47 | name=ua.name,
48 | uatype=ua.type,
49 | hidden_value=ua.value_is_hidden,
50 | user_view=ua.user_can_view,
51 | user_edit=ua.user_can_edit,
52 | default_value=ua.default_value,
53 | teams_val=team_values
54 | )
55 | response.append(looker_ua)
56 | return response
57 |
--------------------------------------------------------------------------------
/lmanage/capturator/user_group_capturation/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/capturator/user_group_capturation/__init__.py
--------------------------------------------------------------------------------
/lmanage/capturator/user_group_capturation/group_config.py:
--------------------------------------------------------------------------------
1 | from looker_sdk import error
2 | from looker_sdk.sdk.api40 import models
3 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color
4 |
5 |
6 | class GetInstanceGroups():
7 | def __init__(self, sdk, logger) -> None:
8 | self.sdk = sdk
9 | self.logger
10 |
11 | def extract_group_names(self):
12 | capture_groups = self.sdk.all_groups()
13 | group_names = [group.name for group in capture_groups]
14 |
15 | return group_names
16 |
17 | def extract_folder_names(self):
18 | folders = self.sdk.all_folders()
19 | test = {}
20 | for folder in folders:
21 | print('high', folder.name)
22 | if folder.parent_id is not None:
23 | print('low', folder.name)
24 |
25 | test[folder.name] = folder.parent_id
26 | else:
27 | print('test', folder.name)
28 |
29 | # for x, y in test.items():
30 |
31 | return test
32 |
33 | def create_group_if_not_exists(self,
34 | sdk,
35 | group_name: str) -> dict:
36 | """ Create a Looker Group and add Group attributes
37 |
38 | :group_name: Name of a Looker group to create.
39 | :rtype: Looker Group object.
40 | """
41 | # get group if exists
42 | try:
43 | self.logger.info(f'Creating group "{group_name}"')
44 | group = sdk.create_group(
45 | body=models.WriteGroup(
46 | can_add_to_content_metadata=True,
47 | name=group_name
48 | )
49 | )
50 | return group
51 | except error.SDKError as grouperr:
52 | err_msg = eh.return_error_message(grouperr)
53 | self.logger.info(
54 | 'you have an error in your group; error = %s', err_msg)
55 | self.logger.debug(grouperr)
56 | group = sdk.search_groups(name=group_name)
57 | return group[0]
58 |
59 | def get_instance_group_metadata(self,
60 | sdk,
61 | unique_group_list: list) -> list:
62 | group_metadata = []
63 |
64 | for group_name in unique_group_list:
65 | group = self.create_group_if_not_exists(sdk, group_name)
66 | temp = {}
67 | temp['group_id'] = group.id
68 | temp['group_name'] = group.name
69 | group_metadata.append(temp)
70 |
71 | return group_metadata
72 |
73 | def sync_groups(self,
74 | sdk,
75 | group_name_list: list) -> str:
76 |
77 | all_groups = sdk.all_groups()
78 | group_dict = {group.name: group.id for group in all_groups}
79 | # Deleting Standard Groups
80 | del group_dict['All Users']
81 |
82 | for group_name in group_dict.keys():
83 | if group_name not in group_name_list:
84 | sdk.delete_group(group_id=group_dict[group_name])
85 | self.logger.info(
86 | f'deleting group {group_name} to sync with yaml config')
87 |
88 | return 'your groups are in sync with your yaml file'
89 |
90 | def execute(self):
91 | team_list = []
92 | # extracting user attribute teams
93 | self.extract_teams(container=self.user_attribute_metadata,
94 | data_storage=team_list)
95 | # extracting role based teams
96 | self.extract_teams(
97 | container=self.role_metadata, data_storage=team_list)
98 | # extract nested folder access teams
99 | self.extract_folder_teams(
100 | container=self.folder_metadata, data_storage=team_list)
101 |
102 | team_list = list(set(team_list))
103 |
104 | # create all the groups
105 | self.get_instance_group_metadata(
106 | sdk=self.sdk, unique_group_list=team_list)
107 |
--------------------------------------------------------------------------------
/lmanage/capturator/user_group_capturation/role_config.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from tqdm import tqdm
3 | from lmanage.utils import looker_object_constructors as loc, errorhandling as eh, logger_creation as log_color
4 |
5 | # logger = log_color.init_logger(__name__, logger_level)
6 |
7 |
8 | class ExtractRoleInfo():
9 | def __init__(self, sdk, logger):
10 | self.sdk = sdk
11 | self.logger = logger
12 | self.role_base = self.get_all_roles()
13 |
14 | def get_all_roles(self):
15 | sdk = self.sdk
16 | response = sdk.all_roles()
17 | for role in enumerate(response):
18 |
19 | if role[1].name == 'Admin':
20 | response.pop(role[0])
21 |
22 | return response
23 |
24 | def create_list_of_permission_sets(self):
25 | response = []
26 | for role in self.role_base:
27 | if role.permission_set is None:
28 | raise Exception(
29 | f'Role name {role.name} has no permission_set, please add one to be captured or delete the role.')
30 | temp = {}
31 | temp['name'] = role.permission_set.name
32 | temp['permissions'] = role.permission_set.permissions
33 | response.append(temp)
34 | return response
35 |
36 | def extract_permission_sets(self):
37 | response = []
38 | p_list = eh.dedup_list_of_dicts(
39 | self.create_list_of_permission_sets())
40 | for role in p_list:
41 | self.logger.debug(role)
42 | perm_set = loc.LookerPermissionSet(
43 | name=role.get('name'),
44 | permissions=role.get('permissions'))
45 | response.append(perm_set)
46 |
47 | return response
48 |
49 | def create_list_of_roles(self):
50 | response = []
51 | for role in self.role_base:
52 | temp = {}
53 | temp['name'] = role.model_set.name
54 | temp['models'] = role.model_set.models
55 | response.append(temp)
56 | return response
57 |
58 | def extract_model_sets(self):
59 | response = []
60 | role_list = eh.dedup_list_of_dicts(
61 | self.create_list_of_roles())
62 | for role in role_list:
63 | model_set = loc.LookerModelSet(
64 | models=role.get('models'), name=role.get('name'))
65 | response.append(model_set)
66 | return response
67 |
68 | def extract_role_info(self):
69 | response = []
70 | for role in tqdm(self.role_base, desc='Role Capture', unit=' roles', colour='#2c8558'):
71 | groups = None
72 | trys = 0
73 | while groups is None:
74 | trys += 1
75 | try:
76 | groups = self.sdk.role_groups(role_id=role.id)
77 | except:
78 | eh.return_sleep_message(call_number=trys)
79 | role_groups = [group.name for group in groups]
80 | lookerrole = loc.LookerRoles(permission_set=role.permission_set.name,
81 | model_set=role.model_set.name, teams=role_groups, name=role.name)
82 | response.append(lookerrole)
83 |
84 | return response
85 |
--------------------------------------------------------------------------------
/lmanage/capturator/user_group_capturation/user_permission.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import looker_sdk
3 | from looker_sdk.sdk.api40 import models
4 | from role_config import CreateRoleBase
5 | from lmanage.utils import logger_creation as log_color
6 |
7 | logging.setLoggerClass(log_color.ColoredLogger)
8 | logger = logging.getLogger(__name__)
9 |
10 |
11 | class CreateInstanceRoles(CreateRoleBase):
12 | def __init__(self, roles, sdk):
13 | self.role_metadata = roles
14 | self.sdk = sdk
15 |
16 | def create_permission_lookup(self):
17 | all_permission_sets = self.get_all_permission_sets()
18 | instance_permission_set_dict = {
19 | perm.name: perm.id for perm in all_permission_sets}
20 | return instance_permission_set_dict
21 |
22 | def create_model_lookup(self):
23 | all_model_sets = self.get_all_model_sets()
24 | instance_model_set_dict = {
25 | model.name: model.id for model in all_model_sets}
26 | return instance_model_set_dict
27 |
28 | def create_allgroup_lookup(self):
29 | all_groups = self.sdk.all_groups()
30 | group_dict = {group.name: group.id for group in all_groups}
31 | return group_dict
32 |
33 | def create_allrole_lookup(self):
34 | all_roles = self.sdk.all_roles()
35 | role_dict = {role.name: role.id for role in all_roles}
36 | return role_dict
37 |
38 | def create_instance_roles(self):
39 | model_lookup = self.create_model_lookup()
40 | permission_lookup = self.create_permission_lookup()
41 |
42 | role_output = []
43 |
44 | for role_name, metadata in self.role_metadata.items():
45 | model_set_id = model_lookup.get(metadata.get('model_set'))
46 | permission_set_id = permission_lookup.get(
47 | metadata.get('permission_set'))
48 |
49 | body = models.WriteRole(
50 | name=role_name,
51 | permission_set_id=permission_set_id,
52 | model_set_id=model_set_id
53 | )
54 | try:
55 | role = self.sdk.create_role(
56 | body=body
57 | )
58 | except looker_sdk.error.SDKError as roleerror:
59 | logger.debug(roleerror)
60 | role_id = self.sdk.search_roles(name=role_name)[0].id
61 | role = self.sdk.update_role(role_id=role_id, body=body)
62 | temp = {}
63 | temp['role_id'] = role.id
64 | temp['role_name'] = role_name
65 | role_output.append(temp)
66 | logger.info(role_output)
67 | return role_output
68 |
69 | def set_role(self, role_id: str, group_id: list) -> str:
70 | try:
71 | self.sdk.set_role_groups(role_id, group_id)
72 | return logger.info(f'attributing {group_id} permissions on instance')
73 | except looker_sdk.error.SDKError:
74 | return logger.info('something went wrong')
75 |
76 | def attribute_instance_roles(self, created_role_metadata: list):
77 | role_lookup = self.create_allrole_lookup()
78 | group_lookup = self.create_allgroup_lookup()
79 |
80 | for role_name, metadata in self.role_metadata.items():
81 | teams = metadata.get('team')
82 | role_id = role_lookup.get(role_name)
83 | group_id_list = []
84 | for team in teams:
85 | group_id = group_lookup.get(team)
86 | group_id_list.append(group_id)
87 | self.set_role(role_id=role_id, group_id=group_id_list)
88 |
89 | def sync_roles(self):
90 | all_role_lookup = self.create_allrole_lookup()
91 | all_role_lookup.pop('Admin')
92 | yaml_role = [role for role in self.role_metadata]
93 | logger.debug(all_role_lookup)
94 |
95 | for role_name in all_role_lookup.keys():
96 | if role_name not in yaml_role:
97 | role_id = self.sdk.search_roles(name=role_name)[0].id
98 | self.sdk.delete_role(role_id=role_id)
99 |
100 | def execute(self):
101 | created_roles = self.create_instance_roles()
102 | self.attribute_instance_roles(created_role_metadata=created_roles)
103 | self.sync_roles()
104 |
--------------------------------------------------------------------------------
/lmanage/cli.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #
3 | # Copyright 2021 Google LLC
4 | #
5 | # Licensed under the Apache License, Version 2.0 (the "License");
6 | # you may not use this file except in compliance with the License.
7 | # You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | import click
17 | import configparser
18 | import os
19 | from functools import wraps
20 | from lmanage.capturator.looker_api_reader import LookerApiReader
21 | from lmanage.capturator.looker_config_saver import LookerConfigSaver
22 | from lmanage.configurator.looker_config_reader import LookerConfigReader
23 | from lmanage.configurator.looker_provisioner import LookerProvisioner
24 | from lmanage.utils import logger_creation as log_color
25 |
26 | # logger = log_color.init_logger(__name__, testing_mode=False)
27 |
28 |
29 | @click.group()
30 | @click.version_option()
31 | def lmanage():
32 | pass
33 |
34 |
35 | def log_args(**kwargs):
36 | log_level = kwargs.get('verbose', False)
37 | logger = log_color.init_logger(__name__, testing_mode=log_level)
38 | for k, v in kwargs.items():
39 | if {v} != None:
40 | logger.info(
41 | f'You have set {v} for your {k} variable')
42 | else:
43 | logger.debug(
44 | f'There is no value set for {k} please use the `--help` flag to see input parameters.')
45 |
46 |
47 | def clean_args(**kwargs):
48 | kwargs['config_dir'] = kwargs['config_dir'].rstrip('/')
49 | return kwargs
50 |
51 |
52 | def common_options(f):
53 | @click.option('-i', '--ini-file',
54 | help='Specify API Credentials in an ini file, if no path is given program will assume these values are set as environmental variables as denoted at https://github.com/looker-open-source/sdk-codegen#environment-variable-configuration.')
55 | @click.option('-cd', '--config-dir',
56 | help='Where to save the YAML files to use for instance configuration.', required=True)
57 | @click.option('-v', '--verbose',
58 | is_flag=True,
59 | help='Add this flag to get a more verbose version of the returned stout text.')
60 | @click.option('-f', '--force',
61 | is_flag=True,
62 | help='Add this flag to skip confirmation prompt for capturing and configuring instance.')
63 | @click.pass_context
64 | @wraps(f)
65 | def decorated_function(ctx, *args, **kwargs):
66 | log_args(**kwargs)
67 | kwargs = clean_args(**kwargs)
68 | return ctx.invoke(f, *args, **kwargs)
69 | return decorated_function
70 |
71 |
72 | @lmanage.command()
73 | @common_options
74 | def capturator(**kwargs):
75 | """Captures security settings for your looker instance"""
76 | ini_file, config_dir, verbose, force = kwargs.get('ini_file'), kwargs.get(
77 | 'config_dir'), kwargs.get('verbose'), kwargs.get('force')
78 |
79 | try:
80 | target_url = get_target_url(ini_file)
81 | if force or (not force and click.confirm(f'\nYou are about to capture settings and content from instance {target_url}. Proceed?')):
82 | config = LookerApiReader(ini_file).get_config()
83 | LookerConfigSaver(config_dir).save(config)
84 | except RuntimeError as e:
85 | print(e)
86 |
87 |
88 | @lmanage.command()
89 | @common_options
90 | def configurator(**kwargs):
91 | """Configures security settings for your looker instance"""
92 | ini_file, config_dir, verbose, force = kwargs.get('ini_file'), kwargs.get(
93 | 'config_dir'), kwargs.get('verbose'), kwargs.get('force')
94 |
95 | try:
96 | target_url = get_target_url(ini_file)
97 | config_reader = LookerConfigReader(config_dir)
98 | config_reader.read()
99 | summary = config_reader.get_summary()
100 | if force or (not force and click.confirm(f'\nYou are about configure instance {target_url} with:\n{summary}\nAre you sure you want to proceed?')):
101 | LookerProvisioner(ini_file, verbose).provision(config_reader.config)
102 | except RuntimeError as e:
103 | print(e)
104 |
105 |
106 | def get_target_url(ini_file):
107 | if ini_file:
108 | config_parser = configparser.ConfigParser()
109 | successful_read = config_parser.read(ini_file)
110 |
111 | if not successful_read:
112 | raise FileNotFoundError(
113 | f'Configuration file "{ini_file}" not found.')
114 |
115 | sections = config_parser.sections()
116 | return config_parser.get(sections[0], 'base_url')
117 | else:
118 | target_url = os.getenv('LOOKERSDK_BASE_URL')
119 | if target_url is None:
120 | raise ValueError(
121 | 'LOOKERSDK_BASE_URL environment variable not set.')
122 |
123 |
124 | if __name__ == '__main__':
125 | lmanage()
126 |
--------------------------------------------------------------------------------
/lmanage/configurator/.vimspector.json:
--------------------------------------------------------------------------------
1 | {
2 | "configurations": {
3 | "run": {
4 | "adapter": "debugpy",
5 | "configuration": {
6 | "request": "launch",
7 | "protocol": "auto",
8 | "stopOnEntry": true,
9 | "console": "integratedTerminal",
10 | "program": "${workspaceRoot}/provision_instance_permission_structure.py",
11 | "cwd": "${workspaceRoot}"
12 | }
13 | }
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/lmanage/configurator/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | __version__ = '0.1.4'
16 |
--------------------------------------------------------------------------------
/lmanage/configurator/content_configuration/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/configurator/content_configuration/__init__.py
--------------------------------------------------------------------------------
/lmanage/configurator/content_configuration/clean_instance_content.py:
--------------------------------------------------------------------------------
1 | from tqdm import tqdm
2 | from lmanage.configurator.create_object import CreateObject
3 |
4 |
5 | class CleanInstanceContent(CreateObject):
6 | def __init__(self, sdk, logger) -> None:
7 | self.sdk = sdk
8 | self.logger = logger
9 |
10 | def execute(self):
11 | # self.__delete_boards()
12 | self.__delete_dashboards()
13 | self.__delete_looks()
14 | self.__delete_scheduled_plans()
15 | self.__empty_dashboard_trash()
16 | self.__empty_look_trash()
17 |
18 | def __delete_boards(self) -> None:
19 | all_boards = self.sdk.all_boards()
20 | delete_board_list = [board.id for board in all_boards]
21 | for board_id in tqdm(delete_board_list, 'Scrubbing Instance Boards', unit='boards', colour="#2c8558"):
22 | self.sdk.delete_board(board_id=board_id)
23 | self.logger.debug(
24 | f'cleaning the trash board {board_id} from instance')
25 |
26 | def __delete_dashboards(self) -> None:
27 | all_dashboards = self.sdk.all_dashboards()
28 | delete_id_list = [
29 | dash.id for dash in all_dashboards if '::' not in dash.id]
30 | for dash_id in tqdm(delete_id_list, 'Scrubbing Instance Dashboards', unit='dashboards', colour="#2c8558"):
31 | self.sdk.delete_dashboard(dashboard_id=dash_id)
32 | self.logger.debug(
33 | f'cleaning the trash dashboard {dash_id} from instance')
34 |
35 | def __delete_looks(self) -> None:
36 | all_looks = self.sdk.all_looks()
37 | delete_id_list = [look.id for look in all_looks]
38 | for look_id in tqdm(delete_id_list, 'Scrubbing Instance Looks', unit='looks', colour="#2c8558"):
39 | self.logger.debug(
40 | f'Deleting look {look_id} from instance')
41 | self.sdk.delete_look(look_id=look_id)
42 |
43 | def __delete_scheduled_plans(self) -> None:
44 | all_schedules = self.sdk.all_scheduled_plans(all_users=True)
45 | schedule_plan_id_list = [schedule.id for schedule in all_schedules]
46 | for schedule_plan_id in tqdm(schedule_plan_id_list, 'Scrubbing Instance Schedules', unit='schedules', colour="#2c8558"):
47 | self.sdk.delete_scheduled_plan(scheduled_plan_id=schedule_plan_id)
48 | self.logger.debug(
49 | f'cleaning the trash schedule {schedule_plan_id}')
50 |
51 | def __empty_dashboard_trash(self) -> None:
52 | trash_dash = self.sdk.search_dashboards(deleted=True)
53 | trash_dash_id_list = [dash.id for dash in trash_dash]
54 | for dash_id in tqdm(trash_dash_id_list, 'Emptying Dashboard Trash', unit='dashboards', colour="#2c8558"):
55 | self.sdk.delete_dashboard(dashboard_id=dash_id)
56 | self.logger.debug(
57 | f'cleaning the trash dashboard {dash_id} from instance')
58 |
59 | def __empty_look_trash(self) -> None:
60 | trash_look = self.sdk.search_looks(deleted=True)
61 | trash_look_id_list = [look.id for look in trash_look]
62 | for look_id in tqdm(trash_look_id_list, 'Emptying Look Trash', unit='Look', colour="#2c8558"):
63 | self.sdk.delete_look(look_id=look_id)
64 | self.logger.debug(
65 | f'cleaning the trash dashboard {look_id} from instance')
66 |
--------------------------------------------------------------------------------
/lmanage/configurator/content_configuration/create_boards.py:
--------------------------------------------------------------------------------
1 | from tqdm import tqdm
2 | from looker_sdk import models40 as models
3 | from lmanage.configurator.create_object import CreateObject
4 | from lmanage.utils import logger_creation as log_color
5 | # logger = log_color.init_logger(__name__, logger_level)
6 |
7 |
8 | class CreateBoards(CreateObject):
9 | def __init__(self, sdk, board_metadata, dashboard_mapping, look_mapping) -> None:
10 | self.sdk = sdk
11 | self.dashboard_mapping = dashboard_mapping
12 | self.look_mapping = look_mapping
13 | self.board_metadata = board_metadata
14 |
15 | def create_boards(self, bmeta: dict):
16 | board_body = models.WriteBoard(
17 | description=bmeta.get('description'),
18 | title=bmeta.get('title'),
19 | section_order=bmeta.get('section_order')
20 | )
21 |
22 | board = self.sdk.create_board(body=board_body)
23 |
24 | return board
25 |
26 | def create_board_section(self, section_object, board_id):
27 | new_board_section = models.WriteBoardSection(
28 | board_id=board_id,
29 | description=section_object.description,
30 | item_order=section_object.item_order,
31 | title=section_object.title
32 | )
33 | resp = self.sdk.create_board_section(body=new_board_section)
34 | return resp.id
35 |
36 | def match_dashboard_id(self, source_dashboard_id: str) -> str:
37 | pass
38 |
39 | def match_look_id(self, source_look_id: str) -> str:
40 | pass
41 |
42 | def create_board_item(self, target_board_section_id):
43 |
44 | dashboard_id = None
45 | look_id = None
46 |
47 | if target_board_section_id.dashboard_id:
48 | dashboard_id = self.match_dashboard_id(
49 | target_board_section_id.dashboard_id)
50 | if target_board_section_id.look_id:
51 | look_id = self.match_look_id(target_board_section_id.look_id)
52 |
53 | new_board_item = models.WriteBoardItem()
54 | new_board_item.__dict__.update(source_board_item_object.__dict__)
55 | new_board_item.dashboard_id = dashboard_id
56 | new_board_item.look_id = look_id
57 | new_board_item.homepage_section_id = target_board_section_id
58 |
59 | logger.info(
60 | "Creating item",
61 | extra={
62 | "section_id": new_board_item.homepage_section_id,
63 | "dashboard_id": new_board_item.dashboard_id,
64 | "look_id": new_board_item.look_id,
65 | "url": new_board_item.url
66 | }
67 | )
68 | resp = target_sdk.create_homepage_item(new_board_item)
69 | logger.info("Item created", extra={"id": resp.id})
70 |
71 | return resp
72 |
73 | def execute(self):
74 | '''
75 | create boards
76 | in boards create sections
77 | in sections create board items
78 | check content access provisions
79 | '''
80 | for board in self.board_metadata:
81 | nb = self.create_boards(bmeta=board)
82 |
83 | print(nb)
84 |
--------------------------------------------------------------------------------
/lmanage/configurator/create_object.py:
--------------------------------------------------------------------------------
1 | from abc import ABC, abstractmethod
2 |
3 |
4 | class CreateObject(ABC):
5 | @abstractmethod
6 | def execute():
7 | pass
8 |
--------------------------------------------------------------------------------
/lmanage/configurator/folder_configuration/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/configurator/folder_configuration/__init__.py
--------------------------------------------------------------------------------
/lmanage/configurator/folder_configuration/create_instance_folders.py:
--------------------------------------------------------------------------------
1 | from looker_sdk import error
2 | from looker_sdk.sdk.api40 import models
3 | from tqdm import tqdm
4 | from lmanage.utils import logger_creation as log_color
5 | import typing
6 | # logger = log_color.init_logger(__name__, logger_level)
7 |
8 |
9 | class CreateInstanceFolders():
10 | def __init__(self, folder_metadata, sdk, logger):
11 | self.sdk = sdk
12 | self.folder_metadata = folder_metadata
13 | self.logger = logger
14 |
15 | def create_folders(self):
16 | self.__delete_top_level_folders()
17 | for folder_tree in tqdm(self.folder_metadata, desc="Unesting Folder Data", unit="unested", colour="#2c8558"):
18 | metadata_list = []
19 | metadata_list = self.__walk_folder_structure(
20 | folder_tree=folder_tree, data_storage=metadata_list)
21 | self.logger.debug('Retrieved YAML folder files')
22 | self.logger.debug(f'Older metadata = {metadata_list}')
23 | return metadata_list
24 |
25 | def create_folder_mapping_dict(self, folder_metadata: list) -> dict:
26 | '''
27 | create a folder mapping dict with legacy folder id as it's key and new folder id as the value for use in content migration
28 | '''
29 | response = {}
30 | for folder_obj in folder_metadata:
31 | lid = folder_obj.get('old_folder_id')
32 | fid = folder_obj.get('new_folder_id')
33 | response[lid] = fid
34 |
35 | return response
36 |
37 | def __delete_top_level_folders(self):
38 | def is_top_level_folder(folder):
39 | return folder['parent_id'] is not None and int(folder['parent_id']) == 1
40 | all_folders = self.sdk.all_folders('id, name, parent_id')
41 | for folder in tqdm(all_folders):
42 | if is_top_level_folder(folder):
43 | try:
44 | self.logger.debug(
45 | f'Deleting folder "{folder["name"]}"')
46 | self.sdk.delete_folder(folder_id=folder['id'])
47 | except error.SDKError as e:
48 | self.logger.error(e)
49 |
50 | def __walk_folder_structure(self, folder_tree: dict, data_storage: list, parent_id: typing.Optional[str] = None):
51 | folder_name = folder_tree.get('name')
52 | try:
53 | if parent_id is None:
54 | folder_id = '1'
55 | else:
56 | folder_id = self.__create_folder(folder_name=folder_name,
57 | parent_id=parent_id).get('id')
58 | temp = {
59 | 'name': folder_name,
60 | 'old_folder_id': folder_tree.get('id'),
61 | 'new_folder_id': folder_id,
62 | 'content_metadata_id': folder_tree.get('content_metadata_id'),
63 | 'team_view': folder_tree.get('team_view'),
64 | 'team_edit': folder_tree.get('team_edit'),
65 | }
66 | self.logger.debug(f'Data structure to be appended = {temp}')
67 | data_storage.append(temp)
68 | if isinstance(folder_tree.get('subfolder'), list):
69 | for subfolder in folder_tree.get('subfolder'):
70 | self.__walk_folder_structure(subfolder, data_storage,
71 | parent_id=folder_id)
72 | except error.SDKError as e:
73 | self.logger.warn('error %s', e)
74 | self.logger.warn(
75 | 'you have a duplicate folder called %s with the same parent, this is against best practice and LManage is ignoring it', folder_name)
76 |
77 | # if isinstance(dict_obj.get('subfolder'), list):
78 | # for subfolder in dict_obj.get('subfolder'):
79 | # self.walk_folder_structure(subfolder, data_storage,
80 | # parent_name=new_folder_id)
81 |
82 | return data_storage
83 |
84 | def __create_folder(self, folder_name: str, parent_id: str):
85 | self.logger.debug(f'Creating folder "{folder_name}"')
86 | folder = self.sdk.create_folder(
87 | body=models.CreateFolder(
88 | name=folder_name,
89 | parent_id=parent_id
90 | )
91 | )
92 | return folder
93 |
94 | # def sync_folders(self, created_folder: list):
95 | # all_folders = self.sdk.all_folders()
96 | # created_folder_ids = [fobj.get('folder_id') for fobj in created_folder]
97 | # all_folder_ids = []
98 |
99 | # for folder in all_folders:
100 | # if folder.is_personal:
101 | # pass
102 | # elif folder.parent_id is None:
103 | # pass
104 | # else:
105 | # all_folder_ids.append(folder.id)
106 |
107 | # for fids in all_folder_ids:
108 | # if fids not in created_folder_ids:
109 | # try:
110 | # self.sdk.delete_folder(folder_id=fids)
111 | # self.logger.debug(
112 | # 'deleting folder %s to sync with yaml config', fids)
113 |
114 | # except error.SDKError as InheritanceError:
115 | # self.logger.debug('root folder has been deleted so %s',
116 | # InheritanceError)
117 | # return 'your folders are in sync with your yaml file'
118 |
--------------------------------------------------------------------------------
/lmanage/configurator/folder_configuration/folder_config.py:
--------------------------------------------------------------------------------
1 | from os import name
2 | from lmanage.utils import logger_creation as log_color
3 |
4 | # logger = log_color.init_logger(__name__, logger_level)
5 |
6 |
7 | class FolderConfig():
8 | ''' Class to read in folder metadata and unested_folder_data '''
9 |
10 | def __init__(self, folders):
11 | self.folder_metadata = folders
12 |
13 | def unnest_folder_data(self):
14 | response = []
15 |
16 | for d in self.folder_metadata:
17 | folder_dict = d
18 | metadata_list = []
19 | metadata_list = self.walk_folder_structure(
20 | dict_obj=folder_dict,
21 | data_storage=metadata_list,
22 | parent_id='1')
23 |
24 | response.append(metadata_list)
25 |
26 | logger.info('retrieved yaml folder files')
27 | logger.debug('folder metadata = %s', response)
28 |
29 | return response
30 |
31 | def walk_folder_structure(self, dict_obj: dict, data_storage: list, parent_id: str):
32 | temp = {}
33 | temp['name'] = dict_obj.get('name')
34 | temp['legacy_id'] = dict_obj.get('id')
35 | temp['team_edit'] = dict_obj.get('team_edit')
36 | temp['team_view'] = dict_obj.get('team_view')
37 | temp['parent_id'] = parent_id
38 | logger.debug('data_structure to be appended = %s', temp)
39 | data_storage.append(temp)
40 |
41 | if isinstance(dict_obj.get('subfolder'), list):
42 | for subfolder in dict_obj.get('subfolder'):
43 | self.walk_folder_structure(subfolder, data_storage,
44 | parent_id=dict_obj.get('name'))
45 |
46 | return data_storage
47 |
--------------------------------------------------------------------------------
/lmanage/configurator/looker_config_reader.py:
--------------------------------------------------------------------------------
1 | from lmanage.utils import parse_yaml as py, logger_creation as log_color
2 |
3 |
4 | class LookerConfigReader:
5 | def __init__(self, config_dir):
6 | logger_level = "INFO" # kwargs.get('verbose')
7 | logger = log_color.init_logger(__name__, logger_level)
8 | logger.info(
9 | "----------------------------------------------------------------------------------------"
10 | )
11 | logger.info("parsing yaml file")
12 | self.settings_yaml = py.Yaml(yaml_path=f"{config_dir}/settings.yaml")
13 | self.content_yaml = py.Yaml(yaml_path=f"{config_dir}/content.yaml")
14 | # self.logger = log_color.init_logger(__name__, logger_level)
15 |
16 | def read(self):
17 | self.config = {
18 | "folder_metadata": self.settings_yaml.get_metadata("folder_metadata"),
19 | "permission_set_metadata": self.settings_yaml.get_metadata(
20 | "permission_set_metadata"
21 | ),
22 | "model_set_metadata": self.settings_yaml.get_metadata("model_set_metadata"),
23 | "role_metadata": self.settings_yaml.get_metadata("role_metadata"),
24 | "user_attribute_metadata": self.settings_yaml.get_metadata(
25 | "user_attribute_metadata"
26 | ),
27 | "look_metadata": self.content_yaml.get_metadata("look_metadata"),
28 | "dashboard_metadata": self.content_yaml.get_metadata("dashboard_metadata"),
29 | # 'board_metadata': self.content_yaml.get_metadata('board_metadata'),
30 | }
31 |
32 | def get_summary(self):
33 | config = self.config
34 | look_scheduled_plan_count = sum(
35 | [
36 | len(l.get("scheduled_plans")) if l.get("scheduled_plans") else 0
37 | for l in config["look_metadata"]
38 | ]
39 | )
40 | scheduled_plan_and_alert_counts = [
41 | (
42 | len(d.get("scheduled_plans")) if d.get("scheduled_plans") else 0,
43 | len(d.get("alerts")) if d.get("alerts") else 0,
44 | )
45 | for d in config["dashboard_metadata"]
46 | ]
47 | scheduled_plan_count = 0
48 | alert_count = 0
49 |
50 | for first, second in scheduled_plan_and_alert_counts:
51 | scheduled_plan_count += first
52 | alert_count += second
53 |
54 | return f"""
55 | {len(config['permission_set_metadata']):<5} permissions
56 | {len(config['model_set_metadata']):<5} models
57 | {len(config['role_metadata']):<5} roles
58 | {len(config['user_attribute_metadata']):<5} user attributes
59 | {len(config['look_metadata']):<5} looks
60 | {look_scheduled_plan_count:<5} look scheduled plans
61 | {len(config['dashboard_metadata']):<5} dashboards
62 | {scheduled_plan_count:<5} dashboard scheduled plans
63 | {alert_count:<5} dashboard alerts
64 | """
65 |
--------------------------------------------------------------------------------
/lmanage/configurator/looker_provisioner.py:
--------------------------------------------------------------------------------
1 | from lmanage.looker_auth import LookerAuth
2 | from lmanage.configurator.user_attribute_configuration import create_and_assign_user_attributes as cuap
3 | from lmanage.configurator.folder_configuration import create_and_provision_instance_folders as cfp, create_instance_folders as cf
4 | from lmanage.configurator.user_group_configuration import create_instance_groups as gc, create_instance_roles as up, create_role_base as rc
5 | from lmanage.configurator.content_configuration import clean_instance_content as ccp, create_looks as cl, create_dashboards as cd, create_boards as cb
6 | from lmanage.utils import logger_creation as log_color
7 |
8 |
9 | class LookerProvisioner():
10 | def __init__(self, ini_file, verbose):
11 | self.sdk = LookerAuth().authenticate(ini_file)
12 | self.logger = log_color.init_logger(__name__, testing_mode=verbose)
13 |
14 | def provision(self, metadata):
15 | ###############
16 | # Role Config #
17 | ###############
18 | # Create Permission and Model Sets
19 | rc.CreateRoleBase(permissions=metadata['permission_set_metadata'],
20 | model_sets=metadata['model_set_metadata'], sdk=self.sdk, logger=self.logger).execute()
21 |
22 | #################
23 | # Folder Config #
24 | #################
25 | # CREATE NEW FOLDERS
26 | folder_objects = cf.CreateInstanceFolders(
27 | folder_metadata=metadata['folder_metadata'], sdk=self.sdk, logger=self.logger)
28 | created_folder_metadata = folder_objects.create_folders()
29 |
30 | # CREATE FOLDER TO FOLDER Dict
31 | folder_mapping_obj = folder_objects.create_folder_mapping_dict(
32 | folder_metadata=created_folder_metadata)
33 |
34 | ################
35 | # Group Config #
36 | ################
37 | # CREATE NEW GROUPS FROM YAML FILE TEAM VALUES
38 | gc.CreateInstanceGroups(
39 | folders=created_folder_metadata,
40 | user_attributes=metadata['user_attribute_metadata'],
41 | roles=metadata['role_metadata'],
42 | sdk=self.sdk,
43 | logger=self.logger).execute()
44 |
45 | ###########################
46 | # Folder Provision Config #
47 | ###########################
48 | # CREATE NEW GROUPS FROM YAML FILE TEAM VALUES
49 | cfp.CreateAndProvisionInstanceFolders(
50 | folders=created_folder_metadata,
51 | sdk=self.sdk, logger=self.logger).execute()
52 |
53 | ###############
54 | # Role Config #
55 | ###############
56 | up.CreateInstanceRoles(roles=metadata['role_metadata'],
57 | sdk=self.sdk,
58 | logger=self.logger).execute()
59 |
60 | #########################
61 | # User Attribute Config #
62 | #########################
63 | # FIND UNIQUE USER ATTRIBUTES AND ATTRIBUTE TO TEAM
64 | cuap.CreateAndAssignUserAttributes(
65 | user_attributes=metadata['user_attribute_metadata'],
66 | sdk=self.sdk,
67 | logger=self.logger).execute()
68 |
69 | ############################
70 | # Content Transport Config #
71 | ############################
72 | # EMPTY TRASH CAN OF ALL DELETED CONTENT
73 | ccp.CleanInstanceContent(sdk=self.sdk, logger=self.logger).execute()
74 |
75 | # FIND LOOKS AND REMAKE THEM
76 | cl.CreateLooks(
77 | sdk=self.sdk,
78 | folder_mapping=folder_mapping_obj,
79 | content_metadata=metadata['look_metadata'],
80 | logger=self.logger).execute()
81 |
82 | # Find DASHBOARDS AND REMAKE THEM
83 | cd.CreateDashboards(
84 | sdk=self.sdk,
85 | folder_mapping=folder_mapping_obj,
86 | content_metadata=metadata['dashboard_metadata'],
87 | logger=self.logger).execute()
88 |
--------------------------------------------------------------------------------
/lmanage/configurator/user_attribute_configuration/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/configurator/user_attribute_configuration/__init__.py
--------------------------------------------------------------------------------
/lmanage/configurator/user_attribute_configuration/create_and_assign_user_attributes.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from looker_sdk import error
3 | from looker_sdk.sdk.api40 import models
4 | from tqdm import tqdm
5 | from tenacity import retry, wait_random, wait_fixed, stop_after_attempt
6 | from lmanage.configurator.create_object import CreateObject
7 |
8 |
9 | class CreateAndAssignUserAttributes(CreateObject):
10 | def __init__(self, user_attributes, sdk, logger):
11 | self.user_attribute_metadata = user_attributes
12 | self.sdk = sdk
13 | self.logger = logger
14 |
15 | def existing_user_attributes(self) -> dict:
16 | all_instance_ua = self.sdk.all_user_attributes()
17 |
18 | all_ua = {ua.name: ua.id for ua in all_instance_ua}
19 | return all_ua
20 |
21 | def create_user_attribute_if_not_exists(self):
22 | existing_ua = self.existing_user_attributes()
23 | yaml_user_attributes = self.user_attribute_metadata
24 |
25 | for ua in tqdm(yaml_user_attributes, desc="User Attribute Creation", unit=" attributes", colour="#2c8558"):
26 | name = ua.get('name')
27 | if name in existing_ua.keys():
28 | logging.warning(
29 | f'user attribute {name} already exists on this instance')
30 | else:
31 | datatype = ua.get('uatype')
32 | value_is_hidden = ua.get('hidden_value')
33 | user_view = ua.get('user_view')
34 | user_edit = ua.get('user_edit')
35 |
36 | try:
37 | ua_permissions = models.WriteUserAttribute(
38 | name=name,
39 | label=name,
40 | type=datatype,
41 | value_is_hidden=value_is_hidden,
42 | user_can_view=user_view,
43 | user_can_edit=user_edit,
44 | default_value=ua.get('default_value')
45 | )
46 | response = self.sdk.create_user_attribute(
47 | body=ua_permissions)
48 | logging.info(f'created user attribute {response.label}')
49 | except error.SDKError as err:
50 | logging.error('skipping permission because already exists')
51 |
52 | def sync_user_attributes(self):
53 | instance_ua = self.existing_user_attributes()
54 | config_ua = [ua.get('name') for ua in self.user_attribute_metadata]
55 | sys_default_ua = [
56 | 'email',
57 | 'first_name',
58 | 'id',
59 | 'google_user_id',
60 | 'landing_page',
61 | 'last_name',
62 | 'number_format',
63 | 'locale',
64 | 'name',
65 | "looker_internal_email_domain_allowlist",
66 | ]
67 |
68 | for ua in sys_default_ua:
69 | if ua in instance_ua:
70 | instance_ua.pop(ua, None)
71 |
72 | for ua_name in instance_ua.keys():
73 | if ua_name not in config_ua:
74 | ua_id = instance_ua.get(ua_name)
75 | self.sdk.delete_user_attribute(ua_id)
76 | logging.info(
77 | f'''deleting ua {ua_name} because
78 | it is not listed in the yaml config''')
79 |
80 | def add_group_values_to_ua(self):
81 | instance_ua = self.existing_user_attributes()
82 | all_instance_groups = self.sdk.all_groups()
83 | group_metadata = {
84 | group.name: group.id for group in all_instance_groups}
85 | yaml_user_attributes = self.user_attribute_metadata
86 | for ua in yaml_user_attributes:
87 | if len(ua.get('teams')) > 0:
88 | for team_val in ua.get('teams'):
89 | group_id = group_metadata.get(list(team_val.keys())[0])
90 |
91 | meta_value = list(team_val.values())[0]
92 | ua_id = instance_ua.get(ua.get('name'))
93 |
94 | params_to_add = models.UserAttributeGroupValue(
95 | value=meta_value,
96 | value_is_hidden=False
97 | )
98 |
99 | self.sdk.update_user_attribute_group_value(
100 | group_id=group_id,
101 | user_attribute_id=ua_id,
102 | body=params_to_add)
103 |
104 | def execute(self):
105 | # CREATE NEW USER ATTRIBUTES
106 | self.create_user_attribute_if_not_exists()
107 |
108 | # DELETE ALL USER ATTRIBUTES THAT DON'T MATCH WITH YAML
109 | self.sync_user_attributes()
110 |
111 | # ADD VALUES TO INSCOPE USER ATTRIBUTES
112 | self.add_group_values_to_ua()
113 |
--------------------------------------------------------------------------------
/lmanage/configurator/user_group_configuration/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/configurator/user_group_configuration/__init__.py
--------------------------------------------------------------------------------
/lmanage/configurator/user_group_configuration/create_instance_groups.py:
--------------------------------------------------------------------------------
1 | from looker_sdk import error
2 | from looker_sdk.sdk.api40 import models
3 | from lmanage.utils.errorhandling import return_error_message
4 | from tqdm import tqdm
5 | from tenacity import retry, wait_fixed, wait_random, stop_after_attempt
6 | from lmanage.configurator.create_object import CreateObject
7 |
8 |
9 | class CreateInstanceGroups(CreateObject):
10 | def __init__(self, folders, user_attributes, roles, sdk, logger) -> None:
11 | self.folder_metadata = folders
12 | self.user_attribute_metadata = user_attributes
13 | self.role_metadata = roles
14 | self.sdk = sdk
15 | self.logger = logger
16 |
17 | def extract_teams(self, container, data_storage, ua: bool):
18 | for team in container:
19 | team_list = team.get('teams')
20 | if len(team_list) > 0:
21 | if ua:
22 | for team_val in team_list:
23 | team_app = list(team_val.keys())[0]
24 | data_storage.append(team_app)
25 | else:
26 | for team_val in team_list:
27 | data_storage.append(team_val)
28 | return data_storage
29 |
30 | def extract_folder_teams(self, container, data_storage):
31 | for folder_element in container:
32 | if isinstance(folder_element.get('team_edit'), list):
33 | edit_group = folder_element.get('team_edit')
34 | for group in edit_group:
35 | data_storage.append(group)
36 | if isinstance(folder_element.get('team_view'), list):
37 | view_group = folder_element.get('team_view')
38 | for group in view_group:
39 | data_storage.append(group)
40 | else:
41 | pass
42 |
43 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
44 | def search_looker_groups(self, sdk, group_name: str) -> list:
45 | s = sdk.search_groups(name=group_name)
46 | return s
47 |
48 | def create_group_if_not_exists(self,
49 | sdk,
50 | group_name: str) -> dict:
51 | """ Create a Looker Group and add Group attributes
52 |
53 | :group_name: Name of a Looker group to create.
54 | :rtype: Looker Group object.
55 | """
56 | # get group if exists
57 | try:
58 | self.logger.debug(f'Creating group "{group_name}"')
59 | group = sdk.create_group(
60 | body=models.WriteGroup(
61 | can_add_to_content_metadata=True,
62 | name=group_name
63 | )
64 | )
65 | return group
66 | except error.SDKError as grouperr:
67 | err_msg = return_error_message(grouperr)
68 | self.logger.debug(
69 | 'You have hit a warning creating your group; warning = %s', err_msg)
70 | self.logger.debug(grouperr)
71 | group = self.search_looker_groups(
72 | sdk=self.sdk, group_name=group_name)
73 | return group[0]
74 |
75 | def get_instance_group_metadata(self,
76 | sdk,
77 | unique_group_list: list) -> list:
78 | group_metadata = []
79 |
80 | for group_name in tqdm(unique_group_list, desc="Creating User Groups", unit=" user groups", colour="#2c8558"):
81 | group = self.create_group_if_not_exists(sdk, group_name)
82 | temp = {}
83 | temp['group_id'] = group.id
84 | temp['group_name'] = group.name
85 | group_metadata.append(temp)
86 |
87 | return group_metadata
88 |
89 | def sync_groups(self,
90 | group_name_list: list) -> str:
91 |
92 | all_groups = self.sdk.all_groups()
93 | group_dict = {group.name: group.id for group in all_groups}
94 | # Deleting Standard Groups
95 | try:
96 | del group_dict['All Users']
97 | except:
98 | pass
99 |
100 | for group_name in group_dict.keys():
101 | if group_name not in group_name_list:
102 | self.sdk.delete_group(group_id=group_dict[group_name])
103 | self.logger.info(
104 | f'deleting group {group_name} to sync with yaml config')
105 |
106 | return 'your groups are in sync with your yaml file'
107 |
108 | def execute(self):
109 | team_list = []
110 | # extracting user attribute teams
111 | self.extract_teams(container=self.user_attribute_metadata,
112 | data_storage=team_list, ua=True)
113 | # extracting role based teams
114 | self.extract_teams(
115 | container=self.role_metadata, data_storage=team_list, ua=False)
116 | # extract nested folder access teams
117 | self.extract_folder_teams(
118 | container=self.folder_metadata, data_storage=team_list)
119 |
120 | team_list = list(set(team_list))
121 |
122 | # create all the groups
123 | self.get_instance_group_metadata(
124 | sdk=self.sdk, unique_group_list=team_list)
125 | self.sync_groups(group_name_list=team_list)
126 |
--------------------------------------------------------------------------------
/lmanage/configurator/user_group_configuration/create_instance_roles.py:
--------------------------------------------------------------------------------
1 | from looker_sdk import error
2 | from looker_sdk.sdk.api40 import models
3 | from lmanage.configurator.user_group_configuration.create_role_base import CreateRoleBase
4 | from lmanage.utils.errorhandling import return_error_message
5 | from tqdm import tqdm
6 | from tenacity import retry, wait_random, wait_fixed, stop_after_attempt
7 |
8 |
9 | class CreateInstanceRoles(CreateRoleBase):
10 | def __init__(self, roles, sdk, logger):
11 | self.role_metadata = roles
12 | self.sdk = sdk
13 | self.logger = logger
14 |
15 | def create_permission_lookup(self):
16 | all_permission_sets = self.get_all_permission_sets()
17 | instance_permission_set_dict = {
18 | perm.name: perm.id for perm in all_permission_sets}
19 | return instance_permission_set_dict
20 |
21 | def create_model_lookup(self):
22 | all_model_sets = self.get_all_model_sets()
23 | instance_model_set_dict = {
24 | model.name: model.id for model in all_model_sets}
25 | return instance_model_set_dict
26 |
27 | def create_allgroup_lookup(self):
28 | all_groups = self.sdk.all_groups()
29 | group_dict = {group.name: group.id for group in all_groups}
30 | return group_dict
31 |
32 | def create_allrole_lookup(self):
33 | all_roles = self.sdk.all_roles()
34 | role_dict = {role.name: role.id for role in all_roles}
35 | return role_dict
36 |
37 | @retry(wait=wait_fixed(3) + wait_random(0, 2), stop=stop_after_attempt(5))
38 | def update_looker_role(self, role_id: str, body: models.WriteRole):
39 | role = self.sdk.update_role(role_id=role_id, body=body)
40 | return role
41 |
42 | def create_instance_roles(self):
43 | model_lookup = self.create_model_lookup()
44 | permission_lookup = self.create_permission_lookup()
45 |
46 | role_output = []
47 |
48 | for r_metadata in tqdm(self.role_metadata, desc="User Role Creation", unit=" user roles", colour="#2c8558"):
49 | role_name = r_metadata.get('name')
50 | model_set_id = model_lookup.get(
51 | r_metadata.get('model_set') if r_metadata.get('model_set') == 'All' else r_metadata.get('model_set').lower())
52 | permission_set_id = permission_lookup.get(
53 | r_metadata.get('permission_set').lower())
54 | if role_name != 'Admin':
55 | body = models.WriteRole(
56 | name=role_name,
57 | permission_set_id=str(permission_set_id),
58 | model_set_id=str(model_set_id)
59 | )
60 | try:
61 | role = self.sdk.create_role(
62 | body=body
63 | )
64 |
65 | except error.SDKError as roleerror:
66 | err_msg = return_error_message(roleerror)
67 | self.logger.debug(
68 | 'You have hit a warning creating your role \'%s\'; warning = %s', role_name, err_msg)
69 | role_id = self.sdk.search_roles(name=role_name)[0].id
70 | role = self.update_looker_role(
71 | role_id=str(role_id), body=body)
72 | temp = {}
73 | temp['role_id'] = role.id
74 | temp['role_name'] = role_name
75 | role_output.append(temp)
76 | else:
77 | pass
78 | self.logger.debug(role_output)
79 | return role_output
80 |
81 | def set_role(self, role_id: str, group_id: list) -> str:
82 | try:
83 | self.sdk.set_role_groups(role_id, group_id)
84 | return self.logger.debug(f'attributing {group_id} permissions on instance')
85 | except error.SDKError as role_err:
86 | err_msg = return_error_message(role_err)
87 | self.logger.debug(
88 | 'You have hit a warning setting your role, warning = %s', err_msg)
89 | self.logger.debug(role_err)
90 |
91 | def attribute_instance_roles(self, created_role_metadata: list):
92 | role_lookup = self.create_allrole_lookup()
93 | group_lookup = self.create_allgroup_lookup()
94 |
95 | for r_metadata in tqdm(self.role_metadata, desc="Instance Role Attribution", unit=" roles", colour="#2c8558"):
96 | role_name = r_metadata.get('name')
97 | teams = r_metadata.get('teams')
98 | role_id = role_lookup.get(role_name)
99 | group_id_list = []
100 | for team in teams:
101 | group_id = group_lookup.get(team)
102 | group_id_list.append(group_id)
103 | self.set_role(role_id=role_id, group_id=group_id_list)
104 |
105 | def sync_roles(self):
106 | all_role_lookup = self.create_allrole_lookup()
107 | all_role_lookup.pop('Admin')
108 | yaml_role = [role.get('name') for role in self.role_metadata]
109 |
110 | for role_name in all_role_lookup.keys():
111 | if role_name not in yaml_role:
112 | role_id = all_role_lookup.get(role_name)
113 | self.sdk.delete_role(role_id=role_id)
114 |
115 | def execute(self):
116 | created_roles = self.create_instance_roles()
117 | self.attribute_instance_roles(created_role_metadata=created_roles)
118 | self.sync_roles()
119 |
--------------------------------------------------------------------------------
/lmanage/find_replace.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # find_and_replace.sh
3 | # set -x #echo on
4 | echo 'p for poetry run added in or np for vimspector'
5 | read INPUT
6 |
7 | echo "your input is $INPUT"
8 |
9 | POETRY="p"
10 | NOTPOETRY="np"
11 |
12 | if [ "$POETRY" == "$INPUT" ]
13 | then
14 | find . -type f -name "*.py" -print0 | xargs -0 sed -i -e 's/from configurator/from lmanage.configurator/g' -e 's/from utils/from lmanage.utils/g' -e 's/from capturator/from lmanage.capturator/g'
15 |
16 | else
17 | echo 'unpoetizing'
18 | fi
19 |
20 | if [ $NOTPOETRY == $INPUT ]
21 | then
22 | find . -type f -name "*.py" -print0 | xargs -0 sed -i -e 's/from lmanage.configurator/from configurator/g' -e 's/from lmanage.utils/from utils/g' -e 's/from lmanage.capturator/from capturator/g'
23 |
24 | else
25 | echo 'poetizing'
26 | fi
27 |
--------------------------------------------------------------------------------
/lmanage/logger_config.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 |
4 | def setup_logger():
5 | # Create a custom logger
6 | logger = logging.getLogger(__name__)
7 |
8 | # Set the level of logging.
9 | # You might want to use logging.DEBUG or logging.ERROR in other scenarios
10 | logger.setLevel(logging.INFO)
11 |
12 | # Create handlers. Here, we create a StreamHandler (for stdout) and a FileHandler (for writing logs to a file)
13 | c_handler = logging.StreamHandler()
14 | f_handler = logging.FileHandler('file.log')
15 | c_handler.setLevel(logging.INFO)
16 | f_handler.setLevel(logging.ERROR)
17 |
18 | # Create formatters and add it to handlers
19 | c_format = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
20 | f_format = logging.Formatter(
21 | '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
22 | c_handler.setFormatter(c_format)
23 | f_handler.setFormatter(f_format)
24 |
25 | # Add handlers to the logger
26 | logger.addHandler(c_handler)
27 | logger.addHandler(f_handler)
28 |
29 | return logger
30 |
--------------------------------------------------------------------------------
/lmanage/looker_auth.py:
--------------------------------------------------------------------------------
1 | import looker_sdk
2 | from lmanage.logger_config import setup_logger
3 | from looker_sdk import error
4 |
5 | logger = setup_logger()
6 |
7 |
8 | class LookerAuth():
9 | def authenticate(self, ini_file):
10 | sdk = looker_sdk.init40(
11 | config_file=ini_file) if ini_file else looker_sdk.init40()
12 | if self.__auth_check(sdk):
13 | logger.info('User is successfully authenticated to the API')
14 | return sdk
15 | else:
16 | raise Exception(
17 | 'User is not successfully authenticated. Please verify credentials in .ini file.')
18 |
19 | def __auth_check(self, sdk):
20 | try:
21 | sdk.me()
22 | return True
23 | except:
24 | return False
25 |
--------------------------------------------------------------------------------
/lmanage/looker_config.py:
--------------------------------------------------------------------------------
1 | class LookerConfig():
2 | def __init__(self, settings, content):
3 | self.settings = settings
4 | self.content = content
5 |
--------------------------------------------------------------------------------
/lmanage/mapview/.vimspector.json:
--------------------------------------------------------------------------------
1 | {
2 | "configurations": {
3 | "run": {
4 | "adapter": "debugpy",
5 | "configuration": {
6 | "python": "/usr/local/google/home/hugoselbie/.cache/pypoetry/virtualenvs/lmanage-UKKoJ9La-py3.8/bin/python",
7 | "request": "launch",
8 | "protocol": "auto",
9 | "stopOnEntry": true,
10 | "console": "integratedTerminal",
11 | "program": "${workspaceRoot}/mapview_execute.py",
12 | "cwd": "${workspaceRoot}"
13 | }
14 | }
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/lmanage/mapview/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
--------------------------------------------------------------------------------
/lmanage/mapview/compare_content_to_lookml.py:
--------------------------------------------------------------------------------
1 | import ast
2 | import coloredlogs
3 | import logging
4 | from lmanage.mapview.utils import parse_lookml
5 |
6 | logger = logging.getLogger(__name__)
7 | coloredlogs.install(level='INFO')
8 | logging.getLogger("looker_sdk").setLevel(logging.WARNING)
9 |
10 |
11 | def format_str_list(list_str):
12 | new_list = ast.literal_eval(list_str)
13 | return new_list
14 |
15 |
16 | def collate_view_files(parsed_lookml):
17 | response = []
18 | for instance_lookml in parsed_lookml:
19 | for path in instance_lookml.keys():
20 | if 'view.lkml' in path:
21 | response.append(instance_lookml)
22 | return response
23 |
24 |
25 | def extract_field_names_from_lookml(parsed_lookml, data_storage):
26 |
27 | try:
28 | dimensions = parsed_lookml['dimensions']
29 | except KeyError:
30 | dimensions = []
31 | logger.warning('no dimensions present in view')
32 | try:
33 | measures = parsed_lookml['measures']
34 | except KeyError:
35 | measures = []
36 | logger.warning('no measures present in view')
37 |
38 | view_name = parsed_lookml.get('name')
39 | for dim in dimensions:
40 | field_name = dim.get('name')
41 | full_name = f'{view_name}.{field_name}'
42 | data_storage[full_name] = view_name
43 |
44 | for meas in measures:
45 | if len(meas) > 0:
46 | field_name = meas.get('name')
47 | full_name = f'{view_name}.{field_name}'
48 | data_storage[full_name] = view_name
49 |
50 | return data_storage
51 |
52 |
53 | def match_fields_to_lookml(field_list, lookml):
54 | matched_fields = []
55 | for field in field_list:
56 | match = lookml.get(field, 'no match')
57 | if match != 'no match':
58 | matched_fields.append(field)
59 |
60 | return matched_fields
61 |
62 |
63 | def extract_view_from_field(field_name):
64 | view_name = field_name.split('.')[0]
65 | return view_name
66 |
67 |
68 | def create_lookml_field_lookup(view_file):
69 | field_lookup = {}
70 | for lookml in view_file:
71 | parsed_lookml = parse_lookml.LookML(lookml)
72 | if parsed_lookml.has_views():
73 | for v in parsed_lookml.views():
74 | extract_field_names_from_lookml(
75 | parsed_lookml=v, data_storage=field_lookup)
76 | return field_lookup
77 |
78 |
79 | def create_viewname_lookup(view_file):
80 | sql_table_name_list = []
81 | for lookml in view_file:
82 | parsed_lookml = parse_lookml.LookML(lookml)
83 | if parsed_lookml.has_views():
84 | for v in parsed_lookml.views():
85 | try:
86 | sql_table_name = v['sql_table_name']
87 | sql_table_name_list.append(sql_table_name)
88 | except KeyError:
89 | logger.warning('no sql_table_name present in view')
90 | return sql_table_name_list
91 |
92 |
93 | def match_element_to_lookml(instancedata, view_file):
94 | field_lookup = create_lookml_field_lookup(view_file)
95 | viewname_lookup = create_viewname_lookup(view_file)
96 |
97 | matches = []
98 | for dashboard in instancedata:
99 | fields_used = format_str_list(dashboard.get('query.formatted_fields'))
100 | views_in_dash = []
101 | for field in fields_used:
102 | x = extract_view_from_field(field)
103 | if x not in views_in_dash:
104 | views_in_dash.append(x)
105 | matched_fields = match_fields_to_lookml(field_list=fields_used,
106 | lookml=field_lookup)
107 |
108 | for views in viewname_lookup:
109 | if views not in views_in_dash:
110 | views_in_dash.append(views)
111 |
112 | temp = {}
113 | temp['used_views'] = views_in_dash
114 | temp['matched_fields'] = matched_fields
115 | temp['dashboard'] = dashboard['dashboard.id']
116 | temp['element_id'] = dashboard['dashboard_element.id']
117 | matches.append(temp)
118 | print(matches)
119 | return matches
120 |
--------------------------------------------------------------------------------
/lmanage/mapview/hugo.csv:
--------------------------------------------------------------------------------
1 | ,used_views,matched_fields
2 | 0,"['content_usage', 'public.distribution_centers', 'public.products', 'public.users', 'public.order_items', 'public.inventory_items', 'public.events', '`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']",[]
3 | 1,"['history', 'source_query_rank', 'public.distribution_centers', 'public.products', 'public.users', 'public.order_items', 'public.inventory_items', 'public.events', '`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']",[]
4 | 2,"['order_items', 'public.distribution_centers', 'public.products', 'public.users', 'public.order_items', 'public.inventory_items', 'public.events', '`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items.id', 'order_items.order_id', 'order_items.sale_price', 'order_items.average_days_to_process']"
5 | 3,"['products', 'order_items', 'public.distribution_centers', 'public.products', 'public.users', 'public.order_items', 'public.inventory_items', 'public.events', '`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['products.brand', 'order_items.gross_margin', 'order_items.id']"
6 |
--------------------------------------------------------------------------------
/lmanage/mapview/instancedata.py:
--------------------------------------------------------------------------------
1 | from looker_sdk.sdk.api40 import models
2 | import json
3 |
4 |
5 | class GetCleanInstanceData():
6 | def __init__(self, sdk):
7 | self.sdk = sdk
8 |
9 | def get_dashboards(self):
10 | """Uses the Looker SDK System__Activity model to extract dashboard
11 | and dashboard_element metadata.
12 | """
13 | query_config = models.WriteQuery(
14 | model="system__activity",
15 | view="dashboard",
16 | fields=[
17 | "dashboard.id",
18 | "dashboard_element.id",
19 | "dashboard_element.type",
20 | "dashboard_element.result_source",
21 | "query.formatted_pivots",
22 | "query.model",
23 | "query.view",
24 | "query.formatted_fields",
25 | "query.id",
26 | "dashboard.title",
27 | "look.id"
28 | ],
29 | filters={
30 | "dashboard_element.type": "-text",
31 | "dashboard.deleted_date": "NULL"
32 | },
33 | limit='5000'
34 | )
35 | query_response = self.sdk.run_inline_query(
36 | result_format='json',
37 | body=query_config
38 | )
39 |
40 | query_response = json.loads(query_response)
41 |
42 | return query_response
43 |
44 | def unpivot_query(self, query_id):
45 | query_metadata = self.sdk.query(query_id=query_id)
46 |
47 | query_body = models.WriteQuery(
48 | model=query_metadata.model,
49 | view=query_metadata.view,
50 | fields=query_metadata.fields,
51 | filters=query_metadata.filters
52 | )
53 |
54 | new_query = self.sdk.create_query(body=query_body)
55 | return new_query.id
56 |
57 | def execute(self):
58 | instance_data = self.get_dashboards()
59 | clean_data = []
60 | for db_element in instance_data:
61 | if db_element['query.formatted_pivots'] is None:
62 | clean_data.append(db_element)
63 | else:
64 | query_id = db_element['query.id']
65 | nqid = self.unpivot_query(query_id)
66 | db_element['query.id'] = nqid
67 | clean_data.append(db_element)
68 | return clean_data
69 |
--------------------------------------------------------------------------------
/lmanage/mapview/main.py:
--------------------------------------------------------------------------------
1 | '''
2 | get dashboards
3 | find elements on dashboards
4 | get query elements of queries
5 | get model files
6 | get explores
7 | get fields
8 |
9 | get folder of lookml, or git repo
10 | match dash field to lookml files
11 | match how many times a lookml file has been referenced
12 | show table definition of matched table
13 | '''
14 | from looker_sdk import init31
15 | import logging
16 | import coloredlogs
17 | import instancedata as idata
18 | import parse_lookml as pl
19 | import mapexplores as me
20 | import matched_field_analysis as mfa
21 |
22 | logger = logging.getLogger(__name__)
23 | coloredlogs.install(level='INFO')
24 | logging.getLogger("looker_sdk").setLevel(logging.WARNING)
25 |
26 |
27 | def main(**kwargs):
28 | div = '-----------------------------------------------------------------------------'
29 |
30 | ini_file = kwargs.get("ini_file")
31 | lookml_file_path = kwargs.get("file_path")
32 | logger.info(div)
33 | sdk = init31(config_file=ini_file)
34 |
35 | dash_to_dash_elements = idata.get_dashboards(sdk=sdk)
36 | lookml_files = pl.get_all_lkml_filepaths(starting_path=lookml_file_path)
37 |
38 | parsed_lookml_dict = pl.get_parsed_lookml(lookml_file_paths=lookml_files)
39 |
40 | elements = me.match_element_to_lookml(
41 | instancedata=dash_to_dash_elements, view_file=lookml_files)
42 | da = mfa.MatchedFieldDataAnalysis(matched_fields=elements)
43 | print(da.most_used_object(looker_object='fields'))
44 | print(da.field_to_dashboard_map(dashboard_id_filter=7))
45 | da.data_analysis()
46 |
47 |
48 | if __name__ == "__main__":
49 | IP = ('/usr/local/google/home/hugoselbie/code_sample/py/ini/k8.ini')
50 | FP = ('/usr/local/google/home/hugoselbie/code_sample/py/mapview/test_lookml_files/the_look')
51 |
52 | main(
53 | ini_file=IP,
54 | file_path=FP)
55 |
--------------------------------------------------------------------------------
/lmanage/mapview/mapexplores.py:
--------------------------------------------------------------------------------
1 | import lkml
2 | import glob
3 | import logging
4 | import coloredlogs
5 | import ast
6 |
7 | import parse_lookml
8 |
9 | logger = logging.getLogger(__name__)
10 | coloredlogs.install(level='INFO')
11 | logging.getLogger("looker_sdk").setLevel(logging.WARNING)
12 |
13 |
14 | def format_str_list(list_str):
15 | new_list = ast.literal_eval(list_str)
16 | return new_list
17 |
18 |
19 | def collate_view_files(parsed_lookml):
20 | response = []
21 | for instance_lookml in parsed_lookml:
22 | for path in instance_lookml.keys():
23 | if 'view.lkml' in path:
24 | response.append(instance_lookml)
25 | return response
26 |
27 |
28 | def extract_field_names_from_lookml(parsed_lookml, data_storage):
29 |
30 | try:
31 | dimensions = parsed_lookml['dimensions']
32 | except KeyError:
33 | dimensions = []
34 | logger.warning('no dimensions present in view')
35 | try:
36 | measures = parsed_lookml['measures']
37 | except KeyError:
38 | measures = []
39 | logger.warning('no measures present in view')
40 |
41 | view_name = parsed_lookml.get('name')
42 | for dim in dimensions:
43 | field_name = dim.get('name')
44 | full_name = f'{view_name}.{field_name}'
45 | data_storage[full_name] = view_name
46 |
47 | for meas in measures:
48 | if len(meas) > 0:
49 | field_name = meas.get('name')
50 | full_name = f'{view_name}.{field_name}'
51 | data_storage[full_name] = view_name
52 |
53 | return data_storage
54 |
55 |
56 | def match_fields_to_lookml(field_list, lookml):
57 | matched_fields = []
58 | for field in field_list:
59 | match = lookml.get(field, 'no match')
60 | if match != 'no match':
61 | matched_fields.append(field)
62 |
63 | print(matched_fields)
64 | return matched_fields
65 |
66 |
67 | def extract_view_from_field(field_name):
68 | view_name = field_name.split('.')[0]
69 | return view_name
70 |
71 |
72 | def match_element_to_lookml(instancedata, view_file):
73 | field_lookup = {}
74 | for lookml in view_file:
75 | parsed_lookml = parse_lookml.LookML(lookml)
76 | if parsed_lookml.has_views():
77 | for v in parsed_lookml.views():
78 | extract_field_names_from_lookml(
79 | parsed_lookml=v, data_storage=field_lookup)
80 |
81 | matches = []
82 | for dashboard in instancedata:
83 | fields_used = format_str_list(dashboard.get('query.formatted_fields'))
84 | views_in_dash = []
85 | for field in fields_used:
86 | x = extract_view_from_field(field)
87 | if x not in views_in_dash:
88 | views_in_dash.append(x)
89 | matched_fields = match_fields_to_lookml(field_list=fields_used,
90 | lookml=field_lookup)
91 |
92 | temp = {}
93 | temp['used_views'] = views_in_dash
94 | temp['matched_fields'] = matched_fields
95 | temp['dashboard'] = dashboard.get('dashboard.id')
96 | matches.append(temp)
97 | return matches
98 |
--------------------------------------------------------------------------------
/lmanage/mapview/mapview_execute.py:
--------------------------------------------------------------------------------
1 | from looker_sdk import init31
2 | import logging
3 | import coloredlogs
4 | import mapview.instancedata as idata
5 | import mapview.parse_sql_tables as pst
6 | import mapview.compare_content_to_lookml as me
7 | import mapview.matched_field_analysis as mfa
8 | from lmanage.mapview.utils import parse_lookml as pl
9 |
10 | logger = logging.getLogger(__name__)
11 | coloredlogs.install(level='INFO')
12 | logging.getLogger("looker_sdk").setLevel(logging.WARNING)
13 |
14 |
15 | def main(**kwargs):
16 | div = '-------------------------------------------------------------------'
17 |
18 | ini_file = kwargs.get("ini_file")
19 | lookml_file_path = kwargs.get("lookml_file_path")
20 | file_output = kwargs.get("output_path")
21 | logger.info(div)
22 | sdk = init31(config_file=ini_file)
23 |
24 | dashboard_metadata = idata.GetCleanInstanceData(sdk=sdk).execute()
25 |
26 | query_elements = pst.ParseSqlTables(dataextract=dashboard_metadata,
27 | sdk=sdk).get_sql_from_elements()
28 |
29 | lookml_files = pl.get_all_lkml_filepaths(starting_path=lookml_file_path)
30 |
31 | elements = me.match_element_to_lookml(
32 | instancedata=query_elements, view_file=lookml_files)
33 | da = mfa.MatchedFieldDataAnalysis(matched_fields=elements)
34 |
35 | da.export_to_excel(file_output)
36 | logger.info('i have finished')
37 |
38 |
39 | if __name__ == "__main__":
40 | IP = ('/usr/local/google/home/hugoselbie/code_sample/py/ini/k8.ini')
41 | FP = ('/usr/local/google/home/hugoselbie/code_sample/py/mapview/test_lookml_files/the_look')
42 |
43 | main(
44 | ini_file=IP,
45 | file_path=FP)
46 |
--------------------------------------------------------------------------------
/lmanage/mapview/matched_field_analysis.py:
--------------------------------------------------------------------------------
1 | from lmanage.attr import field
2 | import lkml
3 | import glob
4 | import logging
5 | import coloredlogs
6 | import ast
7 | import pandas as pd
8 | import xlsxwriter
9 |
10 | logger = logging.getLogger(__name__)
11 | coloredlogs.install(level='INFO')
12 | logging.getLogger("looker_sdk").setLevel(logging.WARNING)
13 |
14 |
15 | class MatchedFieldDataAnalysis():
16 | def __init__(self, matched_fields):
17 | self.matched_data = matched_fields
18 | self.df = pd.DataFrame(self.matched_data)
19 |
20 | def create_df(self):
21 | df = pd.DataFrame(self.matched_data)
22 | return df
23 |
24 | def most_used_object(self, looker_object):
25 | if looker_object == 'views':
26 | xploded = self.df.explode('used_views')
27 | xploded = xploded.groupby(['used_views'])['dashboard'].count()
28 | return xploded
29 | elif looker_object == 'fields':
30 | xploded = self.df.explode('matched_fields')
31 | xploded = xploded.groupby(['matched_fields'])['dashboard'].count()
32 | xploded = xploded.groupby(['used_views'])[
33 | 'dashboard'].count().rename('dashboard_count').sort_values(ascending=False)
34 | return xploded
35 |
36 | def field_to_dashboard_map(self, field_filter=None, dashboard_id_filter=None):
37 | xploded = self.df.explode('matched_fields')
38 | if field_filter:
39 | xploded = xploded.filter(like=field_filter, axis=0)
40 | if dashboard_id_filter:
41 | xploded = xploded.filter(like=dashboard_id_filter)
42 | return xploded
43 |
44 | def data_analysis(self):
45 | df = self.create_df()
46 | print(df)
47 |
48 | def export_to_excel(self, output):
49 | writer = pd.ExcelWriter(output, engine='xlsxwriter')
50 | dashboard_match = self.field_to_dashboard_map()
51 | used_views = self.most_used_object(looker_object='views')
52 | used_obj = self.most_used_object(looker_object='fields')
53 |
54 | # Write each dataframe to a different worksheet.
55 | dashboard_match.to_excel(writer, sheet_name='DashboardToLookMLMapping')
56 | used_obj.to_excel(writer, sheet_name='MostUsedDimensions')
57 | used_views.to_excel(writer, sheet_name='MostUsedViews')
58 |
59 | # Close the Pandas Excel writer and output the Excel file.
60 | writer.save()
61 |
--------------------------------------------------------------------------------
/lmanage/mapview/pandas_multiple.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/lmanage/mapview/pandas_multiple.xlsx
--------------------------------------------------------------------------------
/lmanage/mapview/parse_sql_tables.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import coloredlogs
3 | from lmanage.mapview.utils import parsing_sql
4 | from looker_sdk import error
5 |
6 |
7 | logger = logging.getLogger(__name__)
8 | coloredlogs.install(level='DEBUG')
9 | logging.getLogger("looker_sdk").setLevel(logging.WARNING)
10 |
11 |
12 | class ParseSqlTables():
13 | def __init__(self, dataextract, sdk):
14 | self.dataextract = dataextract
15 | self.sdk = sdk
16 |
17 | def parse_sql(self, qid: int):
18 | """
19 | Idenfies the base tables and joins used by a Looker query.
20 |
21 | Iterates over a list of PyLookML files identifies view files aand returns a
22 | list of the sql_table_names if they exist in an object of type 'View'
23 | qid: (int) query_id from lmanage.a Looker query
24 | Returns:
25 | A list of all the tables that are found from lmanage.a Looker generated
26 | SQL query.
27 | For example:
28 | ['public.order_items','public.inventory_items','public.events']
29 | Exception:
30 | If a query is broken for whatever reason the Exception is caught to
31 | continue the program running
32 | """
33 | try:
34 | sql_response = self.sdk.run_query(
35 | query_id=qid, result_format="sql")
36 | if isinstance(sql_response, str):
37 | tables = parsing_sql.extract_tables(sql_response)
38 | return tables
39 | else:
40 | return sql_response
41 | except error.SDKError:
42 | return('No Content')
43 |
44 | def get_sql_from_elements(self):
45 | """Amends returned SDK System__Activity reponse with sql tables used
46 | from lmanage.the `parse_sql` function.
47 |
48 | Iterates over the response from lmanage.get_dashboards and runs the parse_sql
49 | function for each returned dashboard element, returns the list of tables
50 | and amends the dict response and returns it
51 | Args:
52 | sdk: Looker SDK object
53 | content_results: (dict) response from lmanage.get_dashboards function call
54 | Returns:
55 | An amended dict response with the sql columns used by each element
56 | extracted our of the Looker generated SQL for each dashboard object.
57 | For example:
58 | [{'dashboard.id': 1,
59 | 'dashboard_element.id': 1,
60 | 'dashboard_element.type': 'vis',
61 | 'dashboard_element.result_source': 'Lookless',
62 | 'query.model': 'bq',
63 | 'query.view': 'order_items',
64 | 'query.formatted_fields': '["order_items.count"]',
65 | 'query.id': 59,
66 | 'dashboard.title': 'dash_1',
67 | 'look.id': None,
68 | 'sql_joins': ['`looker-private-demo.ecomm.order_items`']}]
69 | """
70 | for dash in self.dataextract:
71 | query_id = dash['query.id']
72 | sql_value = self.parse_sql(query_id)
73 |
74 | dash['sql_joins'] = sql_value
75 |
76 | return self.dataextract
77 |
--------------------------------------------------------------------------------
/lmanage/mapview/utils/.vimspector.json:
--------------------------------------------------------------------------------
1 | {
2 | "configurations": {
3 | "run": {
4 | "adapter": "debugpy",
5 | "configuration": {
6 | "request": "launch",
7 | "protocol": "auto",
8 | "stopOnEntry": true,
9 | "console": "integratedTerminal",
10 | "program": "${workspaceRoot}/get_groups.py",
11 | "cwd": "${workspaceRoot}"
12 | }
13 | }
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/lmanage/mapview/utils/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
--------------------------------------------------------------------------------
/lmanage/mapview/utils/create_df.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2021 Google LLC
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | https://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 | import os
17 | import logging
18 | from lmanage.pathlib import Path
19 | import json
20 | import pandas as pd
21 |
22 | logger = logging.getLogger(__name__)
23 |
24 |
25 | def create_df(data):
26 | logger.debug(type(data))
27 | data = json.loads(data) if isinstance(data, str) else data
28 | df = pd.DataFrame(data)
29 | return df
30 |
31 |
32 | def check_ini(ini):
33 | cwd = Path.cwd()
34 |
35 | if ini:
36 | parsed_ini_file = cwd.joinpath(ini)
37 | else:
38 | parsed_ini_file = None
39 |
40 | if os.path.isfile(parsed_ini_file):
41 | logger.info(f'opening your ini file at {parsed_ini_file}')
42 | return parsed_ini_file
43 | else:
44 | logger.debug(
45 | f'the path to your ini file is not valid at {parsed_ini_file}')
46 |
--------------------------------------------------------------------------------
/lmanage/mapview/utils/parse_lookml.py:
--------------------------------------------------------------------------------
1 | import os
2 | import lkml
3 | import glob
4 | import logging
5 | import coloredlogs
6 |
7 | logger = logging.getLogger(__name__)
8 | coloredlogs.install(level='INFO')
9 | logging.getLogger("looker_sdk").setLevel(logging.WARNING)
10 |
11 |
12 | def get_all_lkml_filepaths(starting_path):
13 | logger.info(starting_path)
14 | lookml_files = [f for f in glob.glob(
15 | starting_path + "/**/*" + 'lkml', recursive=True)]
16 | logger.debug(lookml_files)
17 | return lookml_files
18 |
19 |
20 | def parse_lookml(path):
21 | with open(path, 'r') as lookml_file:
22 | lookml = lkml.load(lookml_file)
23 | return lookml
24 |
25 |
26 | def get_parsed_lookml(lookml_file_paths):
27 | response = []
28 | for file in lookml_file_paths:
29 | parsed_lookml = parse_lookml(file)
30 | logger.debug(parsed_lookml)
31 | temp = {}
32 | temp[file] = parsed_lookml
33 | response.append(temp)
34 | return response
35 |
36 |
37 | '''
38 | parse a LookML file from lmanage.LookML to JSON
39 | Authors:
40 | Carl Anderson (carl.anderson@weightwatchers.com)
41 | '''
42 |
43 |
44 | class LookML():
45 |
46 | def __init__(self, infilepath):
47 | '''parse the LookML infilepath, convert to JSON, and then read into JSON object
48 | Args:
49 | infilepath (str): path to input LookML file
50 | Returns:
51 | JSON object of LookML
52 | '''
53 | if not os.path.exists(infilepath):
54 | raise IOError("Filename does not exist: %s" % infilepath)
55 |
56 | self.infilepath = infilepath
57 | if infilepath.endswith(".model.lkml"):
58 | self.filetype = 'model'
59 | elif infilepath.endswith(".view.lkml"):
60 | self.filetype = 'view'
61 | elif infilepath.endswith(".explore.lkml"):
62 | self.filetype = 'explore'
63 | elif infilepath.endswith(".lkml"):
64 | self.filetype = 'generic'
65 |
66 | else:
67 | raise Exception("Unsupported filename " + infilepath)
68 | self.base_filename = os.path.basename(infilepath)
69 | self.base_name = self.base_filename.replace(".model.lkml", "").replace(
70 | ".explore.lkml", "").replace(".view.lkml", "")
71 |
72 | with open(infilepath, 'r') as file:
73 | self.json_data = lkml.load(file)
74 |
75 | def views(self):
76 | """get views (if any) from lmanage.the LookML
77 | Returns:
78 | views (list) if any, None otherwise
79 | """
80 | if 'views' in self.json_data:
81 | return self.json_data['views']
82 | return None
83 |
84 | def has_views(self):
85 | """does this have one or more views?
86 | Returns:
87 | bool, whether this has views
88 | """
89 | vs = self.views()
90 | return (vs and len(vs) > 0)
91 |
92 | def explores(self):
93 | """get explores (if any) from lmanage.the LookML
94 | Returns:
95 | explores (list) if any, None otherwise
96 | """
97 | if 'explores' in self.json_data:
98 | return self.json_data['explores']
99 | return None
100 |
101 | def has_explores(self):
102 | """does this have one or more explores?
103 | Returns:
104 | bool, whether this has explores
105 | """
106 | es = self.explores()
107 | return (es and len(es) > 0)
108 |
--------------------------------------------------------------------------------
/lmanage/utils/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
--------------------------------------------------------------------------------
/lmanage/utils/errorhandling.py:
--------------------------------------------------------------------------------
1 | import ast
2 | import logging
3 | from looker_sdk import error
4 | from time import sleep
5 |
6 | logger = logging.getLogger(__name__)
7 |
8 |
9 | def calc_done_percent(iterator: int, total: int) -> str:
10 | percent = (iterator/total) * 100
11 | return f'{int(percent)} %'
12 |
13 |
14 | def return_error_message(input: str):
15 | # err_response = input.args[0]
16 | # err_response = json.loads(err_response)
17 | if len(input.errors) == 0:
18 | err_response = input.message
19 | else:
20 | err_response = input.errors[0].message
21 |
22 | return err_response
23 |
24 |
25 | def test_object_data(object: list) -> bool:
26 | if object:
27 | return True
28 | else:
29 | return False
30 |
31 |
32 | def user_authentication_test(sdk) -> bool:
33 | try:
34 | sdk.me()
35 | return True
36 | except error.SDKError():
37 | return False
38 |
39 |
40 | def return_sleep_message(call_number=0, quiet=False):
41 | call_number = call_number+1
42 | sleep_number = 2 ** call_number
43 | sleep(sleep_number)
44 | if not quiet:
45 | logger.warn(f'Looker is overwhelmed sleeping for {sleep_number} secs')
46 |
47 |
48 | def dedup_list_of_dicts(input: list):
49 | stringinput = list(set([str(x) for x in input]))
50 | deduped = [ast.literal_eval(x) for x in stringinput]
51 | return deduped
52 |
53 |
54 | def counter(i=0):
55 | i += 1
56 | return i
57 |
--------------------------------------------------------------------------------
/lmanage/utils/helpers.py:
--------------------------------------------------------------------------------
1 | def xstr(value):
2 | return value if value is not None else ''
3 |
4 |
5 | def nstr(value):
6 | return None if value == '' else value
7 |
--------------------------------------------------------------------------------
/lmanage/utils/logger_creation.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from rich.logging import RichHandler
3 | import datetime
4 | import os
5 |
6 |
7 | def create_timestamp():
8 | timestamp = datetime.datetime.now()
9 | unix_ts = timestamp.timestamp()
10 | trunc_unix_ts = unix_ts - (unix_ts % 60)
11 | return int(trunc_unix_ts)
12 |
13 |
14 | def init_logger(dunder_name, testing_mode) -> logging.Logger:
15 | log_format = (
16 | '%(asctime)s - '
17 | '%(name)s - '
18 | '%(funcName)s - '
19 | '%(levelname)s - '
20 | '%(message)s'
21 | )
22 |
23 | logging.basicConfig(
24 | format=log_format,
25 | datefmt=".",
26 | handlers=[RichHandler()],
27 | )
28 | logger = logging.getLogger(dunder_name)
29 |
30 | if testing_mode:
31 | logger.setLevel(logging.DEBUG)
32 | else:
33 | logger.setLevel(logging.INFO)
34 |
35 | unix_ts = create_timestamp()
36 |
37 | # Create logs folder if not exists
38 | os.makedirs('logs', exist_ok=True)
39 |
40 | # Output full log
41 | fh = logging.FileHandler(f'logs/lomt_{unix_ts}.log')
42 | fh.setLevel(logging.DEBUG)
43 | formatter = logging.Formatter(log_format)
44 | fh.setFormatter(formatter)
45 | logger.addHandler(fh)
46 |
47 | return logger
48 |
--------------------------------------------------------------------------------
/lmanage/utils/parse_yaml.py:
--------------------------------------------------------------------------------
1 | import json
2 | import ruamel.yaml as yaml
3 | import logging
4 | from lmanage.utils import logger_creation as log_color
5 |
6 |
7 | class Yaml:
8 | def __init__(self, yaml_path):
9 | self.yaml_path = yaml_path
10 | with open(self.yaml_path, 'r') as file:
11 | self.parsed_yaml = yaml.load(file, Loader=yaml.BaseLoader)
12 |
13 | def get_metadata(self, type):
14 | if type == 'folder_metadata':
15 | metadata = self.__get_folder_metadata()
16 | elif type == 'permission_set_metadata':
17 | metadata = self.__get_permission_metadata()
18 | elif type == 'model_set_metadata':
19 | metadata = self.__get_model_set_metadata()
20 | elif type == 'role_metadata':
21 | metadata = self.__get_role_metadata()
22 | elif type == 'user_attribute_metadata':
23 | metadata = self.__get_user_attribute_metadata()
24 | elif type == 'look_metadata':
25 | metadata = self.__get_look_metadata()
26 | elif type == 'dashboard_metadata':
27 | metadata = self.__get_dashboard_metadata()
28 | elif type == 'board_metadata':
29 | metadata = self.__get_board_metadata()
30 |
31 | if not metadata:
32 | print('')
33 | # log f'No {type} specified. Please check your yaml file at {self.yaml_path}.')
34 |
35 | return metadata
36 |
37 | def __get_permission_metadata(self):
38 | response = []
39 | for objects in self.parsed_yaml:
40 | if 'permissions' in list(objects.keys()):
41 | response.append(objects)
42 | response = [r for r in response if r.get('name') != 'Admin']
43 | return response
44 |
45 | def __get_model_set_metadata(self):
46 | response = []
47 | for objects in self.parsed_yaml:
48 | if 'models' in list(objects.keys()):
49 | response.append(objects)
50 | return response
51 |
52 | def __get_role_metadata(self):
53 | response = []
54 | for objects in self.parsed_yaml:
55 | if 'permission_set' and 'model_set' in list(objects.keys()):
56 | response.append(objects)
57 | response = [r for r in response if r.get('name') != 'Admin']
58 | return response
59 |
60 | def __get_folder_metadata(self):
61 | response = []
62 | for objects in self.parsed_yaml:
63 | if 'content_metadata_id' in list(objects.keys()):
64 | response.append(objects)
65 | return response
66 |
67 | def __get_user_attribute_metadata(self):
68 | response = []
69 | for objects in self.parsed_yaml:
70 | if 'hidden_value' in list(objects.keys()):
71 | response.append(objects)
72 | return response
73 |
74 | def __get_look_metadata(self):
75 | response = []
76 | for objects in self.parsed_yaml:
77 | if 'query_obj' in list(objects.keys()):
78 | response.append(objects)
79 | return response
80 |
81 | def __get_dashboard_metadata(self):
82 | response = []
83 | for objects in self.parsed_yaml:
84 | if 'lookml' in list(objects.keys()):
85 | response.append(objects)
86 | return response
87 |
88 | def __get_board_metadata(self):
89 | response = []
90 | for objects in self.parsed_yaml:
91 | if 'board_sections' in list(objects.keys()):
92 | response.append(objects)
93 | return response
94 |
--------------------------------------------------------------------------------
/output/hugo.csv:
--------------------------------------------------------------------------------
1 | ,dashboard_id,element_id,sql_joins,fields_used,sql_table_name,potential_join,used_joins,used_view_names,unused_joins
2 | 0,1,1,['`looker-private-demo.ecomm.order_items`'],"[""order_items.created_month"", ""order_items.count""]","['`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items', 'order_facts', 'inventory_items', 'users', 'user_order_facts', 'products', 'repeat_purchase_facts', 'distribution_centers', 'test_ndt']",['`looker-private-demo.ecomm.order_items`'],['order_items'],"['distribution_centers', 'inventory_items', 'order_facts', 'products', 'repeat_purchase_facts', 'test_ndt', 'user_order_facts', 'users']"
3 | 1,1,2,"['`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.products`']","[""inventory_items.product_distribution_center_id"", ""products.department"", ""products.count""]","['`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items', 'order_facts', 'inventory_items', 'users', 'user_order_facts', 'products', 'repeat_purchase_facts', 'distribution_centers', 'test_ndt']","['`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.products`']","['products', 'order_items', 'inventory_items']","['distribution_centers', 'order_facts', 'repeat_purchase_facts', 'test_ndt', 'user_order_facts', 'users']"
4 | 2,1,3,"['`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.products`']","[""products.department"", ""products.count"", ""inventory_items.count"", ""users.city"", ""users.age""]","['`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items', 'order_facts', 'inventory_items', 'users', 'user_order_facts', 'products', 'repeat_purchase_facts', 'distribution_centers', 'test_ndt']","['`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.products`']","['products', 'users', 'order_items', 'inventory_items']","['distribution_centers', 'order_facts', 'repeat_purchase_facts', 'test_ndt', 'user_order_facts']"
5 | 3,1,5,"['`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', 'test_ndt', '`looker-private-demo.ecomm.order_items`']","[""order_items.total_revenue"", ""order_items.created_month"", ""order_items.user_id"", ""test_ndt.total_revenue"", ""test_ndt.user_id"", ""products.brand""]","['`looker-private-demo.ecomm.distribution_centers`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', '`looker-private-demo.ecomm.order_items`', '`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.events`']","['order_items', 'order_facts', 'inventory_items', 'users', 'user_order_facts', 'products', 'repeat_purchase_facts', 'distribution_centers', 'test_ndt']","['`looker-private-demo.ecomm.inventory_items`', '`looker-private-demo.ecomm.products`', '`looker-private-demo.ecomm.users`', 'test_ndt', '`looker-private-demo.ecomm.order_items`']","['test_ndt', 'products', 'users', 'order_items', 'inventory_items']","['distribution_centers', 'order_facts', 'repeat_purchase_facts', 'user_order_facts']"
6 |
--------------------------------------------------------------------------------
/output/instance_mapview.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/output/instance_mapview.xlsx
--------------------------------------------------------------------------------
/output/l233_output.yaml:
--------------------------------------------------------------------------------
1 | # FOLDER_PERMISSIONS
2 | # Opening Session Welcome to the Capturator, this is the Folder place
3 | # -----------------------------------------------------
4 |
5 | - !LookerFolder
6 | parent_id:
7 | id: '1'
8 | name: Shared
9 | subfolder:
10 | - !LookerFolder
11 | parent_id: '1'
12 | id: '83'
13 | name: monkey
14 | subfolder:
15 | - !LookerFolder
16 | parent_id: '83'
17 | id: '84'
18 | name: monkey_sub
19 | subfolder:
20 | - !LookerFolder
21 | parent_id: '84'
22 | id: '85'
23 | name: monkey_sub_sub
24 | subfolder: []
25 | content_metadata_id: '100'
26 | team_edit: []
27 | team_view: []
28 | content_metadata_id: '99'
29 | team_edit: []
30 | team_view: []
31 | content_metadata_id: '98'
32 | team_edit: []
33 | team_view: []
34 | - !LookerFolder
35 | parent_id: '1'
36 | id: '81'
37 | name: test
38 | subfolder:
39 | - !LookerFolder
40 | parent_id: '81'
41 | id: '82'
42 | name: blok
43 | subfolder: []
44 | content_metadata_id: '97'
45 | team_edit: []
46 | team_view: []
47 | content_metadata_id: '96'
48 | team_edit: []
49 | team_view: []
50 | content_metadata_id: '1'
51 | team_edit:
52 | - All Users
53 | team_view: []
54 | # Looker Role
55 | # Opening Session Welcome to the Capturator, this is the Role place
56 | # -----------------------------------------------------
57 |
58 |
59 |
60 | # PERMISSION SETS
61 | - !LookerPermissionSet
62 | permissions:
63 | - access_data
64 | - clear_cache_refresh
65 | - download_without_limit
66 | - mobile_app_access
67 | - schedule_look_emails
68 | - see_drill_overlay
69 | - see_lookml_dashboards
70 | - see_looks
71 | - see_user_dashboards
72 | - send_to_integration
73 | name: Viewer
74 | - !LookerPermissionSet
75 | permissions:
76 | - access_data
77 | - can_create_forecast
78 | - clear_cache_refresh
79 | - create_custom_fields
80 | - create_table_calculations
81 | - download_without_limit
82 | - explore
83 | - manage_spaces
84 | - mobile_app_access
85 | - save_content
86 | - schedule_look_emails
87 | - see_drill_overlay
88 | - see_lookml
89 | - see_lookml_dashboards
90 | - see_looks
91 | - see_sql
92 | - see_user_dashboards
93 | - send_to_integration
94 | name: User
95 | - !LookerPermissionSet
96 | permissions:
97 | - access_data
98 | - can_create_forecast
99 | - clear_cache_refresh
100 | - create_custom_fields
101 | - create_table_calculations
102 | - deploy
103 | - develop
104 | - download_without_limit
105 | - explore
106 | - manage_spaces
107 | - mobile_app_access
108 | - save_content
109 | - schedule_look_emails
110 | - see_drill_overlay
111 | - see_lookml
112 | - see_lookml_dashboards
113 | - see_looks
114 | - see_pdts
115 | - see_sql
116 | - see_user_dashboards
117 | - send_to_integration
118 | - use_sql_runner
119 | name: Developer
120 |
121 |
122 | # MODEL SETS
123 | - !LookerModelSet
124 | models:
125 | - naga_234_test
126 | - lookml-diagram
127 | - large_project
128 | - data_block_acs_bigquery
129 | - extension-api-explorer
130 | - cloud_logging
131 | name: All
132 |
133 |
134 | # LOOKER ROLES
135 | - !LookerRoles
136 | permission_set: User
137 | model_set: All
138 | teams:
139 | - User
140 | name: User
141 | - !LookerRoles
142 | permission_set: Developer
143 | model_set: All
144 | teams: []
145 | name: Developer
146 | - !LookerRoles
147 | permission_set: Viewer
148 | model_set: All
149 | teams: []
150 | name: Viewer
151 | - !LookerRoles
152 | permission_set: Developer
153 | model_set: All
154 | teams: []
155 | name: haylen_mtr
156 |
157 |
158 | # USER_ATTRIBUTES
159 | - !LookerUserAttribute
160 | name: first_name
161 | uatype: string
162 | hidden_value: false
163 | user_view: 'True'
164 | user_edit: 'True'
165 | default_value: ''
166 | teams: []
167 | - !LookerUserAttribute
168 | name: id
169 | uatype: number
170 | hidden_value: false
171 | user_view: 'True'
172 | user_edit: 'False'
173 | default_value: ''
174 | teams: []
175 | - !LookerUserAttribute
176 | name: landing_page
177 | uatype: relative_url
178 | hidden_value: false
179 | user_view: 'True'
180 | user_edit: 'True'
181 | default_value: /browse
182 | teams: []
183 | - !LookerUserAttribute
184 | name: locale
185 | uatype: string
186 | hidden_value: false
187 | user_view: 'True'
188 | user_edit: 'False'
189 | default_value: ''
190 | teams: []
191 | - !LookerUserAttribute
192 | name: number_format
193 | uatype: string
194 | hidden_value: false
195 | user_view: 'True'
196 | user_edit: 'False'
197 | default_value: ''
198 | teams: []
199 | - !LookerUserAttribute
200 | name: rajk
201 | uatype: string
202 | hidden_value: false
203 | user_view: 'True'
204 | user_edit: 'True'
205 | default_value: ''
206 | teams: []
207 |
--------------------------------------------------------------------------------
/output/l233_output_content.yaml:
--------------------------------------------------------------------------------
1 |
2 |
3 | # LookData
4 | - !LookObject
5 | legacy_folder_id: '81'
6 | look_id: '6'
7 | title: Total Test
8 | query_obj:
9 | model: system__activity
10 | view: event
11 | fields:
12 | - event.created_date
13 | - event.count
14 | pivots:
15 | fill_fields:
16 | - event.created_date
17 | filters:
18 | filter_expression:
19 | sorts:
20 | - event.created_date desc
21 | limit: '500'
22 | column_limit: '50'
23 | total: true
24 | row_total:
25 | subtotals:
26 | vis_config:
27 | show_view_names: false
28 | show_row_numbers: true
29 | transpose: false
30 | truncate_text: true
31 | hide_totals: false
32 | hide_row_totals: false
33 | size_to_fit: true
34 | table_theme: white
35 | limit_displayed_rows: false
36 | enable_conditional_formatting: false
37 | header_text_alignment: left
38 | header_font_size: 12
39 | rows_font_size: 12
40 | conditional_formatting_include_totals: false
41 | conditional_formatting_include_nulls: false
42 | hidden_pivots: {}
43 | type: looker_grid
44 | defaults_version: 1
45 | filter_config: {}
46 | visible_ui_sections:
47 | dynamic_fields: '[{"category":"table_calculation","expression":" sum(${event.count})\n","label":"event
48 | count","value_format":null,"value_format_name":null,"_kind_hint":"measure","table_calculation":"event_count","_type_hint":"number"}]'
49 | query_timezone:
50 | description: ''
51 | - !LookObject
52 | legacy_folder_id: '83'
53 | look_id: '7'
54 | title: look
55 | query_obj:
56 | model: system__activity
57 | view: look
58 | fields:
59 | - look.id
60 | - look.moved_to_trash
61 | - look.link
62 | - history.id
63 | - look.title
64 | - look.count
65 | pivots:
66 | fill_fields:
67 | filters:
68 | filter_expression:
69 | sorts:
70 | - look.count desc 0
71 | limit: '500'
72 | column_limit:
73 | total:
74 | row_total:
75 | subtotals:
76 | vis_config:
77 | show_view_names: false
78 | show_row_numbers: true
79 | transpose: false
80 | truncate_text: true
81 | hide_totals: false
82 | hide_row_totals: false
83 | size_to_fit: true
84 | table_theme: white
85 | limit_displayed_rows: false
86 | enable_conditional_formatting: false
87 | header_text_alignment: left
88 | header_font_size: 12
89 | rows_font_size: 12
90 | conditional_formatting_include_totals: false
91 | conditional_formatting_include_nulls: false
92 | type: looker_grid
93 | defaults_version: 1
94 | filter_config: {}
95 | visible_ui_sections:
96 | dynamic_fields:
97 | query_timezone:
98 | description: ''
99 | - !LookObject
100 | legacy_folder_id: '85'
101 | look_id: '8'
102 | title: look
103 | query_obj:
104 | model: system__activity
105 | view: look
106 | fields:
107 | - look.id
108 | - look.moved_to_trash
109 | - look.link
110 | - history.id
111 | - look.title
112 | - look.count
113 | pivots:
114 | fill_fields:
115 | filters:
116 | filter_expression:
117 | sorts:
118 | - look.count desc 0
119 | limit: '500'
120 | column_limit:
121 | total:
122 | row_total:
123 | subtotals:
124 | vis_config:
125 | show_view_names: false
126 | show_row_numbers: true
127 | transpose: false
128 | truncate_text: true
129 | hide_totals: false
130 | hide_row_totals: false
131 | size_to_fit: true
132 | table_theme: white
133 | limit_displayed_rows: false
134 | enable_conditional_formatting: false
135 | header_text_alignment: left
136 | header_font_size: 12
137 | rows_font_size: 12
138 | conditional_formatting_include_totals: false
139 | conditional_formatting_include_nulls: false
140 | type: looker_grid
141 | defaults_version: 1
142 | filter_config: {}
143 | visible_ui_sections:
144 | dynamic_fields:
145 | query_timezone:
146 | description: ''
147 |
148 |
149 | # Dashboard Content
150 | - !DashboardObject
151 | legacy_folder_id: '81'
152 | lookml: "- dashboard: new_dashboard\n title: New Dashboard\n layout: newspaper\n\
153 | \ preferred_viewer: dashboards-next\n description: ''\n preferred_slug: SkRfx99hbJYDPBuNXZRsdi\n\
154 | \ elements:\n - title: Untitled\n name: Untitled\n model: system__activity\n\
155 | \ explore: field_usage\n type: table\n fields: [field_usage.field]\n\
156 | \ limit: 500\n row:\n col:\n width:\n height:\n"
157 | dashboard_id: '7'
158 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [tool.poetry]
2 | name = "lmanage"
3 | version = "0.3.36"
4 | description = "LManage is a collection of useful tools for Looker admins to help curate and cleanup content and it's associated source LookML."
5 | keywords = ["keyword", "another_keyword"]
6 | authors = ["hselbie "]
7 | readme = "README.md"
8 | homepage = "https://github.com/looker-open-source/lmanage"
9 | repository = "https://github.com/looker-open-source/lmanage"
10 |
11 | [tool.poetry.dependencies]
12 | python = ">=3.8,<3.12"
13 | looker-sdk = "^23.8.1"
14 | poetry = "^1.4.2"
15 | rich = "^13.4.2"
16 | click = "^8.1.3"
17 | ruamel-yaml = "^0.17.32"
18 | tqdm = "^4.65.0"
19 | tenacity = "^8.2.2"
20 | yaspin = "^2.3.0"
21 |
22 | [tool.poetry.dev-dependencies]
23 |
24 | [build-system]
25 | requires = ["poetry-core>=1.0.0", "setuptools", "wheel", "Cython"]
26 | build-backend = "poetry.core.masonry.api"
27 |
28 | [tool.poetry.scripts]
29 | lmanage = 'lmanage.cli:lmanage'
30 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | __version__ = '0.1.4'
16 |
--------------------------------------------------------------------------------
/tests/example_yamls/dev_output.yaml:
--------------------------------------------------------------------------------
1 | # FOLDER_PERMISSIONS
2 | # Opening Session Welcome to the Capturator, this is the Folder place
3 | # -----------------------------------------------------
4 |
5 | - !LookerFolder
6 | parent_id:
7 | id: '1'
8 | name: Shared
9 | subfolder:
10 | - !LookerFolder
11 | parent_id: '1'
12 | id: '202'
13 | name: test
14 | subfolder:
15 | - !LookerFolder
16 | parent_id: '202'
17 | id: '203'
18 | name: emea_cosmo
19 | subfolder: []
20 | content_metadata_id: '205'
21 | team_edit: []
22 | team_view:
23 | - new york team
24 | - Freddy
25 | - HugoTesty
26 | content_metadata_id: '204'
27 | team_edit: []
28 | team_view: []
29 | - !LookerFolder
30 | parent_id: '1'
31 | id: '201'
32 | name: emea_cosmo
33 | subfolder:
34 | - !LookerFolder
35 | name: hugo_test
36 | subfolder: []
37 | content_metadata_id: '205'
38 | team_edit: []
39 | team_view:
40 | - new york team
41 | - Freddy
42 | - HugoTesty
43 | content_metadata_id: '204'
44 | team_edit: []
45 | team_view: []
46 | content_metadata_id: '203'
47 | team_edit: []
48 | team_view:
49 | - BusinessOperations
50 | - emea
51 | content_metadata_id: '1'
52 | team_edit: []
53 | team_view:
54 | - All Users
55 | # Looker Role
56 | # Opening Session Welcome to the Capturator, this is the Role place
57 | # -----------------------------------------------------
58 |
59 |
60 |
61 | # PERMISSION SETS
62 | - !LookerPermissionSet
63 | permissions:
64 | - access_data
65 | - download_with_limit
66 | - explore
67 | - schedule_look_emails
68 | - see_looks
69 | name: gghhhh
70 | - !LookerPermissionSet
71 | permissions:
72 | - access_data
73 | - see_looks
74 | - see_user_dashboards
75 | name: test
76 | - !LookerPermissionSet
77 | permissions:
78 | - access_data
79 | - clear_cache_refresh
80 | - create_alerts
81 | - create_prefetches
82 | - create_public_looks
83 | - create_table_calculations
84 | - deploy
85 | - develop
86 | - download_with_limit
87 | - download_without_limit
88 | - explore
89 | - follow_alerts
90 | - manage_homepage
91 | - manage_models
92 | - manage_spaces
93 | - manage_stereo
94 | - save_content
95 | - schedule_external_look_emails
96 | - schedule_look_emails
97 | - see_drill_overlay
98 | - see_lookml
99 | - see_lookml_dashboards
100 | - see_looks
101 | - see_sql
102 | - see_user_dashboards
103 | - send_outgoing_webhook
104 | - send_to_integration
105 | - send_to_s3
106 | - send_to_sftp
107 | - support_access_toggle
108 | - use_sql_runner
109 | name: addfa
110 |
111 |
112 | # MODEL SETS
113 | - !LookerModelSet
114 | models:
115 | - extension-api-explorer
116 | name: All
117 |
118 |
119 | # LOOKER ROLES
120 | - !LookerRoles
121 | permission_set: test
122 | model_set: All
123 | teams:
124 | - BusinessOperations
125 | - Freddy
126 | - HugoTesty
127 | name: Operations_User
128 | - !LookerRoles
129 | permission_set: addfa
130 | model_set: All
131 | teams:
132 | - westlake
133 | - DrewEdit
134 | - BusinessOperations_BO_Dev
135 | name: BusinessOperations_Developer
136 | - !LookerRoles
137 | permission_set: gghhhh
138 | model_set: All
139 | teams:
140 | - BusinessOperations
141 | name: BusinessOperations_User
142 |
143 |
144 | # USER_ATTRIBUTES
145 | - !LookerUserAttribute
146 | name: city
147 | uatype: string
148 | hidden_value: false
149 | user_view: 'True'
150 | user_edit: 'False'
151 | default_value: '%, NULL'
152 | teams:
153 | - london team: London
154 | - new york team: New York
155 | - safetyfirst: least_privilege
156 | - !LookerUserAttribute
157 | name: first_name
158 | uatype: string
159 | hidden_value: false
160 | user_view: 'True'
161 | user_edit: 'True'
162 | default_value: ''
163 | teams: []
164 | - !LookerUserAttribute
165 | name: landing_page
166 | uatype: relative_url
167 | hidden_value: false
168 | user_view: 'True'
169 | user_edit: 'True'
170 | default_value: /browse
171 | teams: []
172 | - !LookerUserAttribute
173 | name: locale
174 | uatype: string
175 | hidden_value: false
176 | user_view: 'True'
177 | user_edit: 'False'
178 | default_value: en
179 | teams: []
180 | - !LookerUserAttribute
181 | name: number_format
182 | uatype: string
183 | hidden_value: false
184 | user_view: 'True'
185 | user_edit: 'False'
186 | default_value: 1,234.56
187 | teams: []
188 |
--------------------------------------------------------------------------------
/tests/example_yamls/folder_permission_input.yaml:
--------------------------------------------------------------------------------
1 | BusinessOperations_folder_permissions:
2 | folder:
3 | name: BusinessOperations
4 | team_edit: BusinessOperations
5 | team_view:
6 | - Product
7 | - EngineeringSecurity
8 | - Friends_Maps
9 | Cameos_folder_permissions:
10 | folder:
11 | name: Cameos
12 | team_edit: Cameos
13 | team_view:
14 | - Product
15 | - InformationTechnology
16 | - DataScience
17 | - ProductManagement
18 | lk2bitmoji_folder_permissions:
19 | folder:
20 | name: lk2bitmoji
21 | team_edit: lk2bitmoji
22 | team_view:
23 | - Product
24 | - InformationTechnology
25 | - Strategy
26 |
27 |
--------------------------------------------------------------------------------
/tests/example_yamls/folder_yaml.yaml:
--------------------------------------------------------------------------------
1 | # We implement three categories of Looker user groups:
2 | #
3 | # 1. Roles: these groups are assigned Looker Roles, which control what
4 | # functionality a user has access to in Looker.
5 | # 2. Content: these groups control access to Folders, which organize
6 | # content (dashboards and Looks) in Looker.
7 | # 3. Attributes: these groups are assigned attributes, which are used to
8 | # control access to rows and columns of data via access filters and
9 | # access frants.
10 |
11 | #########
12 | # MODEL_SET_ROLES #
13 | #########
14 | role_admin:
15 | role: Admin
16 | team:
17 | - Cameos
18 |
19 | role_BusinessOperations_Developer:
20 | role: BODevelopers
21 | permissions:
22 | - access_data
23 | - can_see_system_activity
24 | - see_dashboards
25 | - clear_cache_refresh
26 | - create_table_calculations
27 | - deploy
28 | - develop
29 | - download_without_limit
30 | - explore
31 | - manage_spaces
32 | - mobile_app_access
33 | - save_content
34 | - schedule_look_emails
35 | - see_drill_overlay
36 | - see_lookml
37 | - see_sql
38 | - see_user_dashboards
39 | - send_to_integration
40 | - use_sql_runner
41 | model_set:
42 | - lk2_business_operations
43 | - DrewsModel
44 | team:
45 | - BusinessOperations_BO_Dev
46 | - DrewEdit
47 |
48 | role_BusinessOperations_User:
49 | role: BOUser
50 | permissions:
51 | - access_data
52 | - clear_cache_refresh
53 | - create_table_calculations
54 | - download_without_limit
55 | - explore
56 | - manage_spaces
57 | - mobile_app_access
58 | - save_content
59 | - schedule_look_emails
60 | - see_drill_overlay
61 | - see_lookml
62 | - see_lookml_dashboards
63 | - see_looks
64 | - see_sql
65 | - see_user_dashboards
66 | - send_to_integration
67 | model_set:
68 | - lk4_huggy
69 | models:
70 | - testy
71 | - buggy
72 | - lk2_jungy
73 | team:
74 | - BusinessOperations
75 | - Freddy
76 |
77 |
78 |
79 | #####################
80 | # FOLDER PERMISSONS #
81 | #####################
82 | BusinessOperations_folder_permissions:
83 | folder:
84 | name: BusinessOperations
85 | team_edit:
86 | - BusinessOperations
87 | team_view:
88 | - Snaptest
89 | - Cameos
90 | path:
91 | - Shared/
92 |
93 | Cameos_folder_permissions:
94 | folder:
95 | name: Cameos
96 | team_edit:
97 | - Cameos
98 | team_view:
99 | - BusinessOperations
100 | - HugoGroup
101 | Hugo_folder_permissions:
102 | folder:
103 | name: Hugo
104 | team_edit:
105 | - Hugo
106 | - HugoEdit
107 | team_view:
108 | - BusinessOperations
109 | - NotHugoGroup
110 |
111 | Drews_folder_permissions:
112 | folder:
113 | name: Drew
114 | team_edit:
115 | - DrewEdit
116 | team_view:
117 | - BusinessOperations
118 |
119 |
120 |
121 |
122 | ###################
123 | # User Attributes #
124 | ###################
125 | # attr_region:
126 | ua_region_all:
127 | name: region_all
128 | type: string
129 | hidden_value: false
130 | user_view: true
131 | user_edit: false
132 | value:
133 | - us
134 | - ag
135 | - bb
136 | - dd
137 | team:
138 | - Cameos
139 | - Freddy
140 | - AudreyGroup
141 |
142 | ua_region_testy:
143 | name: region_testy
144 | type: string
145 | hidden_value: false
146 | user_view: true
147 | user_edit: false
148 | value:
149 | - us
150 | - ag
151 | - bb
152 | team:
153 | - Cameos
154 | - AudreyGroup
155 |
156 |
157 | ua_attr_can_see_hugo:
158 | name: can_see_hugo
159 | type: string
160 | hidden_value: false
161 | user_view: true
162 | user_edit: false
163 | value:
164 | - 'No'
165 | team:
166 | - Cameos
167 | - CanSeeDAUGroup
168 |
169 | ua_attr_can_see_dau:
170 | name: can_see_dau
171 | type: string
172 | hidden_value: false
173 | user_view: true
174 | user_edit: false
175 | value:
176 | - yes
177 | team:
178 | - Cameos
179 | - CanSeeDAUGroup
180 |
--------------------------------------------------------------------------------
/tests/example_yamls/fullinstance.yaml:
--------------------------------------------------------------------------------
1 | #########
2 | # MODEL_SET_ROLES #
3 | #########
4 | permission_sets:
5 | developer:
6 | permissions:
7 | - access_data
8 | - can_see_system_activity
9 | - see_dashboards
10 | - clear_cache_refresh
11 | - create_table_calculations
12 | - deploy
13 | - develop
14 | - download_without_limit
15 | - explore
16 | - manage_spaces
17 | - mobile_app_access
18 | - save_content
19 | - schedule_look_emails
20 | - see_drill_overlay
21 | - see_lookml
22 | - see_sql
23 | - see_user_dashboards
24 | - send_to_integration
25 | - use_sql_runner
26 |
27 | nodeploy_developer:
28 | permissions:
29 | - access_data
30 | - can_see_system_activity
31 | - see_dashboards
32 | - clear_cache_refresh
33 | - create_table_calculations
34 | - develop
35 | - download_without_limit
36 | - explore
37 | - manage_spaces
38 | - mobile_app_access
39 | - save_content
40 | - schedule_look_emails
41 | - see_drill_overlay
42 | - see_lookml
43 | - see_sql
44 | - see_user_dashboards
45 | - send_to_integration
46 | - use_sql_runner
47 |
48 | user:
49 | permissions:
50 | - access_data
51 | - clear_cache_refresh
52 | - create_table_calculations
53 | - download_without_limit
54 | - explore
55 | - manage_spaces
56 | - mobile_app_access
57 | - save_content
58 | - schedule_look_emails
59 | - see_drill_overlay
60 | - see_lookml
61 | - see_lookml_dashboards
62 | - see_looks
63 | - see_sql
64 | - see_user_dashboards
65 | - send_to_integration
66 |
67 | model_sets:
68 | lk1_test_set:
69 | models:
70 | - test
71 | - test2
72 | lk4_huggy:
73 | models:
74 | - test
75 | - test1
76 | - test2
77 |
78 | roles:
79 | BusinessOperations_Developer:
80 | permission_set: developer
81 | model_set: lk1_test_set
82 | team:
83 | - BusinessOperations_BO_Dev
84 | - DrewEdit
85 | - westlake
86 |
87 | BusinessOperations_User:
88 | permission_set: user
89 | model_set: lk4_huggy
90 | team:
91 | - BusinessOperations
92 | - Freddy
93 |
94 | Operations_User:
95 | permission_set: user
96 | model_set: lk4_huggy
97 | team:
98 | - BusinessOperations
99 | - Freddy
100 | - HugoTesty
101 |
102 | #####################
103 | # FOLDER PERMISSONS #
104 | #####################
105 | folder_permissions:
106 | business_operations_folder:
107 | - name: 'Business Operations'
108 | team_view:
109 | - BusinessOperations
110 | - Snaptest
111 | subfolder:
112 | - name: test_sub
113 | team_edit:
114 | - Freddy
115 | team_view:
116 | - hugo
117 | - name: test_sub2
118 | subfolder:
119 | - name: test_sub_sub
120 | team_edit:
121 | - Famke
122 | team_view:
123 | - hugle
124 | - name: subdiddy
125 | subfolder:
126 | - name: hugle_testy
127 | team_edit:
128 | - Freddy
129 | team_view:
130 | - hugle
131 |
132 | sexy_time_folder:
133 | - name: 'sexy time'
134 | team_edit:
135 | - sexy_group1
136 | team_view:
137 | - sexy_group2
138 | subfolder:
139 | - name: sexy_sub
140 |
141 | suffering_succotash_folder:
142 | - name: 'suffering succotash'
143 | team_edit:
144 | - sexy_group1
145 | team_view:
146 | - sexy_group2
147 | subfolder:
148 | - name: another_one
149 | team_edit:
150 | - HugoTesty
151 | - new_group
152 | team_view:
153 | - newer_group
154 | subfolder:
155 | - name: sharkytesttest
156 | team_edit:
157 | - colin
158 | - sharon
159 | team_view:
160 | - nick
161 | - name: ps_allhands
162 | team_view:
163 | - ps_allhand1
164 | team_edit:
165 | - ps_allhand2
166 |
167 |
168 |
169 | ###################
170 | # User Attributes #
171 | ###################
172 | # attr_region:
173 | user_attributes:
174 | region_all:
175 | type: string
176 | hidden_value: false
177 | user_view: true
178 | user_edit: false
179 | value:
180 | - us
181 | - ag
182 | - bb
183 | - dd
184 | team:
185 | - Cameos
186 | - Freddy
187 | - AudreyGroup
188 |
189 | can_see_sharky:
190 | type: string
191 | hidden_value: false
192 | user_view: true
193 | user_edit: false
194 | value:
195 | - 'Sharky'
196 | team:
197 | - HugoTesty
198 | - CanSeeDAUGroup
199 |
200 | can_see_hugo:
201 | type: string
202 | hidden_value: false
203 | user_view: true
204 | user_edit: false
205 | value:
206 | - 'No'
207 | team:
208 | - Cameos
209 | - CanSeeDAUGroup
210 |
211 |
212 |
--------------------------------------------------------------------------------
/tests/migrate_schedule.py:
--------------------------------------------------------------------------------
1 | import looker_sdk
2 | import os
3 | import json
4 | from lmanage.datetime import datetime
5 | from lmanage.typing import Callable, List
6 |
7 | ini = '/usr/local/google/home/hugoselbie/code_sample/py/ini/k8.ini'
8 | # os.environ["LOOKERSDK_BASE_URL"] = "https://profservices.dev.looker.com:19999" #If your looker URL has .cloud in it (hosted on GCP), do not include :19999 (ie: https://your.cloud.looker.com).
9 | # os.environ["LOOKERSDK_API_VERSION"] = "3.1" #3.1 is the default version. You can change this to 4.0 if you want.
10 | # os.environ["LOOKERSDK_VERIFY_SSL"] = "true" #Defaults to true if not set. SSL verification should generally be on unless you have a real good reason not to use it. Valid options: true, y, t, yes, 1.
11 | # os.environ["LOOKERSDK_TIMEOUT"] = "120" #Seconds till request timeout. Standard default is 120.
12 |
13 | # #Get the following values from lmanage.your Users page in the Admin panel of your Looker instance > Users > Your user > Edit API keys. If you know your user id, you can visit https://your.looker.com/admin/users//edit.
14 | # os.environ["LOOKERSDK_CLIENT_ID"] = "xxxxx" #No defaults.
15 | # os.environ["LOOKERSDK_CLIENT_SECRET"] = "xxxxx" #No defaults. This should be protected at all costs. Please do not leave it sitting here, even if you don't share this document.
16 |
17 | sdk = looker_sdk.init31(config_file=ini)
18 | print('Looker SDK 3.1 initialized successfully.')
19 |
20 | # How to define old user..
21 | # User hasn't logged in for n_days based on credentials_email or credentials_saml
22 | n_days = 60
23 |
24 | # function to calculate number of days between last log in date and current date
25 |
26 |
27 | def days_between(d1: datetime, d2: datetime) -> int:
28 | """Return abs difference in days between two timestamps"""
29 | d1 = datetime.strptime(d1, "%Y-%m-%d")
30 | d2 = datetime.strptime(d2, "%Y-%m-%d")
31 | return abs((d2 - d1).days)
32 |
33 | # function to get a list of the old users
34 |
35 |
36 | def fetch_old_users(n_days: int, exclude_email: bool = True, exclude_saml: bool = True):
37 | """Fetch all users who haven't logged in for more than n_days"""
38 | users = sdk.all_users(
39 | fields='id,email,credentials_email(logged_in_at),credentials_saml(logged_in_at)')
40 | if exclude_email:
41 | users = [u for u in users
42 | if u.get('credentials_email') is not None
43 | ]
44 | inactive_email = [u for u in users
45 | if u['credentials_email']['logged_in_at'] != None
46 | and days_between(
47 | u['credentials_email']['logged_in_at'].split('T')[0],
48 | str(datetime.today()).split()[0]
49 | ) > n_days
50 | ]
51 | if exclude_saml:
52 | users = [u for u in users
53 | if u.get('credentials_saml') is not None
54 | ]
55 | inactive_saml = [u for u in users
56 | if u['credentials_saml']['logged_in_at'] != None
57 | and days_between(
58 | u['credentials_saml']['logged_in_at'].split('T')[0],
59 | str(datetime.today()).split()[0]
60 | ) > n_days
61 | ]
62 | user_id_inactive_email = [u.id for u in inactive_email]
63 | user_id_inactive_saml = [u.id for u in inactive_saml]
64 | # Check if user id is in email list and if not then merge saml list user ids into email list
65 | for x in user_id_inactive_saml:
66 | if x not in user_id_inactive_email:
67 | user_id_inactive_email.append(x)
68 | return user_id_inactive_email
69 |
70 |
71 | user_list = fetch_old_users(n_days)
72 | print(user_list)
73 |
74 | # For testing purposes
75 | # ids 236,18,758
76 | # names - alejandro@looker.com, jon.bale@looker.com, jcarnes+profservices@google.com
77 | user_list = [236, 18, 758]
78 | new_owner_id = 11
79 |
80 | # Find all schedules of a particular user id
81 |
82 |
83 | def find_schedules(current_owner_id: int):
84 | result = {}
85 | scheduled_plans = sdk.all_scheduled_plans(user_id=current_owner_id)
86 | for i in scheduled_plans:
87 | result[i['name']] = i['id']
88 | return result
89 |
90 | # Transfer all schedules of a user to a new user.
91 |
92 |
93 | def update_owner(current_owner_id: int, new_owner_id: int):
94 | body = {}
95 | body['user_id'] = new_owner_id
96 | find = find_schedules(current_owner_id)
97 | for i in find.values():
98 | sdk.update_scheduled_plan(i, body)
99 | return body
100 |
101 | # Loop through list of users to find schedules and transfer schedules
102 |
103 |
104 | def transfer_schedule(user_list: list, create_user: False):
105 | # find old users
106 | old_user_list = fetch_old_users(n_days=1)
107 | # find their schedules
108 | for user in old_user_list:
109 | x = find_schedules(user)
110 | print(x)
111 | update_owner(user, 2)
112 | # find reassign them to meta user
113 | # do i need to create meta user
114 | # if create_user:
115 | # sd
116 | # update owner
117 | #
118 | # update_owner(u, new_owner_id)
119 |
120 | # return f'Schedule transferred for {user_list}'
121 |
122 |
123 | if __name__ == "__main__":
124 | reassign_schedules = transfer_schedule(user_list)
125 | print(reassign_schedules)
126 |
127 | # from lmanage.pprint import pprint
128 | # import snoop - then @snoop
129 |
--------------------------------------------------------------------------------
/tests/test_capture_looks.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from os import name
3 | import pytest
4 | from lmanage.capturator.content_capturation import look_capture as lc
5 | from tests import fake_methods_data
6 |
7 | input_data = fake_methods_data.input_data
8 |
9 |
10 | def test_get_all_looks_metadata(mocker):
11 | sdk = fake_methods_data.MockSDK()
12 | look_obj = lc.LookCapture(sdk=sdk, content_folders=123)
13 | all_look_data = fake_methods_data.MockAll_look_Response(
14 | id='123', folder='123')
15 | mocker.patch.object(sdk, "all_looks")
16 | sdk.all_looks.return_value = all_look_data
17 | response = [look_obj.get_all_looks_metadata()]
18 |
19 | assert isinstance(response, list)
20 | assert 'id' in response[0]
21 | assert 'folder' in response[0]
22 |
--------------------------------------------------------------------------------
/tests/test_create_folders.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from looker_sdk.sdk.api40 import models
3 | from os import name
4 | import pytest
5 | from lmanage.configurator.folder_configuration import create_folders as cf
6 | from tests import fake_methods_data
7 |
8 | input_data = fake_methods_data.input_data
9 | LOGGER = logging.getLogger(__name__)
10 |
11 |
12 | def test_create_folder(mocker):
13 | # Tests if there is a folder exists we raise an exception
14 | sdk = fake_methods_data.MockSDK()
15 | sf_data = fake_methods_data.MockCreateFolder(
16 | name='frankie_fish', parent_id='123')
17 | mocker.patch.object(sdk, "create_folder")
18 | sdk.create_folder.return_value = sf_data
19 |
20 | test = cf.CreateInstanceFolders(sdk=sdk, folder_metadata='5555').create_folder(
21 | folder_name='test', parent_id='123')
22 | assert isinstance(test, fake_methods_data.MockCreateFolder)
23 | assert test.name == 'frankie_fish'
24 | sdk.create_folder.assert_called_with(
25 | body=models.CreateFolder(name='test', parent_id='123'))
26 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/dashboards/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/tests/test_lookml_files/the_look/dashboards/.gitkeep
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/models/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/tests/test_lookml_files/the_look/models/.gitkeep
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/models_aws/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/tests/test_lookml_files/the_look/models_aws/.gitkeep
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/looker-open-source/lmanage/1ed47fc2d14c29c67cfcda54ea1ca2b81018e708/tests/test_lookml_files/the_look/views/.gitkeep
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/03_inventory_items.view.lkml:
--------------------------------------------------------------------------------
1 | view: inventory_items {
2 | sql_table_name: `looker-private-demo.ecomm.inventory_items` ;;
3 |
4 | ## DIMENSIONS ##
5 |
6 | dimension: id {
7 | primary_key: yes
8 | type: number
9 | sql: ${TABLE}.id ;;
10 | }
11 |
12 | dimension: cost {
13 | type: number
14 | value_format_name: usd
15 | sql: ${TABLE}.cost ;;
16 | }
17 |
18 | dimension_group: created {
19 | type: time
20 | timeframes: [time, date, week, month, raw]
21 | #sql: cast(CASE WHEN ${TABLE}.created_at = "\\N" THEN NULL ELSE ${TABLE}.created_at END as timestamp) ;;
22 | sql: CAST(${TABLE}.created_at AS TIMESTAMP) ;;
23 | }
24 |
25 | dimension: product_id {
26 | type: number
27 | hidden: yes
28 | sql: ${TABLE}.product_id ;;
29 | }
30 |
31 | dimension_group: sold {
32 | type: time
33 | timeframes: [time, date, week, month, raw]
34 | # sql: cast(CASE WHEN ${TABLE}.sold_at = "\\N" THEN NULL ELSE ${TABLE}.sold_at END as timestamp) ;;
35 | sql: ${TABLE}.sold_at ;;
36 | }
37 |
38 | dimension: is_sold {
39 | type: yesno
40 | sql: ${sold_raw} is not null ;;
41 | }
42 |
43 | dimension: days_in_inventory {
44 | description: "days between created and sold date"
45 | type: number
46 | sql: TIMESTAMP_DIFF(coalesce(${sold_raw}, CURRENT_TIMESTAMP()), ${created_raw}, DAY) ;;
47 | }
48 |
49 | dimension: days_in_inventory_tier {
50 | type: tier
51 | sql: ${days_in_inventory} ;;
52 | style: integer
53 | tiers: [0, 5, 10, 20, 40, 80, 160, 360]
54 | }
55 |
56 | dimension: days_since_arrival {
57 | description: "days since created - useful when filtering on sold yesno for items still in inventory"
58 | type: number
59 | sql: TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), ${created_raw}, DAY) ;;
60 | }
61 |
62 | dimension: days_since_arrival_tier {
63 | type: tier
64 | sql: ${days_since_arrival} ;;
65 | style: integer
66 | tiers: [0, 5, 10, 20, 40, 80, 160, 360]
67 | }
68 |
69 | dimension: product_distribution_center_id {
70 | hidden: yes
71 | sql: ${TABLE}.product_distribution_center_id ;;
72 | }
73 |
74 | ## MEASURES ##
75 |
76 | measure: sold_count {
77 | type: count
78 | drill_fields: [detail*]
79 |
80 | filters: {
81 | field: is_sold
82 | value: "Yes"
83 | }
84 | }
85 |
86 | measure: sold_percent {
87 | type: number
88 | value_format_name: percent_2
89 | sql: 1.0 * ${sold_count}/(CASE WHEN ${count} = 0 THEN NULL ELSE ${count} END) ;;
90 | }
91 |
92 | measure: total_cost {
93 | type: sum
94 | value_format_name: usd
95 | sql: ${cost} ;;
96 | }
97 |
98 | measure: average_cost {
99 | type: average
100 | value_format_name: usd
101 | sql: ${cost} ;;
102 | }
103 |
104 | measure: count {
105 | type: count
106 | drill_fields: [detail*]
107 | }
108 |
109 | measure: number_on_hand {
110 | type: count
111 | drill_fields: [detail*]
112 |
113 | filters: {
114 | field: is_sold
115 | value: "No"
116 | }
117 | }
118 |
119 | measure: stock_coverage_ratio {
120 | type: number
121 | description: "Stock on Hand vs Trailing 28d Sales Ratio"
122 | sql: 1.0 * ${number_on_hand} / nullif(${order_items.count_last_28d},0) ;;
123 | value_format_name: decimal_2
124 | html: {{ rendered_value }}
;;
125 | }
126 |
127 | set: detail {
128 | fields: [id, products.item_name, products.category, products.brand, products.department, cost, created_time, sold_time]
129 | }
130 | }
131 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/04_products.view.lkml:
--------------------------------------------------------------------------------
1 | view: products {
2 | sql_table_name: `looker-private-demo.ecomm.products` ;;
3 |
4 | ### DIMENSIONS ###
5 |
6 | dimension: id {
7 | primary_key: yes
8 | type: number
9 | sql: ${TABLE}.id ;;
10 | }
11 |
12 | dimension: category {
13 | sql: TRIM(${TABLE}.category) ;;
14 | drill_fields: [item_name]
15 | }
16 |
17 | dimension: item_name {
18 | sql: TRIM(${TABLE}.name) ;;
19 | }
20 |
21 | dimension: brand {
22 | sql: TRIM(${TABLE}.brand) ;;
23 | link: {
24 | label: "Website"
25 | url: "http://www.google.com/search?q={{ value | encode_uri }}+clothes&btnI"
26 | icon_url: "http://www.google.com/s2/favicons?domain=www.{{ value | encode_uri }}.com"
27 | }
28 | link: {
29 | label: "Facebook"
30 | url: "http://www.google.com/search?q=site:facebook.com+{{ value | encode_uri }}+clothes&btnI"
31 | icon_url: "https://upload.wikimedia.org/wikipedia/commons/c/c2/F_icon.svg"
32 | }
33 | link: {
34 | label: "{{value}} Analytics Dashboard"
35 | url: "/dashboards/CRMxoGiGJUv4eGALMHiAb0?Brand%20Name={{ value | encode_uri }}"
36 | icon_url: "http://www.looker.com/favicon.ico"
37 | }
38 | drill_fields: [category, item_name]
39 | }
40 |
41 | dimension: retail_price {
42 | type: number
43 | sql: ${TABLE}.retail_price ;;
44 | }
45 |
46 | dimension: department {
47 | sql: TRIM(${TABLE}.department) ;;
48 | }
49 |
50 | dimension: sku {
51 | sql: ${TABLE}.sku ;;
52 | }
53 |
54 | dimension: distribution_center_id {
55 | type: number
56 | sql: CAST(${TABLE}.distribution_center_id AS INT64) ;;
57 | }
58 |
59 | ## MEASURES ##
60 |
61 | measure: count {
62 | type: count
63 | drill_fields: [detail*]
64 | }
65 |
66 | measure: brand_count {
67 | type: count_distinct
68 | sql: ${brand} ;;
69 | drill_fields: [brand, detail2*, -brand_count] # show the brand, a bunch of counts (see the set below), don't show the brand count, because it will always be 1
70 | }
71 |
72 | measure: category_count {
73 | alias: [category.count]
74 | type: count_distinct
75 | sql: ${category} ;;
76 | drill_fields: [category, detail2*, -category_count] # don't show because it will always be 1
77 | }
78 |
79 | measure: department_count {
80 | alias: [department.count]
81 | type: count_distinct
82 | sql: ${department} ;;
83 | drill_fields: [department, detail2*, -department_count] # don't show because it will always be 1
84 | }
85 |
86 | set: detail {
87 | fields: [id, item_name, brand, category, department, retail_price, customers.count, orders.count, order_items.count, inventory_items.count]
88 | }
89 |
90 | set: detail2 {
91 | fields: [category_count, brand_count, department_count, count, customers.count, orders.count, order_items.count, inventory_items.count, products.count]
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/05_distribution_centers.view.lkml:
--------------------------------------------------------------------------------
1 | view: distribution_centers {
2 | sql_table_name: `looker-private-demo.ecomm.distribution_centers` ;;
3 | dimension: location {
4 | type: location
5 | sql_latitude: ${TABLE}.latitude ;;
6 | sql_longitude: ${TABLE}.longitude ;;
7 | }
8 |
9 | dimension: latitude {
10 | sql: ${TABLE}.latitude ;;
11 | hidden: yes
12 | }
13 |
14 | dimension: longitude {
15 | sql: ${TABLE}.longitude ;;
16 | hidden: yes
17 | }
18 |
19 | dimension: id {
20 | type: number
21 | primary_key: yes
22 | sql: ${TABLE}.id ;;
23 | }
24 |
25 | dimension: name {
26 | sql: ${TABLE}.name ;;
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/11_order_facts.view.lkml:
--------------------------------------------------------------------------------
1 | include: "/models/**/thelook.model.lkml"
2 | view: order_facts {
3 | derived_table: {
4 | explore_source: order_items {
5 | column: order_id {field: order_items.order_id_no_actions }
6 | column: items_in_order { field: order_items.count }
7 | column: order_amount { field: order_items.total_sale_price }
8 | column: order_cost { field: inventory_items.total_cost }
9 | column: user_id {field: order_items.user_id }
10 | column: created_at {field: order_items.created_raw}
11 | column: order_gross_margin {field: order_items.total_gross_margin}
12 | derived_column: order_sequence_number {
13 | sql: RANK() OVER (PARTITION BY user_id ORDER BY created_at) ;;
14 | }
15 | }
16 | datagroup_trigger: ecommerce_etl
17 | }
18 |
19 | dimension: order_id {
20 | type: number
21 | hidden: yes
22 | primary_key: yes
23 | sql: ${TABLE}.order_id ;;
24 | }
25 |
26 | dimension: items_in_order {
27 | type: number
28 | sql: ${TABLE}.items_in_order ;;
29 | }
30 |
31 | dimension: order_amount {
32 | type: number
33 | value_format_name: usd
34 | sql: ${TABLE}.order_amount ;;
35 | }
36 |
37 | dimension: order_cost {
38 | type: number
39 | value_format_name: usd
40 | sql: ${TABLE}.order_cost ;;
41 | }
42 |
43 | dimension: order_gross_margin {
44 | type: number
45 | value_format_name: usd
46 | }
47 |
48 | dimension: order_sequence_number {
49 | type: number
50 | sql: ${TABLE}.order_sequence_number ;;
51 | }
52 |
53 | dimension: is_first_purchase {
54 | type: yesno
55 | sql: ${order_sequence_number} = 1 ;;
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/12_user_order_facts.view.lkml:
--------------------------------------------------------------------------------
1 | view: user_order_facts {
2 | derived_table: {
3 | sql:
4 | SELECT
5 | user_id
6 | , COUNT(DISTINCT order_id) AS lifetime_orders
7 | , SUM(sale_price) AS lifetime_revenue
8 | , CAST(MIN(created_at) AS TIMESTAMP) AS first_order
9 | , CAST(MAX(created_at) AS TIMESTAMP) AS latest_order
10 | , COUNT(DISTINCT FORMAT_TIMESTAMP('%Y%m', created_at)) AS number_of_distinct_months_with_orders
11 | --, FIRST_VALUE(CONCAT(uniform(2, 9, random(1)),uniform(0, 9, random(2)),uniform(0, 9, random(3)),'-',uniform(0, 9, random(4)),uniform(0, 9, random(5)),uniform(0, 9, random(6)),'-',uniform(0, 9, random(7)),uniform(0, 9, random(8)),uniform(0, 9, random(9)),uniform(0, 9, random(10)))) OVER (PARTITION BY user_id ORDER BY user_id) AS phone_number
12 | FROM looker-private-demo.ecomm.order_items
13 | GROUP BY user_id
14 | ;;
15 | datagroup_trigger: ecommerce_etl
16 | }
17 |
18 | dimension: user_id {
19 | primary_key: yes
20 | hidden: yes
21 | sql: ${TABLE}.user_id ;;
22 | }
23 |
24 | # dimension: phone_number {
25 | # type: string
26 | # tags: ["phone"]
27 | # sql: ${TABLE}.phone_number ;;
28 | # }
29 |
30 |
31 | ##### Time and Cohort Fields ######
32 |
33 | dimension_group: first_order {
34 | type: time
35 | timeframes: [date, week, month, year]
36 | sql: ${TABLE}.first_order ;;
37 | }
38 |
39 | dimension_group: latest_order {
40 | type: time
41 | timeframes: [date, week, month, year]
42 | sql: ${TABLE}.latest_order ;;
43 | }
44 |
45 | dimension: days_as_customer {
46 | description: "Days between first and latest order"
47 | type: number
48 | sql: TIMESTAMP_DIFF(${TABLE}.latest_order, ${TABLE}.first_order, DAY)+1 ;;
49 | }
50 |
51 | dimension: days_as_customer_tiered {
52 | type: tier
53 | tiers: [0, 1, 7, 14, 21, 28, 30, 60, 90, 120]
54 | sql: ${days_as_customer} ;;
55 | style: integer
56 | }
57 |
58 | ##### Lifetime Behavior - Order Counts ######
59 |
60 | dimension: lifetime_orders {
61 | type: number
62 | sql: ${TABLE}.lifetime_orders ;;
63 | }
64 |
65 | dimension: repeat_customer {
66 | description: "Lifetime Count of Orders > 1"
67 | type: yesno
68 | sql: ${lifetime_orders} > 1 ;;
69 | }
70 |
71 | dimension: lifetime_orders_tier {
72 | type: tier
73 | tiers: [0, 1, 2, 3, 5, 10]
74 | sql: ${lifetime_orders} ;;
75 | style: integer
76 | }
77 |
78 | measure: average_lifetime_orders {
79 | type: average
80 | value_format_name: decimal_2
81 | sql: ${lifetime_orders} ;;
82 | }
83 |
84 | dimension: distinct_months_with_orders {
85 | type: number
86 | sql: ${TABLE}.number_of_distinct_months_with_orders ;;
87 | }
88 |
89 | ##### Lifetime Behavior - Revenue ######
90 |
91 | dimension: lifetime_revenue {
92 | type: number
93 | value_format_name: usd
94 | sql: ${TABLE}.lifetime_revenue ;;
95 | }
96 |
97 | dimension: lifetime_revenue_tier {
98 | type: tier
99 | tiers: [0, 25, 50, 100, 200, 500, 1000]
100 | sql: ${lifetime_revenue} ;;
101 | style: integer
102 | }
103 |
104 | measure: average_lifetime_revenue {
105 | type: average
106 | value_format_name: usd
107 | sql: ${lifetime_revenue} ;;
108 | }
109 | }
110 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/13_repeat_purchase_facts.view.lkml:
--------------------------------------------------------------------------------
1 | view: repeat_purchase_facts {
2 | derived_table: {
3 | datagroup_trigger: ecommerce_etl
4 | sql: SELECT
5 | order_items.order_id as order_id
6 | , order_items.created_at
7 | , COUNT(DISTINCT repeat_order_items.id) AS number_subsequent_orders
8 | , MIN(repeat_order_items.created_at) AS next_order_date
9 | , MIN(repeat_order_items.order_id) AS next_order_id
10 | FROM looker-private-demo.ecomm.order_items as order_items
11 | LEFT JOIN looker-private-demo.ecomm.order_items repeat_order_items
12 | ON order_items.user_id = repeat_order_items.user_id
13 | AND order_items.created_at < repeat_order_items.created_at
14 | GROUP BY 1, 2
15 | ;;
16 | }
17 |
18 | dimension: order_id {
19 | type: number
20 | hidden: yes
21 | primary_key: yes
22 | sql: ${TABLE}.order_id ;;
23 | }
24 |
25 | dimension: next_order_id {
26 | type: number
27 | hidden: yes
28 | sql: ${TABLE}.next_order_id ;;
29 | }
30 |
31 | dimension: has_subsequent_order {
32 | type: yesno
33 | sql: ${next_order_id} > 0 ;;
34 | }
35 |
36 | dimension: number_subsequent_orders {
37 | type: number
38 | sql: ${TABLE}.number_subsequent_orders ;;
39 | }
40 |
41 | dimension_group: next_order {
42 | type: time
43 | timeframes: [raw, date]
44 | hidden: yes
45 | sql: CAST(${TABLE}.next_order_date AS TIMESTAMP) ;;
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/25_trailing_sales_snapshot.view.lkml:
--------------------------------------------------------------------------------
1 | view: trailing_sales_snapshot {
2 | derived_table: {
3 | datagroup_trigger: ecommerce_etl
4 | sql: with calendar as
5 | (select distinct created_at as snapshot_date
6 | from looker-private-demo.ecomm.inventory_items
7 | -- where dateadd('day',90,created_at)>=current_date
8 | )
9 |
10 | select
11 | inventory_items.product_id
12 | ,date(order_items.created_at) as snapshot_date
13 | ,count(*) as trailing_28d_sales
14 | from looker-private-demo.ecomm.order_items
15 | join looker-private-demo.ecomm.inventory_items
16 | on order_items.inventory_item_id = inventory_items.id
17 | join calendar
18 | on date(order_items.created_at) <= date_add(calendar.snapshot_date, interval 28 day)
19 | and date(order_items.created_at) >= calendar.snapshot_date
20 | -- where dateadd('day',90,calendar.snapshot_date)>=current_date
21 | group by 1,2
22 | ;;
23 | }
24 |
25 | # measure: count {
26 | # type: count
27 | # drill_fields: [detail*]
28 | # }
29 |
30 | dimension: product_id {
31 | type: number
32 | sql: ${TABLE}.product_id ;;
33 | }
34 |
35 | dimension: snapshot_date {
36 | type: date
37 | sql: cast(${TABLE}.snapshot_date as timestamp) ;;
38 | }
39 |
40 | dimension: trailing_28d_sales {
41 | type: number
42 | hidden: yes
43 | sql: ${TABLE}.trailing_28d_sales ;;
44 | }
45 |
46 | measure: sum_trailing_28d_sales {
47 | type: sum
48 | sql: ${trailing_28d_sales} ;;
49 | }
50 |
51 | measure: sum_trailing_28d_sales_yesterday {
52 | type: sum
53 | hidden: yes
54 | sql: ${trailing_28d_sales} ;;
55 | filters: {
56 | field: snapshot_date
57 | value: "yesterday"
58 | }
59 | }
60 |
61 | measure: sum_trailing_28d_sales_last_wk {
62 | type: sum
63 | hidden: yes
64 | sql: ${trailing_28d_sales} ;;
65 | filters: {
66 | field: snapshot_date
67 | value: "8 days ago for 1 day"
68 | }
69 | }
70 |
71 | set: detail {
72 | fields: [product_id, snapshot_date, trailing_28d_sales]
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/51_events.view.lkml:
--------------------------------------------------------------------------------
1 | view: events {
2 | sql_table_name: `looker-private-demo.ecomm.events` ;;
3 |
4 | dimension: event_id {
5 | type: number
6 | primary_key: yes
7 | tags: ["mp_event_id"]
8 | sql: ${TABLE}.id ;;
9 | }
10 |
11 | dimension: session_id {
12 | type: number
13 | hidden: yes
14 | sql: ${TABLE}.session_id ;;
15 | }
16 |
17 | dimension: ip {
18 | label: "IP Address"
19 | view_label: "Visitors"
20 | sql: ${TABLE}.ip_address ;;
21 | }
22 |
23 | dimension: user_id {
24 | sql: ${TABLE}.user_id ;;
25 | }
26 |
27 | dimension_group: event {
28 | type: time
29 | # timeframes: [time, date, hour, time_of_day, hour_of_day, week, day_of_week_index, day_of_week]
30 | sql: ${TABLE}.created_at ;;
31 | }
32 |
33 | dimension: sequence_number {
34 | type: number
35 | description: "Within a given session, what order did the events take place in? 1=First, 2=Second, etc"
36 | sql: ${TABLE}.sequence_number ;;
37 | }
38 |
39 | dimension: is_entry_event {
40 | type: yesno
41 | description: "Yes indicates this was the entry point / landing page of the session"
42 | sql: ${sequence_number} = 1 ;;
43 | }
44 |
45 | dimension: is_exit_event {
46 | type: yesno
47 | label: "UTM Source"
48 | sql: ${sequence_number} = ${sessions.number_of_events_in_session} ;;
49 | description: "Yes indicates this was the exit point / bounce page of the session"
50 | }
51 |
52 | measure: count_bounces {
53 | type: count
54 | description: "Count of events where those events were the bounce page for the session"
55 |
56 | filters: {
57 | field: is_exit_event
58 | value: "Yes"
59 | }
60 | }
61 |
62 | measure: bounce_rate {
63 | type: number
64 | value_format_name: percent_2
65 | description: "Percent of events where those events were the bounce page for the session, out of all events"
66 | sql: ${count_bounces}*1.0 / nullif(${count}*1.0,0) ;;
67 | }
68 |
69 | dimension: full_page_url {
70 | sql: ${TABLE}.uri ;;
71 | }
72 |
73 | dimension: viewed_product_id {
74 | type: number
75 | sql: CASE WHEN ${event_type} = 'Product' THEN
76 | CAST(SPLIT(${full_page_url}, '/')[OFFSET(ARRAY_LENGTH(SPLIT(${full_page_url}, '/'))-1)] AS INT64)
77 | END
78 | ;;
79 | }
80 |
81 | dimension: event_type {
82 | sql: ${TABLE}.event_type ;;
83 | tags: ["mp_event_name"]
84 | }
85 |
86 | dimension: funnel_step {
87 | description: "Login -> Browse -> Add to Cart -> Checkout"
88 | sql: CASE
89 | WHEN ${event_type} IN ('Login', 'Home') THEN '(1) Land'
90 | WHEN ${event_type} IN ('Category', 'Brand') THEN '(2) Browse Inventory'
91 | WHEN ${event_type} = 'Product' THEN '(3) View Product'
92 | WHEN ${event_type} = 'Cart' THEN '(4) Add Item to Cart'
93 | WHEN ${event_type} = 'Purchase' THEN '(5) Purchase'
94 | END
95 | ;;
96 | }
97 |
98 | measure: unique_visitors {
99 | type: count_distinct
100 | description: "Uniqueness determined by IP Address and User Login"
101 | view_label: "Visitors"
102 | sql: ${ip} ;;
103 | drill_fields: [visitors*]
104 | }
105 |
106 | dimension: location {
107 | type: location
108 | view_label: "Visitors"
109 | sql_latitude: ${TABLE}.latitude ;;
110 | sql_longitude: ${TABLE}.longitude ;;
111 | }
112 |
113 | dimension: approx_location {
114 | type: location
115 | view_label: "Visitors"
116 | sql_latitude: round(${TABLE}.latitude,1) ;;
117 | sql_longitude: round(${TABLE}.longitude,1) ;;
118 | }
119 |
120 | dimension: has_user_id {
121 | type: yesno
122 | view_label: "Visitors"
123 | description: "Did the visitor sign in as a website user?"
124 | sql: ${users.id} > 0 ;;
125 | }
126 |
127 | dimension: browser {
128 | view_label: "Visitors"
129 | sql: ${TABLE}.browser ;;
130 | }
131 |
132 | dimension: os {
133 | label: "Operating System"
134 | view_label: "Visitors"
135 | sql: ${TABLE}.os ;;
136 | }
137 |
138 | measure: count {
139 | type: count
140 | drill_fields: [simple_page_info*]
141 | }
142 |
143 | measure: sessions_count {
144 | type: count_distinct
145 | sql: ${session_id} ;;
146 | }
147 |
148 | measure: count_m {
149 | label: "Count (MM)"
150 | type: number
151 | hidden: yes
152 | sql: ${count}/1000000.0 ;;
153 | drill_fields: [simple_page_info*]
154 | value_format: "#.### \"M\""
155 | }
156 |
157 | measure: unique_visitors_m {
158 | label: "Unique Visitors (MM)"
159 | view_label: "Visitors"
160 | type: number
161 | sql: count (distinct ${ip}) / 1000000.0 ;;
162 | description: "Uniqueness determined by IP Address and User Login"
163 | value_format: "#.### \"M\""
164 | hidden: yes
165 | drill_fields: [visitors*]
166 | }
167 |
168 | measure: unique_visitors_k {
169 | label: "Unique Visitors (k)"
170 | view_label: "Visitors"
171 | type: number
172 | hidden: yes
173 | description: "Uniqueness determined by IP Address and User Login"
174 | sql: count (distinct ${ip}) / 1000.0 ;;
175 | value_format: "#.### \"k\""
176 | drill_fields: [visitors*]
177 | }
178 |
179 | set: simple_page_info {
180 | fields: [event_id, event_time, event_type, full_page_url, user_id, funnel_step]
181 | }
182 |
183 | set: visitors {
184 | fields: [ip, os, browser, user_id, count]
185 | }
186 | }
187 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/explores.lkml:
--------------------------------------------------------------------------------
1 | explore: distribution_centers {
2 | join: products {
3 | type: left_outer
4 | sql_on: ${products.distribution_center_id} = ${distribution_centers.id} ;;
5 | relationship: many_to_one
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/tests/test_lookml_files/the_look/views/test_ndt.view.lkml:
--------------------------------------------------------------------------------
1 | view: test_ndt {
2 | derived_table: {
3 | explore_source: order_items {
4 | column: total_revenue {}
5 | column: created_month {}
6 | column: user_id {}
7 | }
8 | }
9 | dimension: total_revenue {
10 | type: number
11 | }
12 | dimension: created_month {
13 | type: date_month
14 | }
15 | dimension: user_id {
16 | type: number
17 | }
18 | }
19 |
20 |
--------------------------------------------------------------------------------