├── .dockerignore ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── documentation_template.md │ └── feature_request.md ├── pull_request_template.md └── workflows │ ├── on_release.yml │ ├── pre-commit.yml │ ├── sync_opal_plus.yml │ └── tests.yml ├── .gitignore ├── .gitmodules ├── .isort.cfg ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── app-tests ├── README.md ├── docker-compose-app-tests.yml └── run.sh ├── docker ├── Dockerfile ├── docker-compose-api-policy-source-example.yml ├── docker-compose-example-cedar.yml ├── docker-compose-example.yml ├── docker-compose-git-webhook.yml ├── docker-compose-scopes-example.yml ├── docker-compose-with-callbacks.yml ├── docker-compose-with-kafka-example.yml ├── docker-compose-with-oauth-initial.yml ├── docker-compose-with-rate-limiting.yml ├── docker-compose-with-security.yml ├── docker-compose-with-statistics.yml ├── docker_files │ ├── bundle_files │ │ ├── bundle.tar.gz │ │ └── bundle.tar.gz.bak │ ├── cedar_data │ │ └── data.json │ ├── nginx.conf │ └── policy_test │ │ └── authz.rego ├── run-example-with-scopes.sh └── run-example-with-security.sh ├── documentation ├── .gitignore ├── babel.config.js ├── docs │ ├── FAQ.mdx │ ├── fetch-providers.mdx │ ├── getting-started │ │ ├── configuration.mdx │ │ ├── intro.mdx │ │ ├── quickstart │ │ │ ├── docker-compose-config │ │ │ │ ├── opal-client.mdx │ │ │ │ ├── opal-server.mdx │ │ │ │ ├── overview.mdx │ │ │ │ └── postgres-database.mdx │ │ │ └── opal-playground │ │ │ │ ├── overview.mdx │ │ │ │ ├── publishing-data-update.mdx │ │ │ │ ├── run-server-and-client.mdx │ │ │ │ ├── send-queries-to-opa.mdx │ │ │ │ └── updating-the-policy.mdx │ │ ├── running-opal │ │ │ ├── as-python-package │ │ │ │ ├── opal-client-setup.mdx │ │ │ │ ├── opal-server-setup.mdx │ │ │ │ ├── overview.mdx │ │ │ │ ├── running-in-prod.mdx │ │ │ │ └── secure-mode-setup.mdx │ │ │ ├── config-variables.mdx │ │ │ ├── download-docker-images.mdx │ │ │ ├── overview.mdx │ │ │ ├── run-docker-containers.mdx │ │ │ ├── run-opal-client │ │ │ │ ├── data-topics.mdx │ │ │ │ ├── get-client-image.mdx │ │ │ │ ├── lets-run-the-client.mdx │ │ │ │ ├── obtain-jwt-token.mdx │ │ │ │ ├── opa-runner-parameters.mdx │ │ │ │ ├── server-uri.mdx │ │ │ │ └── standalone-opa-uri.mdx │ │ │ ├── run-opal-server │ │ │ │ ├── broadcast-interface.mdx │ │ │ │ ├── data-sources.mdx │ │ │ │ ├── get-server-image.mdx │ │ │ │ ├── policy-repo-location.mdx │ │ │ │ ├── policy-repo-syncing.mdx │ │ │ │ ├── putting-all-together.mdx │ │ │ │ └── security-parameters.mdx │ │ │ └── troubleshooting.mdx │ │ └── tldr.mdx │ ├── opal-plus │ │ ├── deploy.mdx │ │ ├── features.mdx │ │ ├── introduction.mdx │ │ └── troubleshooting.mdx │ ├── overview │ │ ├── _security.mdx │ │ ├── architecture.mdx │ │ ├── design.mdx │ │ ├── modules.mdx │ │ └── scopes.md │ ├── release-updates.mdx │ ├── tutorials │ │ ├── _configure_backbone_pubsub.mdx │ │ ├── cedar.mdx │ │ ├── configure_external_data_sources.mdx │ │ ├── configure_opal.mdx │ │ ├── healthcheck_policy_and_update_callbacks.mdx │ │ ├── helm-chart-for-kubernetes.mdx │ │ ├── install_as_python_packages.mdx │ │ ├── monitoring_opal.mdx │ │ ├── run_opal_with_kafka.mdx │ │ ├── run_opal_with_pulsar.mdx │ │ ├── track_a_git_repo.mdx │ │ ├── track_an_api_bundle_server.mdx │ │ ├── trigger_data_updates.mdx │ │ ├── use_self_signed_certificates.mdx │ │ └── write_your_own_fetch_provider.mdx │ └── welcome.mdx ├── docusaurus.config.js ├── package-lock.json ├── package.json ├── sidebars.js ├── src │ └── css │ │ ├── custom.scss │ │ └── prism-theme.js └── static │ ├── .nojekyll │ └── img │ ├── FAQ-1.png │ ├── favicon.ico │ ├── opal.png │ └── opal_plus.png ├── packages ├── __packaging__.py ├── opal-client │ ├── opal_client │ │ ├── __init__.py │ │ ├── callbacks │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── register.py │ │ │ └── reporter.py │ │ ├── cli.py │ │ ├── client.py │ │ ├── config.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── fetcher.py │ │ │ ├── rpc.py │ │ │ └── updater.py │ │ ├── engine │ │ │ ├── __init__.py │ │ │ ├── healthcheck │ │ │ │ ├── example-transaction.json │ │ │ │ └── opal.rego │ │ │ ├── logger.py │ │ │ ├── options.py │ │ │ └── runner.py │ │ ├── limiter.py │ │ ├── logger.py │ │ ├── main.py │ │ ├── policy │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── fetcher.py │ │ │ ├── options.py │ │ │ ├── topics.py │ │ │ └── updater.py │ │ ├── policy_store │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── base_policy_store_client.py │ │ │ ├── cedar_client.py │ │ │ ├── mock_policy_store_client.py │ │ │ ├── opa_client.py │ │ │ ├── policy_store_client_factory.py │ │ │ └── schemas.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── data_updater_test.py │ │ │ ├── engine_runner_test.py │ │ │ ├── opa_client_test.py │ │ │ └── server_to_client_intergation_test.py │ │ └── utils.py │ ├── requires.txt │ └── setup.py ├── opal-common │ ├── opal_common │ │ ├── __init__.py │ │ ├── async_utils.py │ │ ├── authentication │ │ │ ├── __init__.py │ │ │ ├── authz.py │ │ │ ├── casting.py │ │ │ ├── deps.py │ │ │ ├── signer.py │ │ │ ├── tests │ │ │ │ ├── __init__.py │ │ │ │ └── jwt_signer_test.py │ │ │ ├── types.py │ │ │ └── verifier.py │ │ ├── cli │ │ │ ├── __init__.py │ │ │ ├── commands.py │ │ │ ├── docs.py │ │ │ └── typer_app.py │ │ ├── confi │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ ├── confi.py │ │ │ └── types.py │ │ ├── config.py │ │ ├── corn_utils.py │ │ ├── emport.py │ │ ├── engine │ │ │ ├── __init__.py │ │ │ ├── parsing.py │ │ │ ├── paths.py │ │ │ ├── py.typed │ │ │ └── tests │ │ │ │ ├── fixtures │ │ │ │ ├── invalid-package.rego │ │ │ │ ├── jwt.rego │ │ │ │ ├── no-package.rego │ │ │ │ ├── play.rego │ │ │ │ └── rbac.rego │ │ │ │ ├── parsing_test.py │ │ │ │ └── paths_test.py │ │ ├── fetcher │ │ │ ├── __init__.py │ │ │ ├── engine │ │ │ │ ├── __init__.py │ │ │ │ ├── base_fetching_engine.py │ │ │ │ ├── core_callbacks.py │ │ │ │ ├── fetch_worker.py │ │ │ │ └── fetching_engine.py │ │ │ ├── events.py │ │ │ ├── fetch_provider.py │ │ │ ├── fetcher_register.py │ │ │ ├── logger.py │ │ │ ├── providers │ │ │ │ ├── __init__.py │ │ │ │ ├── fastapi_rpc_fetch_provider.py │ │ │ │ └── http_fetch_provider.py │ │ │ └── tests │ │ │ │ ├── __init__.py │ │ │ │ ├── failure_handler_test.py │ │ │ │ ├── http_fetch_test.py │ │ │ │ └── rpc_fetch_test.py │ │ ├── git_utils │ │ │ ├── __init__.py │ │ │ ├── branch_tracker.py │ │ │ ├── bundle_maker.py │ │ │ ├── bundle_utils.py │ │ │ ├── commit_viewer.py │ │ │ ├── diff_viewer.py │ │ │ ├── env.py │ │ │ ├── exceptions.py │ │ │ ├── repo_cloner.py │ │ │ ├── tar_file_to_local_git_extractor.py │ │ │ └── tests │ │ │ │ ├── branch_tracker_test.py │ │ │ │ ├── bundle_maker_test.py │ │ │ │ ├── commit_viewer_test.py │ │ │ │ ├── conftest.py │ │ │ │ ├── diff_viewer_test.py │ │ │ │ ├── repo_cloner_test.py │ │ │ │ └── repo_watcher_test.py │ │ ├── http_utils.py │ │ ├── logger.py │ │ ├── logging_utils │ │ │ ├── __init__.py │ │ │ ├── decorators.py │ │ │ ├── filter.py │ │ │ ├── formatter.py │ │ │ ├── intercept.py │ │ │ └── thirdparty.py │ │ ├── middleware.py │ │ ├── monitoring │ │ │ ├── __init__.py │ │ │ ├── apm.py │ │ │ └── metrics.py │ │ ├── paths.py │ │ ├── schemas │ │ │ ├── __init__.py │ │ │ ├── data.py │ │ │ ├── policy.py │ │ │ ├── policy_source.py │ │ │ ├── scopes.py │ │ │ ├── security.py │ │ │ ├── store.py │ │ │ └── webhook.py │ │ ├── security │ │ │ ├── __init__.py │ │ │ ├── sslcontext.py │ │ │ └── tarsafe.py │ │ ├── sources │ │ │ ├── __init__.py │ │ │ ├── api_policy_source.py │ │ │ ├── base_policy_source.py │ │ │ └── git_policy_source.py │ │ ├── synchronization │ │ │ ├── __init__.py │ │ │ ├── expiring_redis_lock.py │ │ │ ├── hierarchical_lock.py │ │ │ └── named_lock.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── hierarchical_lock_test.py │ │ │ ├── path_utils_test.py │ │ │ ├── test_config.py │ │ │ ├── test_utils.py │ │ │ └── url_utils_test.py │ │ ├── topics │ │ │ ├── __init__.py │ │ │ ├── listener.py │ │ │ ├── publisher.py │ │ │ └── utils.py │ │ ├── urls.py │ │ └── utils.py │ ├── requires.txt │ └── setup.py ├── opal-server │ ├── opal_server │ │ ├── __init__.py │ │ ├── cli.py │ │ ├── config.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── data_update_publisher.py │ │ │ └── tests │ │ │ │ └── test_data_update_publisher.py │ │ ├── git_fetcher.py │ │ ├── loadlimiting.py │ │ ├── main.py │ │ ├── policy │ │ │ ├── __init__.py │ │ │ ├── bundles │ │ │ │ ├── __init__.py │ │ │ │ └── api.py │ │ │ ├── watcher │ │ │ │ ├── __init__.py │ │ │ │ ├── callbacks.py │ │ │ │ ├── factory.py │ │ │ │ └── task.py │ │ │ └── webhook │ │ │ │ ├── __init__.py │ │ │ │ ├── api.py │ │ │ │ ├── deps.py │ │ │ │ └── listener.py │ │ ├── publisher.py │ │ ├── pubsub.py │ │ ├── redis_utils.py │ │ ├── scopes │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── loader.py │ │ │ ├── scope_repository.py │ │ │ ├── service.py │ │ │ └── task.py │ │ ├── security │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ └── jwks.py │ │ ├── server.py │ │ ├── statistics.py │ │ └── tests │ │ │ └── policy_repo_webhook_test.py │ ├── requires.txt │ └── setup.py └── requires.txt ├── pytest.ini ├── requirements.txt ├── scripts ├── gunicorn_conf.py ├── start.sh └── wait-for.sh └── semver2pypi.py /.dockerignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | documentation/ 3 | rego_clone/ 4 | .github/ 5 | docker/ 6 | Makefile 7 | Dockerfile 8 | .dockerignore 9 | .gitignore 10 | .pre-commit-config.yaml 11 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root=true 2 | 3 | [*] 4 | charset=utf-8 5 | end_of_line=lf 6 | insert_final_newline=false 7 | indent_style=space 8 | indent_size=2 9 | trim_trailing_whitespace=true 10 | 11 | [*.py] 12 | indent_size=4 13 | 14 | [Makefile] 15 | indent_size=2 16 | indent_style=tab 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Submit a bug report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Please include steps to reproduce the bug: i.e: run OPAL, submit a realtime update through the API, etc. 15 | 16 | Please also include: 17 | - Full logs of opal server and opal client 18 | - Your configuration for OPAL server and OPAL client 19 | - i.e: docker compose, Kubernetes YAMLs, environment variables, etc. 20 | - you should redact any secrets/tokens/api keys in your config 21 | 22 | **Expected behavior** 23 | A clear and concise description of what you expected to happen. 24 | 25 | **Screenshots** 26 | If applicable, add screenshots to help explain your problem. 27 | 28 | **OPAL version** 29 | - Version: [e.g. 0.1.18] 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation_template.md: -------------------------------------------------------------------------------- 1 | We welcome any suggestions to improve the documentation. 2 | 3 | Please let us know what is not clear / missing from the docs. 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Fixes Issue 4 | 5 | 6 | 7 | 8 | 9 | ## Changes proposed 10 | 11 | 12 | 13 | 14 | 20 | 21 | ## Check List (Check all the applicable boxes) 22 | 23 | - [x] I sign off on contributing this submission to open-source 24 | - [ ] My code follows the code style of this project. 25 | - [ ] My change requires changes to the documentation. 26 | - [ ] I have updated the documentation accordingly. 27 | - [ ] All new and existing tests passed. 28 | - [ ] This PR does not contain plagiarized content. 29 | - [ ] The title of my pull request is a short description of the requested changes. 30 | 31 | ## Screenshots 32 | 33 | 34 | 35 | ## Note to reviewers 36 | 37 | 38 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | name: pre-commit 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [master, main] 7 | 8 | jobs: 9 | pre-commit: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-python@v4 14 | with: 15 | python-version: 3.x 16 | - name: install pre-commit 17 | run: python -m pip install 'pre-commit<4' 18 | - name: show environment 19 | run: python -m pip freeze --local 20 | - uses: actions/cache@v4 21 | with: 22 | path: ~/.cache/pre-commit 23 | key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} 24 | - name: run pre-commit 25 | run: pre-commit run --show-diff-on-failure --color=always --all-files 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # editors 132 | .vscode/ 133 | .idea 134 | *.iml 135 | 136 | .DS_Store 137 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "cedar-agent"] 2 | path = cedar-agent 3 | url = https://github.com/permitio/cedar-agent.git 4 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | profile=black 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | - repo: https://github.com/psf/black 9 | rev: 23.1.0 10 | hooks: 11 | - id: black 12 | - repo: https://github.com/pycqa/isort 13 | rev: 5.12.0 14 | hooks: 15 | - id: isort 16 | - repo: https://github.com/codespell-project/codespell 17 | rev: v2.2.4 18 | hooks: 19 | - id: codespell 20 | args: [--skip, "*.json"] 21 | - repo: https://github.com/PyCQA/docformatter 22 | rev: v1.7.5 23 | hooks: 24 | - id: docformatter 25 | args: [--in-place] 26 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # OPAL Community Code of Conduct 2 | 3 | OPAL follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 4 | 5 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting 6 | the maintainers via . 7 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md LICENSE 2 | -------------------------------------------------------------------------------- /app-tests/README.md: -------------------------------------------------------------------------------- 1 | # OPAL Application Tests 2 | 3 | To fully test OPAL's core features as part of our CI flow, 4 | We're using a bash script and a docker-compose configuration that enables most of OPAL's important features. 5 | 6 | ## How To Run Locally 7 | 8 | ### Controlling the image tag 9 | 10 | By default, tests would run with the `latest` image tag (for both server & client). 11 | 12 | To configure another specific version: 13 | 14 | ```bash 15 | export OPAL_IMAGE_TAG=0.7.1 16 | ``` 17 | 18 | Or if you want to test locally built images 19 | ```bash 20 | make docker-build-next 21 | export OPAL_IMAGE_TAG=next 22 | ``` 23 | 24 | ### Using a policy repo 25 | 26 | To test opal's git tracking capabilities, `run.sh` uses a dedicated GitHub repo ([opal-tests-policy-repo](https://github.com/permitio/opal-tests-policy-repo)) in which it creates branches and pushes new commits. 27 | 28 | If you're not accessible to that repo (not in `Permit.io`), Please fork our public [opal-example-policy-repo](https://github.com/permitio/opal-example-policy-repo), and override the repo URL to be used: 29 | ```bash 30 | export OPAL_POLICY_REPO_URL=git@github.com:your-org/your-repo.git 31 | ``` 32 | 33 | As `run.sh` requires push permissions, and as `opal-server` itself might need to authenticate GitHub (if your repo is private). If your GitHub ssh private key is not stored at `~/.ssh/id_rsa`, provide it using: 34 | ```bash 35 | # Use an absolute path 36 | export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./your_github_ssh_private_key) 37 | ``` 38 | 39 | 40 | ### Putting it all together 41 | 42 | ```bash 43 | make docker-build-next # To locally build opal images 44 | export OPAL_IMAGE_TAG=next # Otherwise would default to "latest" 45 | 46 | export OPAL_POLICY_REPO_URL=git@github.com:your-org/your-repo.git # To use your own repo for testing (if you're not an Permit.io employee yet...) 47 | export OPAL_POLICY_REPO_SSH_KEY_PATH=$(realpath ./your_github_ssh_private_key) # If your GitHub ssh key isn't in "~.ssh/id_rsa" 48 | 49 | cd app-tests 50 | ./run.sh 51 | ``` 52 | -------------------------------------------------------------------------------- /app-tests/docker-compose-app-tests.yml: -------------------------------------------------------------------------------- 1 | services: 2 | broadcast_channel: 3 | image: postgres:alpine 4 | environment: 5 | - POSTGRES_DB=postgres 6 | - POSTGRES_USER=postgres 7 | - POSTGRES_PASSWORD=postgres 8 | 9 | opal_server: 10 | image: permitio/opal-server:${OPAL_IMAGE_TAG:-latest} 11 | deploy: 12 | mode: replicated 13 | replicas: 2 14 | endpoint_mode: vip 15 | environment: 16 | - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres 17 | - UVICORN_NUM_WORKERS=4 18 | - OPAL_POLICY_REPO_URL=${OPAL_POLICY_REPO_URL:-git@github.com:permitio/opal-tests-policy-repo.git} 19 | - OPAL_POLICY_REPO_MAIN_BRANCH=${POLICY_REPO_BRANCH} 20 | - OPAL_POLICY_REPO_SSH_KEY=${OPAL_POLICY_REPO_SSH_KEY} 21 | - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}} 22 | - OPAL_LOG_FORMAT_INCLUDE_PID=true 23 | - OPAL_POLICY_REPO_WEBHOOK_SECRET=xxxxx 24 | - OPAL_POLICY_REPO_WEBHOOK_PARAMS={"secret_header_name":"x-webhook-token","secret_type":"token","secret_parsing_regex":"(.*)","event_request_key":"gitEvent","push_event_value":"git.push"} 25 | - OPAL_AUTH_PUBLIC_KEY=${OPAL_AUTH_PUBLIC_KEY} 26 | - OPAL_AUTH_PRIVATE_KEY=${OPAL_AUTH_PRIVATE_KEY} 27 | - OPAL_AUTH_PRIVATE_KEY_PASSPHRASE=${OPAL_AUTH_PRIVATE_KEY_PASSPHRASE} 28 | - OPAL_AUTH_MASTER_TOKEN=${OPAL_AUTH_MASTER_TOKEN} 29 | - OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/ 30 | - OPAL_AUTH_JWT_ISSUER=https://opal.ac/ 31 | - OPAL_STATISTICS_ENABLED=true 32 | ports: 33 | - "7002-7003:7002" 34 | depends_on: 35 | - broadcast_channel 36 | 37 | opal_client: 38 | image: permitio/opal-client:${OPAL_IMAGE_TAG:-latest} 39 | deploy: 40 | mode: replicated 41 | replicas: 2 42 | endpoint_mode: vip 43 | environment: 44 | - OPAL_SERVER_URL=http://opal_server:7002 45 | - OPAL_LOG_FORMAT_INCLUDE_PID=true 46 | - OPAL_INLINE_OPA_LOG_FORMAT=http 47 | - OPAL_SHOULD_REPORT_ON_DATA_UPDATES=True 48 | - OPAL_DEFAULT_UPDATE_CALLBACKS={"callbacks":[["http://opal_server:7002/data/callback_report",{"method":"post","process_data":false,"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}","content-type":"application/json"}}]]} 49 | - OPAL_OPA_HEALTH_CHECK_POLICY_ENABLED=True 50 | - OPAL_CLIENT_TOKEN=${OPAL_CLIENT_TOKEN} 51 | - OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/ 52 | - OPAL_AUTH_JWT_ISSUER=https://opal.ac/ 53 | - OPAL_STATISTICS_ENABLED=true 54 | ports: 55 | - "7766-7767:7000" 56 | - "8181-8182:8181" 57 | depends_on: 58 | - opal_server 59 | command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" 60 | -------------------------------------------------------------------------------- /docker/docker-compose-scopes-example.yml: -------------------------------------------------------------------------------- 1 | name: opal-scopes-example 2 | 3 | services: 4 | redis: 5 | image: redis 6 | ports: 7 | - "6379:6379" 8 | 9 | opal_server: 10 | # by default we run opal-server from latest official image 11 | image: permitio/opal-server:latest 12 | environment: 13 | # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) 14 | - OPAL_BROADCAST_URI=redis://redis:6379 15 | # number of uvicorn workers to run inside the opal-server container 16 | - UVICORN_NUM_WORKERS=4 17 | - OPAL_LOG_FORMAT_INCLUDE_PID=true 18 | # using scopes requires having a running redis instance 19 | - OPAL_REDIS_URL=redis://redis:6379 20 | - OPAL_SCOPES=1 21 | - OPAL_POLICY_REFRESH_INTERVAL=30 22 | - OPAL_BASE_DIR=/opal 23 | ports: 24 | # exposes opal server on the host machine, you can access the server at: http://localhost:7002 25 | - "7002:7002" 26 | depends_on: 27 | - redis 28 | 29 | my_opal_client: 30 | # by default we run opal-client from latest official image 31 | image: permitio/opal-client:latest 32 | environment: 33 | - OPAL_SERVER_URL=http://opal_server:7002 34 | - OPAL_LOG_FORMAT_INCLUDE_PID=true 35 | - OPAL_INLINE_OPA_LOG_FORMAT=http 36 | - OPAL_SCOPE_ID=myscope 37 | ports: 38 | # exposes opal client on the host machine, you can access the client at: http://localhost:7766 39 | - "7766:7000" 40 | # exposes the OPA agent (being run by OPAL) on the host machine 41 | # you can access the OPA api that you know and love at: http://localhost:8181 42 | # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ 43 | - "8181:8181" 44 | depends_on: 45 | - opal_server 46 | # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments 47 | # to make sure that opal-server is already up before starting the client. 48 | command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" 49 | 50 | her_opal_client: 51 | image: permitio/opal-client:latest 52 | environment: 53 | - OPAL_SERVER_URL=http://opal_server:7002 54 | - OPAL_LOG_FORMAT_INCLUDE_PID=true 55 | - OPAL_INLINE_OPA_LOG_FORMAT=http 56 | - OPAL_SCOPE_ID=herscope 57 | ports: 58 | - "7767:7000" 59 | - "8182:8181" 60 | depends_on: 61 | - opal_server 62 | command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" 63 | -------------------------------------------------------------------------------- /docker/docker_files/bundle_files/bundle.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/docker/docker_files/bundle_files/bundle.tar.gz -------------------------------------------------------------------------------- /docker/docker_files/bundle_files/bundle.tar.gz.bak: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/docker/docker_files/bundle_files/bundle.tar.gz.bak -------------------------------------------------------------------------------- /docker/docker_files/cedar_data/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "attrs": { 4 | "confidenceScore": { 5 | "__extn": { 6 | "arg": "33.57", 7 | "fn": "decimal" 8 | } 9 | }, 10 | "department": "HardwareEngineering", 11 | "homeIp": { 12 | "__extn": { 13 | "arg": "222.222.222.7", 14 | "fn": "ip" 15 | } 16 | }, 17 | "jobLevel": 5 18 | }, 19 | "parents": [ 20 | { 21 | "id": "Editor", 22 | "type": "Role" 23 | } 24 | ], 25 | "uid": { 26 | "id": "someone@permit.io", 27 | "type": "User" 28 | } 29 | }, 30 | { 31 | "attrs": {}, 32 | "parents": [ 33 | { 34 | "id": "document", 35 | "type": "ResourceType" 36 | }, 37 | { 38 | "id": "Editor", 39 | "type": "Role" 40 | } 41 | ], 42 | "uid": { 43 | "id": "document:delete", 44 | "type": "Action" 45 | } 46 | }, 47 | { 48 | "attrs": {}, 49 | "parents": [ 50 | { 51 | "id": "document", 52 | "type": "ResourceType" 53 | }, 54 | { 55 | "id": "Editor", 56 | "type": "Role" 57 | } 58 | ], 59 | "uid": { 60 | "id": "document:create", 61 | "type": "Action" 62 | } 63 | }, 64 | { 65 | "attrs": {}, 66 | "parents": [], 67 | "uid": { 68 | "id": "document", 69 | "type": "ResourceType" 70 | } 71 | }, 72 | { 73 | "attrs": {}, 74 | "parents": [ 75 | { 76 | "id": "Editor", 77 | "type": "Role" 78 | }, 79 | { 80 | "id": "document", 81 | "type": "ResourceType" 82 | } 83 | ], 84 | "uid": { 85 | "id": "document:update", 86 | "type": "Action" 87 | } 88 | }, 89 | { 90 | "attrs": {}, 91 | "parents": [ 92 | { 93 | "id": "document", 94 | "type": "ResourceType" 95 | }, 96 | { 97 | "id": "Editor", 98 | "type": "Role" 99 | } 100 | ], 101 | "uid": { 102 | "id": "document:list", 103 | "type": "Action" 104 | } 105 | }, 106 | { 107 | "attrs": {}, 108 | "parents": [ 109 | { 110 | "id": "document", 111 | "type": "ResourceType" 112 | }, 113 | { 114 | "id": "Editor", 115 | "type": "Role" 116 | } 117 | ], 118 | "uid": { 119 | "id": "document:get", 120 | "type": "Action" 121 | } 122 | }, 123 | { 124 | "attrs": {}, 125 | "parents": [], 126 | "uid": { 127 | "id": "Editor", 128 | "type": "Role" 129 | } 130 | } 131 | ] 132 | -------------------------------------------------------------------------------- /docker/docker_files/nginx.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile off; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | -------------------------------------------------------------------------------- /docker/docker_files/policy_test/authz.rego: -------------------------------------------------------------------------------- 1 | package system.authz 2 | 3 | default allow := true 4 | -------------------------------------------------------------------------------- /docker/run-example-with-scopes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [ ! -f "docker-compose-scopes-example.yml" ]; then 6 | echo "did not find compose file - run this script from the 'docker/' directory under opal root!" 7 | exit 8 | fi 9 | 10 | echo "------------------------------------------------------------------" 11 | echo "This script will run the docker-compose-scopes-example.yml example" 12 | echo "configuration, and demonstrates how to correctly create scopes in OPAL server" 13 | echo "------------------------------------------------------------------" 14 | 15 | echo "Run OPAL server with scopes in the background" 16 | docker compose -f docker-compose-scopes-example.yml up -d opal_server --remove-orphans 17 | 18 | sleep 2 19 | 20 | echo "Create scope 'myscope'" 21 | curl --request PUT 'http://localhost:7002/scopes' --header 'Content-Type: application/json' --data-raw '{ 22 | "scope_id": "myscope", 23 | "policy": { 24 | "source_type": "git", 25 | "url": "https://github.com/permitio/opal-example-policy-repo", 26 | "auth": { 27 | "auth_type": "none" 28 | }, 29 | "extensions": [ 30 | ".rego", 31 | ".json" 32 | ], 33 | "manifest": ".manifest", 34 | "poll_updates": "true", 35 | "branch": "master" 36 | }, 37 | "data": { 38 | "entries": [] 39 | } 40 | }' 41 | 42 | echo "Create scope 'herscope'" 43 | curl --request PUT 'http://localhost:7002/scopes' --header 'Content-Type: application/json' --data-raw '{ 44 | "scope_id": "herscope", 45 | "policy": { 46 | "source_type": "git", 47 | "url": "https://github.com/permitio/opal-example-policy-repo", 48 | "auth": { 49 | "auth_type": "none" 50 | }, 51 | "extensions": [ 52 | ".rego", 53 | ".json" 54 | ], 55 | "manifest": ".manifest", 56 | "poll_updates": true, 57 | "branch": "master" 58 | }, 59 | "data": { 60 | "entries": [] 61 | } 62 | }' 63 | 64 | echo "Bring up OPAL clients to use newly created scopes" 65 | docker compose -f docker-compose-scopes-example.yml up 66 | -------------------------------------------------------------------------------- /docker/run-example-with-security.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Runs the docker-compose-with-security.yml example with 4 | # crypto keys configured via environment variables 5 | # 6 | # Usage: 7 | # 8 | # $ ./run-example-with-security.sh 9 | # 10 | 11 | set -e 12 | 13 | if [ ! -f "docker-compose-with-security.yml" ]; then 14 | echo "did not find compose file - run this script from the 'docker/' directory under opal root!" 15 | exit 16 | fi 17 | 18 | echo "------------------------------------------------------------------" 19 | echo "This script will run the docker-compose-with-security.yml example" 20 | echo "configuration, and demonstrates how to correctly generate crypto" 21 | echo "keys and run OPAL in *secure mode*." 22 | echo "------------------------------------------------------------------" 23 | 24 | echo "generating opal crypto keys..." 25 | export OPAL_AUTH_PRIVATE_KEY_PASSPHRASE="123456" 26 | ssh-keygen -q -t rsa -b 4096 -m pem -f opal_crypto_key -N "$OPAL_AUTH_PRIVATE_KEY_PASSPHRASE" 27 | 28 | echo "saving crypto keys to env vars and removing temp key files..." 29 | export OPAL_AUTH_PUBLIC_KEY=`cat opal_crypto_key.pub` 30 | export OPAL_AUTH_PRIVATE_KEY=`cat opal_crypto_key | tr '\n' '_'` 31 | rm opal_crypto_key.pub opal_crypto_key 32 | 33 | echo "generating master token..." 34 | export OPAL_AUTH_MASTER_TOKEN=`openssl rand -hex 16` 35 | 36 | if ! command -v opal-server &> /dev/null 37 | then 38 | echo "opal-server cli was not found, run: 'pip install opal-server'" 39 | exit 40 | fi 41 | 42 | if ! command -v opal-client &> /dev/null 43 | then 44 | echo "opal-client cli was not found, run: 'pip install opal-client'" 45 | exit 46 | fi 47 | 48 | echo "running OPAL server so we can sign on JWT tokens..." 49 | OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/ OPAL_AUTH_JWT_ISSUER=https://opal.ac/ OPAL_REPO_WATCHER_ENABLED=0 opal-server run & 50 | 51 | sleep 2; 52 | 53 | echo "obtaining client JWT token..." 54 | export OPAL_CLIENT_TOKEN=`opal-client obtain-token $OPAL_AUTH_MASTER_TOKEN --type client` 55 | 56 | echo "killing opal server..." 57 | ps -ef | grep opal | grep -v grep | awk '{print $2}' | xargs kill 58 | 59 | sleep 5; 60 | 61 | echo "Saving your config to .env file..." 62 | rm -f .env 63 | echo "OPAL_AUTH_PUBLIC_KEY=\"$OPAL_AUTH_PUBLIC_KEY\"" >> .env 64 | echo "OPAL_AUTH_PRIVATE_KEY=\"$OPAL_AUTH_PRIVATE_KEY\"" >> .env 65 | echo "OPAL_AUTH_MASTER_TOKEN=\"$OPAL_AUTH_MASTER_TOKEN\"" >> .env 66 | echo "OPAL_CLIENT_TOKEN=\"$OPAL_CLIENT_TOKEN\"" >> .env 67 | echo "OPAL_AUTH_PRIVATE_KEY_PASSPHRASE=\"$OPAL_AUTH_PRIVATE_KEY_PASSPHRASE\"" >> .env 68 | 69 | echo "--------" 70 | echo "ready to run..." 71 | echo "--------" 72 | 73 | docker compose -f docker-compose-with-security.yml --env-file .env up --force-recreate 74 | -------------------------------------------------------------------------------- /documentation/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Production 5 | /build 6 | 7 | # Generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # Misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | -------------------------------------------------------------------------------- /documentation/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | }; 4 | -------------------------------------------------------------------------------- /documentation/docs/fetch-providers.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Available Fetch Providers 3 | --- 4 | 5 | | Type of OPAL Fetcher | Link to GitHub Project | Authors | Company | 6 | | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- |----------------------------------------| 7 | | **Postgres** | [View here](https://github.com/permitio/opal-fetcher-postgres) | [`@asafc`](https://github.com/asafc) | [Permit.io](https://permit.io/) | 8 | | **LDAP** | [View here](https://github.com/phi1010/opal-fetcher-ldap) | [`@phi1010`](https://github.com/phi1010) | | 9 | | **CosmosDB** | [View here](https://github.com/avo-sepp/opal-fetcher-cosmos) | [`@avo-sepp`](https://github.com/avo-sepp) | | 10 | | **FastAPI RPC** | [View here](https://github.com/permitio/opal/blob/master/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py) | [`@orweis`](https://github.com/orweis) | [Permit.io](https://permit.io/) | 11 | | **HTTP** | [View here](https://github.com/permitio/opal/blob/master/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py) | [`@orweis`](https://github.com/orweis) | [Permit.io](https://permit.io/) | 12 | | **MongoDB** | [View here](https://github.com/treedomtrees/opal-fetcher-mongodb) | [`@OancaAndrei`](https://github.com/OancaAndrei) | [Treedom](https://www.treedom.net/en) | 13 | | **MySQL** | [View here](https://github.com/bhimeshagrawal/opal-fetcher-mysql) | [`@bhimeshagrawal`](https://github.com/bhimeshagrawal)| | 14 | 15 | :::tip 16 | **We are always looking for new custom OPAL fetchers providers to be created**. We love the help of our community, so if you 17 | do create a fetcher, please **[reach out to us on Slack](https://bit.ly/permit-slack)**, and we will send you some company SWAG :) 18 | ::: 19 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/intro.mdx: -------------------------------------------------------------------------------- 1 | # Introduction to OPAL 2 | 3 | ## What is OPAL? 4 | 5 | Modern applications are complex, distributed, multi-tenant and run at scale - often creating overwhelming authorization challenges. 6 | 7 | OPA (Open Policy Agent) brings the power of decoupled policy to the infrastructure layer (especially K8s), and light applications. 8 | 9 | OPAL supercharges OPA to meet the pace of live applications, where the state relevant to authorization decisions may change with every user click and API call. 10 | 11 | - OPAL builds on top of OPA adding realtime updates (via Websocket Pub/Sub) for both policy and data. 12 | 13 | - OPAL embraces decoupling of policy and code, and doubles down on decoupling policy (git driven) and data (distributed data-source fetching engines). 14 | 15 | ### Why use OPAL 16 | 17 | - OPAL is the easiest way to keep your solution's authorization layer up-to-date in realtime. 18 | - OPAL aggregates policy and data from across the field and integrates them seamlessly into the authorization layer. 19 | - OPAL is microservices and cloud-native (see [Key concepts and design](../overview/design)) 20 | 21 | ### Why OPA + OPAL == 💜 22 | 23 | OPA (Open Policy Agent) is great! It decouples policy from code in a highly-performant and elegant way. But the challenge of keeping policy agents up-to-date is hard - especially in applications - where each user interaction or API call may affect access-control decisions. 24 | OPAL runs in the background, supercharging policy-agents, keeping them in sync with events in realtime. 25 | 26 | ## AWS Cedar + OPAL == 💪 27 | 28 | Cedar is a very powerful policy language, which powers AWS' AVP (Amazon Verified Permissions) - but what if you want to enjoy the power of Cedar on another cloud, locally, or on premise? 29 | This is where [Cedar-Agent](https://github.com/permitio/cedar-agent) and OPAL come in. 30 | 31 | ### What OPAL _is not_ 32 | 33 | #### OPAL is not a Policy Engine: 34 | 35 | - OPAL uses policy-engines, but isn't one itself - 36 | - Check out Cedar-Agent, Open-Policy-Agent, and OSO 37 | 38 | #### OPAL is not a database for permission data 39 | 40 | - Check out Google-Zanzibar 41 | 42 | #### Fullstack permissions: 43 | 44 | - OPAL + a policy-agent essentially provide microservices for authorization 45 | - Developers still need to add control interfaces on top (e.g. user-management, api-key-management, audit, impersonation, invites) both as APIs and UIs 46 | - Check out Permit.io 47 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/quickstart/docker-compose-config/opal-client.mdx: -------------------------------------------------------------------------------- 1 | # OPAL Client 2 | 3 | The OPAL Client has three main functionalities that need to be highlighted. 4 | 5 | ```yml showLineNumbers 6 | service: 7 | opal_client: 8 | image: permitio/opal-client:latest 9 | environment: 10 | - OPAL_SERVER_URL=http://opal_server:7002 11 | - OPAL_LOG_FORMAT_INCLUDE_PID=true 12 | - OPAL_INLINE_OPA_LOG_FORMAT=http 13 | ports: 14 | - "7766:7000" 15 | - "8181:8181" 16 | depends_on: 17 | - opal_server 18 | command: sh -c "./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" 19 | ``` 20 | 21 | ### 1. The OPAL client can run OPA for you as an inline process 22 | 23 | The OPAL client [docker image](https://hub.docker.com/r/permitio/opal-client) contains a built-in OPA agent, 24 | and can serve as fully-functional **authorization microservice**. OPA is solely responsible for **enforcing** and 25 | **evaluating authorization queries**. 26 | 27 | :::tip FACT 28 | **OPAL** is solely responsible for state-management, meaning it will keep the **policy** and **data** needed to evaluate queries 29 | **up-to-date**. 30 | ::: 31 | 32 | In our example `docker-compose.yml` file, OPA is enabled and runs on port `:8181` which is exposed on the host machine. 33 | 34 | ```yml showLineNumbers {3} 35 | ports: 36 | - "7766:7000" 37 | - "8181:8181" 38 | ``` 39 | 40 | **OPAL will manage the OPA process**. If the OPA process fails for some reason, OPAL will restart OPA and 41 | rehydrate the OPA cache with valid and up-to-date state. By rehydration, we mean that the policies and data 42 | will be **re-downloaded**. 43 | 44 | ### 2. The OPAL client syncs OPA with latest policy code 45 | 46 | OPAL **listens** to policy code update notifications and **downloads up-to-date policy bundles** from the server. 47 | 48 | ### 3. The OPAL client syncs OPA with latest policy data 49 | 50 | OPAL **listens** to policy data update notifications and **fetches the data from the sources** specified by the instructions 51 | sent from the server. OPAL can aggregate data from multiple sources. This may include your **APIs**, **databases** and **3rd party SaaS**. 52 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx: -------------------------------------------------------------------------------- 1 | # Understanding the Docker Compose Example Configuration 2 | 3 | The example file we will be referring to in this guide can be seen **[here](https://github.com/permitio/opal/blob/master/docker/docker-compose-example.yml)**. 4 | 5 | This example is running three containers that we have mentioned at the beginning of this guide. 6 | 7 | 1. **Broadcast Channel** 8 | 2. **OPAL Server** 9 | 3. **OPAL CLient** 10 | 11 | Here is an overview of the whole `docker-compose.yml` file, but don't worry, we will be referring to each section separately. 12 | 13 | ```yml showLineNumbers 14 | services: 15 | broadcast_channel: 16 | image: postgres:alpine 17 | environment: 18 | - POSTGRES_DB=postgres 19 | - POSTGRES_USER=postgres 20 | - POSTGRES_PASSWORD=postgres 21 | opal_server: 22 | image: permitio/opal-server:latest 23 | environment: 24 | - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres 25 | - UVICORN_NUM_WORKERS=4 26 | - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo 27 | - OPAL_POLICY_REPO_POLLING_INTERVAL=30 28 | - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} 29 | - OPAL_LOG_FORMAT_INCLUDE_PID=true 30 | ports: 31 | - "7002:7002" 32 | depends_on: 33 | - broadcast_channel 34 | opal_client: 35 | image: permitio/opal-client:latest 36 | environment: 37 | - OPAL_SERVER_URL=http://opal_server:7002 38 | - OPAL_LOG_FORMAT_INCLUDE_PID=true 39 | - OPAL_INLINE_OPA_LOG_FORMAT=http 40 | ports: 41 | - "7766:7000" 42 | - "8181:8181" 43 | depends_on: 44 | - opal_server 45 | command: sh -c "./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" 46 | ``` 47 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/quickstart/docker-compose-config/postgres-database.mdx: -------------------------------------------------------------------------------- 1 | # Postgres Database acting as a Broadcast Channel 2 | 3 | One of the containers that is handled inside the `docker-compose.yml` config is the broadcast channel. 4 | 5 | When **scaling** the OPAL Server to **multiple nodes and/or multiple workers**, we use a broadcast channel to **sync 6 | between all the instances** of the OPAL Server. 7 | 8 | ```yml showLineNumbers 9 | services: 10 | broadcast_channel: 11 | image: postgres:alpine 12 | environment: 13 | - POSTGRES_DB=postgres 14 | - POSTGRES_USER=postgres 15 | - POSTGRES_PASSWORD=postgres 16 | ``` 17 | 18 | With this configuration, you can specify **what channel we want to subscribe too**, and in this case, it's a 19 | **PostgreSQL Database**. 20 | 21 | :::tip 22 | If you run only a **single worker** it is **not necessary to deploy a broadcast backend**. 23 | 24 | **We do not recommend running a single worker in production.** 25 | ::: 26 | 27 | These are the currently three supported [broadcast backends](https://github.com/encode/broadcaster#available-backends): 28 | 29 | 1. PostgreSQL 30 | 2. Redis 31 | 3. Kafka 32 | 33 | The format of the broadcaster URI string `OPAL_BROADCAST_URI` is specified below for **Postgres**. A similar pattern will apply for 34 | **Redis** and **Kafka**. 35 | 36 | ```yml 37 | environment: 38 | - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres 39 | ``` 40 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/quickstart/opal-playground/overview.mdx: -------------------------------------------------------------------------------- 1 | # OPAL Playground 2 | 3 | This tutorial will show you what you can do with **OPAL**, and teach you about **OPAL core features**. 4 | 5 | We built an example configuration that you can run in **docker compose**. The example was built specifically for 6 | you to **explore OPAL quickly**, understand the core features and see what OPAL can do for you. 7 | 8 | You can get a running OPAL environment by running one `docker compose` command. 9 | 10 | Let's take OPAL for a swing! 11 | 12 | 20 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/quickstart/opal-playground/run-server-and-client.mdx: -------------------------------------------------------------------------------- 1 | # Step 1: run docker compose to start the opal server and client 2 | 3 | Download and run a working configuration of OPAL server and OPAL client on your machine: 4 | 5 | ```bash 6 | curl -L https://raw.githubusercontent.com/permitio/opal/master/docker/docker-compose-example.yml \ 7 | > docker-compose.yml && docker-compose up 8 | ``` 9 | 10 | You can alternatively **clone the OPAL repository** and run the example compose file from your local clone: 11 | 12 | ``` 13 | git clone https://github.com/permitio/opal.git 14 | 15 | cd opal 16 | 17 | docker compose -f docker/docker-compose-example.yml up 18 | ``` 19 | 20 | The `docker-compose.yml` we just downloaded - [view the file here](https://github.com/permitio/opal/blob/master/docker/docker-compose-example.yml) - is **running 3 containers**: 21 | 22 | 1. A **Broadcast Channel** Container 23 | 2. An **OPAL Server** Container 24 | 3. An **OPAL Client** Container 25 | 26 | We provide a detailed review of exactly **what is running and why** later in this tutorial. 27 | You can [jump there by clicking this link](#compose-recap) to gain a deeper understanding, and then come back here, 28 | or you can continue with the hands-on tutorial. 29 | 30 | **OPAL** (and also **OPA**) are now running on your machine. You should be aware of the following ports that are exposed on `localhost`: 31 | 32 | - **OPAL Server** - PORT `:7002` - the **_OPAL client_** (and potentially the CLI) can connect to this port. 33 | - **OPAL Client** - PORT `:7766` - the **_OPAL client_** has its own API, but it's irrelevant to this tutorial. 34 | - **OPA** - PORT `:8181` - the port of the **_OPA Agent_** that is running **running in server mode**. 35 | 36 | :::info 37 | **OPA** is being **run by OPAL client** in its container as a **managed process**. 38 | ::: 39 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/quickstart/opal-playground/send-queries-to-opa.mdx: -------------------------------------------------------------------------------- 1 | # Step 2: Sending authorization queries to OPA 2 | 3 | As mentioned above, the **_OPA Agent_** & it's **REST API** is running on port `:8181`. 4 | 5 | Let's explore the current state and send some authorization queries to the agent. 6 | 7 | The default policy in the [example repo](https://github.com/permitio/opal-example-policy-repo) is a simple 8 | [RBAC](https://en.wikipedia.org/wiki/Role-based_access_control) policy, to which we can issue the below request to get the user's 9 | **role assignment and metadata**. 10 | 11 | ```bash 12 | curl --request GET 'http://localhost:8181/v1/data/users' --header 'Content-Type: application/json' | python -m json.tool 13 | ``` 14 | 15 | The expected response should be like the one below. 16 | 17 | ```js showLineNumbers 18 | { 19 | "result": { 20 | "alice": { 21 | "location": { 22 | "country": "US", 23 | "ip": "8.8.8.8" 24 | }, 25 | "roles": [ 26 | "admin" 27 | ] 28 | }, 29 | 30 | ... 31 | } 32 | } 33 | ``` 34 | 35 | With some user data gathered, let's now issue an **authorization** query. In OPA, an authorization query is a query **with input**. 36 | 37 | This below query asks whether the user `bob` can `read` the `finance` resource, where the id of the object is `id123`. 38 | 39 | ```bash 40 | curl -w '\n' --request POST 'http://localhost:8181/v1/data/app/rbac/allow' \ 41 | --header 'Content-Type: application/json' \ 42 | --data-raw '{"input": {"user": "bob", "action": "read", "object": "id123", "type": "finance"}}' 43 | ``` 44 | 45 | The expected result is `true`, meaning the access is granted. 46 | 47 | ```bash 48 | {"result": true} 49 | ``` 50 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx: -------------------------------------------------------------------------------- 1 | # Step 3: Changing and updating the policy in realtime 2 | 3 | 9 | 10 | In the `docker-compose.yml` example file that we have mentioned earlier, it is defined that OPAL should 11 | track [this repository](https://github.com/permitio/opal-example-policy-repo). 12 | 13 | Here is a snippet of code from that repo: 14 | 15 | ```yml {13} showLineNumbers 16 | opal_server: 17 | # by default we run opal-server from latest official image 18 | image: permitio/opal-server:latest 19 | environment: 20 | # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) 21 | - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres 22 | # number of uvicorn workers to run inside the opal-server container 23 | - UVICORN_NUM_WORKERS=4 24 | # the git repo hosting our policy 25 | # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) 26 | # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy 27 | # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo 28 | - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo 29 | # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). 30 | # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. 31 | # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo 32 | - OPAL_POLICY_REPO_POLLING_INTERVAL=30 33 | ``` 34 | 35 | You can also simply change the tracked repo in the example `docker-compose.yml` file by editing these variables: 36 | 37 | ```yml {7,9,11} showLineNumbers 38 | services: 39 | ... 40 | opal_server: 41 | environment: 42 | ... 43 | - OPAL_POLICY_REPO_URL= 44 | # use this if you want to setup policy updates via git webhook (recommended) 45 | - OPAL_POLICY_REPO_WEBHOOK_SECRET= 46 | # use this if you want to setup policy updates via polling (not recommended) 47 | - POLICY_REPO_POLLING_INTERVAL= 48 | ``` 49 | 50 | You can then issue a commit affecting the policy and see that OPA state is indeed changing. 51 | 52 | :::info 53 | If you would like more information on managing and tracking a git repo, check out this [tutorial](/tutorials/track_a_git_repo). 54 | ::: 55 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/as-python-package/opal-client-setup.mdx: -------------------------------------------------------------------------------- 1 | # Setup up the OPAL Client 2 | 3 | ## Installing the package 4 | 5 | Firstly, let's install the `opal-client`. 6 | 7 | ```sh 8 | pip install opal-client 9 | ``` 10 | 11 | ## Installing OPA 12 | 13 | Next, we will need to install the policy-agent, or in other words OPA to run alongside the OPAL Client. 14 | 15 | For installing OPA, please follow [these instructions](https://www.openpolicyagent.org/docs/latest/#1-download-opa. 16 | 17 | If you would like OPAL to execute OPA for you, and act as a watchdog for OPA, we need to make sure it can **find the OPA** program 18 | and **make it executable**. To do this, please follow the **[guidance here](https://unix.stackexchange.com/questions/3809/how-can-i-make-a-program-executable-from-everywhere)**, 19 | but below is an example of what needs to be added. 20 | 21 | ``` 22 | export PATH=$PATH:/path/to/file 23 | ``` 24 | 25 | If you are currently in the directory where you will be adding OPA, you can run: 26 | 27 | ``` 28 | export PATH=$PATH:~ 29 | ``` 30 | 31 | :::note 32 | The client **needs network access** to this agent to be able to **administer updates** to it. 33 | ::: 34 | 35 | ## Running the OPAL Client and OPA 36 | 37 | To view the general commands and options offered by the `opal-client` command, please run: 38 | 39 | ```sh 40 | opal-client --help 41 | ``` 42 | 43 | If you need to learn about specific run option configurations and help, please run: 44 | 45 | ```sh 46 | opal-client run --help 47 | ``` 48 | 49 | Running the OPAL Client: 50 | 51 | ```sh 52 | opal-client run 53 | ``` 54 | 55 | Just like the server, all top-level options can be configured using environment-variables files like 56 | **[.env / .ini](https://pypi.org/project/python-decouple/#env-file)** and **command-line options**. 57 | 58 | #### The key options to be aware of: 59 | 60 | - Use options starting with `--server` to control how the client connects to the server. You will mainly need `--server-url` to 61 | point at the server. 62 | 63 | - Use options starting with `--client-api-` to control how the client's API service is running. 64 | 65 | - Use `--data-topics` to control which topics for data updates the client would subscribe to. 66 | 67 | - Use `--policy-subscription-dirs` to declare what directories in the repository we should subscribe to. 68 | 69 | ### Client install & run recording 70 | 71 |

72 | 76 | 77 | 78 |

79 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/as-python-package/running-in-prod.mdx: -------------------------------------------------------------------------------- 1 | # Running the OPAL Server and Client in Production 2 | 3 | ## Production run Server 4 | 5 | When running the server in production, we should set the server to work with a production server ([GUNICORN](https://gunicorn.org/)) 6 | and a backbone pub/sub. 7 | 8 | ### Gunicorn 9 | 10 | Simply use the `run` command with the `--engine-type gunicorn` option. 11 | 12 | ```sh 13 | opal-server run --engine-type gunicorn 14 | ``` 15 | 16 | - (run `opal-server run --help` to see more info on the `run` command) 17 | - use `--server-worker-count` to control the amount of workers (default is set to cpu-count) 18 | - You can of course put another server or proxy (e.g. NGNIX, ENVOY) in front of the OPAL-SERVER, instead of or in addition to Gunicorn 19 | 20 | ### Backbone Pub/Sub 21 | 22 | - While OPAL-servers provide a lightweight websocket pub/sub channel for the clients; in order for all OPAL-servers (workers of same server, and of course servers on other nodes) to be synced (And in turn their clients to be synced) they need to connect through a shared channel - which we refer to as the backbone pub/sub or broadcast channel. 23 | - Backbone Pub/Sub options: Kafka, Postgres LISTEN/NOTIFY, Redis 24 | - Use the `broadcast-uri` option (or `OPAL_BROADCAST_URI` env-var) to configure an OPAL-server to work with a backbone. 25 | - for example `OPAL_BROADCAST_URI=postgres://localhost/mydb opal-server run` 26 | 27 | ### Put it all together: 28 | 29 | ```sh 30 | OPAL_BROADCAST_URI=postgres://localhost/mydb opal-server run --engine-type gunicorn 31 | ``` 32 | 33 | ### Production run Client 34 | 35 | Unlike the server, the opal-client currently supports working only with a single worker process (so there's no need to run it with Gunicorn). 36 | This will change in future releases. 37 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/config-variables.mdx: -------------------------------------------------------------------------------- 1 | # Configuring Variables 2 | 3 | ### Configuration Variables 4 | 5 | We will now explain how to pass configuration variables to OPAL. 6 | 7 | - In its dockerized form, OPAL server and client containers pick up their configuration variables from **environment variables** prefixed with `OPAL_` (e.g: `OPAL_DATA_CONFIG_SOURCES`, `OPAL_POLICY_REPO_URL`, etc). 8 | - The OPAL CLI can pick up config vars from either **environment variables** prefixed with `OPAL_` or from **CLI arguments** (interchangeable). 9 | - Supported CLI options are listed in `--help`. 10 | - Each cli argument can match to a **corresponding** environment variable: 11 | - Simply convert the cli argument name to [SCREAMING_SNAKE_CASE](), and prefix it with `OPAL_`. 12 | - Examples: 13 | - `--server-url` becomes `OPAL_SERVER_URL` 14 | - `--data-config-sources` becomes `OPAL_DATA_CONFIG_SOURCES` 15 | 16 | ### Security Considerations (for production environments) 17 | 18 | Soon the OPAL Security Model will be available, we have listed the mandatory checklist below: 19 | 20 | - OPAL server should **always** be protected with a TLS/SSL certificate (i.e: HTTPS). 21 | - OPAL server should **always** run in secure mode - meaning JWT token verification should be active. 22 | - OPAL server should be configured with a **master token**. 23 | - Sensitive configuration variables (i.e: environment variables with sensitive values) should **always** be stored in a dedicated **Secret Store** 24 | - Example secret stores: AWS Secrets Manager, HashiCorp Vault, etc. 25 | - **NEVER EVER EVER** store secrets as part of your source code (e.g: in your git repository). 26 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/download-docker-images.mdx: -------------------------------------------------------------------------------- 1 | # Download OPAL Docker Images 2 | 3 | ## How to get the OPAL images from Docker Hub 4 | 5 | 6 | 7 | 8 | 9 | 12 | 15 | 16 | 17 | 20 | 23 | 35 | 36 | 37 | 40 | 43 | 60 | 61 | 62 | 67 | 70 | 82 | 83 | 84 |
Image Name 10 | How to Download 11 | 13 | Description 14 |
18 | OPAL Server 19 | 21 | docker pull permitio/opal-server 22 | 24 |
    25 |
  • Creates a Pub/Sub channel clients subscribe to
    • 26 |
  • 27 | Tracks a git repository (via webhook / polling) for updates to 28 | policy and static data 29 |
    • 30 |
  • Accepts data update notifications via Rest API
    • 31 |
  • Serves default data source configuration for clients
    • 32 |
  • Pushes policy and data updates to clients
    • 33 |
34 |
38 | OPAL Client 39 | 41 | docker pull permitio/opal-client 42 | 44 |
    45 |
  • Prebuilt with an OPA agent inside the image
    • 46 |
  • 47 | Keeps the OPA agent cache up to date with realtime updates pushed 48 | from the server 49 |
    • 50 |
  • 51 | Can selectively subscribe to specific topics of policy code (rego) 52 | and policy data 53 |
    • 54 |
  • 55 | Fetches data from multiple sources (e.g. DBs, APIs, 3rd party 56 | services) 57 |
    • 58 |
59 |
63 | 64 | OPAL Client (Standalone) 65 | 66 | 68 | docker pull permitio/opal-client-standalone 69 | 71 |
    72 |
  • 73 | Same as OPAL Client, you want only one of them 74 |
    • 75 |
  • This image does not come with OPA installed
    • 76 |
  • 77 | Intended to be used when you prefer to deploy OPA separately in its 78 | own container 79 |
    • 80 |
81 |
85 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/overview.mdx: -------------------------------------------------------------------------------- 1 | # Get started with OPAL docker containers 2 | 3 | This tutorial will teach you how to run OPAL using the official docker images. 4 | 5 | 6 | 7 | 8 | 11 | 20 | 21 | 22 | 27 | 42 | 43 | 44 |
9 | Use this tutorial if you 10 | 12 |
    13 |
  • Understand what OPAL is for (main features, how it works).
    • 14 |
  • 15 | Want to run OPAL with a real configuration. 16 |
    • 17 |
  • Want a step-by-step guide for deploying in production.
    • 18 |
19 |
23 | Use the{" "} 24 | other{" "} 25 | tutorial if you 26 | 28 |
    29 |
  • 30 | Want to explore OPAL quickly. 31 |
    • 32 |
  • 33 | Get a working playground with one{" "} 34 | docker compose command. 35 |
    • 36 |
  • 37 | Want to learn about OPAL core features and see what 38 | OPAL can do for you. 39 |
    • 40 |
41 |
45 | 46 | ## Table of Content 47 | 48 | - [Download OPAL images from Docker Hub](/getting-started/running-opal/download-docker-images) 49 | - [How to run OPAL Client](/getting-started/running-opal/run-opal-client/lets-run-the-client) 50 | - [How to run OPAL Server](/getting-started/running-opal/run-opal-server/putting-all-together) 51 | - [Troubleshooting](/getting-started/running-opal/troubleshooting) 52 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/run-opal-client/data-topics.mdx: -------------------------------------------------------------------------------- 1 | # Subscribe to Data Topics 2 | 3 | ### Step 4: Client config - data topics (Optional) 4 | 5 | You can configure which topics for data updates the client will subscribe to. This is great if you want more granularity in your data model, for example: 6 | 7 | - **Enabling multi-tenancy:** you deploy each customer (tenant) with his own OPA agent, each agent's OPAL client will subscribe only to the relevant tenant's topic. 8 | - **Sharding large datasets:** you split a big data set (i.e: policies based on user attributes and you have **many** users) to many instances of OPA agent, each agent's OPAL client will subscribe only to the relevant's shard topic. 9 | 10 | If you do not specify data topics in your configuration, OPAL client will automatically subscribe to a single topic: `policy_data` (the default). 11 | 12 | Use this env var to control which topics the client will subscribe to: 13 | 14 | | Env Var Name | Function | 15 | | :--------------- | :---------------------------------------- | 16 | | OPAL_DATA_TOPICS | data topics delimited by comma (i,e: `,`) | 17 | 18 | Example value: 19 | 20 | ```sh 21 | export OPAL_DATA_TOPICS=topic1,topic2,topic3 22 | ``` 23 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/run-opal-client/get-client-image.mdx: -------------------------------------------------------------------------------- 1 | # Download the Client Image 2 | 3 | ### Step 1: Get the client image from docker hub 4 | 5 | #### Running with inline OPA (default / recommended) 6 | 7 | Run this command to get the image that comes with built-in OPA (recommended if you don't already have OPA installed in your environment): 8 | 9 | ``` 10 | docker pull permitio/opal-client 11 | ``` 12 | 13 | If you run in a cloud environment (e.g: AWS ECS), specify `permitio/opal-client` in your task definition or equivalent. 14 | 15 | #### Running with standalone OPA 16 | 17 | Otherwise, if you are already running OPA in your environment, run this command to get the standalone client image instead: 18 | 19 | ``` 20 | docker pull permitio/opal-client-standalone 21 | ``` 22 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/run-opal-client/lets-run-the-client.mdx: -------------------------------------------------------------------------------- 1 | # Run the OPAL Client 2 | 3 | ### Step 7: Running the client 4 | 5 | Let's recap the previous steps with example values: 6 | 7 | #### 1) Get the client image 8 | 9 | First, download opal client docker image: 10 | 11 | ```sh 12 | docker pull permitio/opal-client 13 | ``` 14 | 15 | #### 2) Set configuration 16 | 17 | Then, declare configuration with environment variables: 18 | 19 | ```sh 20 | # let's say this is the (shortened) token we obtained from opal server 21 | export OPAL_CLIENT_TOKEN=eyJ0...8wsk 22 | # and this is where we deployed opal server 23 | export OPAL_SERVER_URL=https://opal.yourdomain.com 24 | # and let's say we subscribe to a specific tenant's data updates (i.e: `tenant1`) 25 | export OPAL_DATA_TOPICS=policy_data/tenant1 26 | ``` 27 | 28 | and let's assume we run opa inline with the default options. 29 | 30 | #### 3) Run the container (local run example) 31 | 32 | ``` 33 | docker run -it \ 34 | --env OPAL_CLIENT_TOKEN \ 35 | --env OPAL_SERVER_URL \ 36 | --env OPAL_DATA_TOPICS \ 37 | -p 7766:7000 \ 38 | -p 8181:8181 \ 39 | permitio/opal-client 40 | ``` 41 | 42 | Please notice opal client exposes two ports when running opa inline: 43 | 44 | - OPAL Client (port `:7766`) - the OPAL client API (i.e: healthcheck, etc). 45 | - OPA (port `:8181`) - the port of the OPA agent (OPA is running in server mode). 46 | 47 | #### 4) Run the container in production 48 | 49 | [Same instructions as for OPAL server](#run-docker-prod). 50 | 51 | ## How to push data updates from an authoritative source 52 | 53 | Now that OPAL is live, we can use OPAL server to push updates to OPAL clients in real time. 54 | 55 | [We have a separate tutorial on how to trigger updates](/tutorials/trigger_data_updates). 56 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/run-opal-client/obtain-jwt-token.mdx: -------------------------------------------------------------------------------- 1 | # Obtain the JWT token 2 | 3 | ### Step 2: Obtain client JWT token (Optional) 4 | 5 | In production environments, OPAL server **should** be running in **secure mode**, and the OPAL client must have a valid identity token (which is a signed JWT) in order to successfully connect to the server. 6 | 7 | Obtaining a token is easy. You'll need the OPAL server's **master token** in order to request a JWT token. 8 | 9 | Let's install the `opal-client` cli to a new python virtualenv (assuming you didn't [already create one](#generate-secret)): 10 | 11 | ```sh 12 | # this command is not necessary if you already created this virtualenv 13 | pyenv virtualenv opal 14 | # this command is not necessary if the virtualenv is already active 15 | pyenv activate opal 16 | # this command installs the client cli 17 | pip install opal-client 18 | ``` 19 | 20 | You can obtain a client token with this cli command: 21 | 22 | ``` 23 | opal-client obtain-token MY_MASTER_TOKEN --server-url=https://opal.yourdomain.com --type client 24 | ``` 25 | 26 | If you don't want to use the cli, you can obtain the JWT directly from the deployed OPAL server via its REST API: 27 | 28 | ``` 29 | curl --request POST 'https://opal.yourdomain.com/token' \ 30 | --header 'Authorization: Bearer MY_MASTER_TOKEN' \ 31 | --header 'Content-Type: application/json' \ 32 | --data-raw '{ 33 | "type": "client" 34 | }' 35 | ``` 36 | 37 | The `/token` API endpoint can receive more parameters, as [documented here](https://opal.permit.io/redoc#operation/generate_new_access_token_token_post). 38 | 39 | This example assumes that: 40 | 41 | - You deployed OPAL server to `https://opal.yourdomain.com` 42 | - The master token of your deployment is `MY_MASTER_TOKEN`. 43 | - However, if you followed our tutorial for the server, you probably generated one [here](#generate-secret) and that is the master token you should use. 44 | 45 | example output: 46 | 47 | ```json 48 | { 49 | "token": "eyJ0...8wsk", 50 | "type": "bearer", 51 | "details": { ... } 52 | } 53 | ``` 54 | 55 | Put the generated token value (the one inside the `token` key) into this environment variable: 56 | 57 | | Env Var Name | Function | 58 | | :---------------- | :--------------------------------------------------------------------------- | 59 | | OPAL_CLIENT_TOKEN | The client identity token (JWT) used for identification against OPAL server. | 60 | 61 | Example: 62 | 63 | ```sh 64 | export OPAL_CLIENT_TOKEN=eyJ0...8wsk 65 | ``` 66 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/run-opal-client/server-uri.mdx: -------------------------------------------------------------------------------- 1 | # Configure the Server URI 2 | 3 | ### Step 3: Client config - server uri 4 | 5 | Set the following environment variable according to the address of the deployed OPAL server: 6 | 7 | | Env Var Name | Function | 8 | | :-------------- | :---------------------------------------------------------------------------------------------------------------------- | 9 | | OPAL_SERVER_URL | The internet address (uri) of the deployed OPAL server. In production, you must use an `https://` address for security. | 10 | 11 | Example, if the OPAL server is available at `https://opal.yourdomain.com`: 12 | 13 | ```sh 14 | export OPAL_SERVER_URL=https://opal.yourdomain.com 15 | ``` 16 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/run-opal-client/standalone-opa-uri.mdx: -------------------------------------------------------------------------------- 1 | # Standalone OPA URI 2 | 3 | ### Step 6: Client config - Standalone OPA uri (Optional) 4 | 5 | If OPA is deployed separately from OPAL (i.e: using the standalone image), you should define the URI of the OPA instance you want to manage with OPAL client with this env var: 6 | 7 | | Env Var Name | Function | 8 | | :-------------------- | :--------------------------------------------------------- | 9 | | OPAL_POLICY_STORE_URL | The internet address (uri) of the deployed standalone OPA. | 10 | 11 | Example, if the standalone OPA is available at `https://opa.billing.yourdomain.com:8181`: 12 | 13 | ```sh 14 | export OPAL_POLICY_STORE_URL=https://opa.billing.yourdomain.com:8181 15 | ``` 16 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/run-opal-server/get-server-image.mdx: -------------------------------------------------------------------------------- 1 | # Get the Server Image 2 | 3 | ### Step 1: Get the server image from docker hub 4 | 5 | If you run the docker image locally, you need docker installed on your machine. 6 | 7 | Run this command to get the image: 8 | 9 | ``` 10 | docker pull permitio/opal-server 11 | ``` 12 | 13 | If you run in a cloud environment (e.g: AWS ECS), specify `permitio/opal-server` in your task definition or equivalent. 14 | 15 | Running the opal server container is simply a command of [docker run](#example-docker-run), but we need to pipe to the OPAL server container the necessary configuration it needs via **environment variables**. The following sections will explain each class of configuration variables and how to set their values, after which we will demonstrate real examples. 16 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/running-opal/run-opal-server/putting-all-together.mdx: -------------------------------------------------------------------------------- 1 | # Running the Server 2 | 3 | ### Step 7: Putting it all together - running the server 4 | 5 | To summarize, the previous steps guided you on how to pick the values of the configuration variables needed to run OPAL server. 6 | 7 | We will now recap with a real example. 8 | 9 | #### 1) Pull the server container image 10 | 11 | ``` 12 | docker pull permitio/opal-server 13 | ``` 14 | 15 | #### 2) Define the environment variables you need 16 | 17 | Multiple workers and broadcast channel (example values from step 2): 18 | 19 | ``` 20 | export OPAL_BROADCAST_URI=postgres://localhost/mydb 21 | export UVICORN_NUM_WORKERS=4 22 | ``` 23 | 24 | Policy repo (example values from step 3): 25 | 26 | ``` 27 | export OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo 28 | ``` 29 | 30 | Policy repo syncing with webhook (example values from step 4): 31 | 32 | ``` 33 | export OPAL_POLICY_REPO_WEBHOOK_SECRET=-cBlFnldg7WCGlj0jsivPWPA5vtfI2GWmp1wVx657Vk 34 | ``` 35 | 36 | Data sources configuration (example values from step 5): 37 | 38 | ``` 39 | export OPAL_DATA_CONFIG_SOURCES='{"config": {"entries": [{"url": "https://api.permit.io/v1/policy-config", "topics": ["policy_data"], "config": {"headers": {"Authorization": "Bearer FAKE-SECRET"}}}]}}' 40 | ``` 41 | 42 | Security parameters (example values from step 6): 43 | 44 | ``` 45 | export OPAL_AUTH_PRIVATE_KEY=-----BEGIN OPENSSH PRIVATE KEY-----_XXX..._..._...XXX==_-----END OPENSSH PRIVATE KEY----- 46 | export OPAL_AUTH_PUBLIC_KEY=ssh-rsa XXX ... XXX== some@one.com 47 | export OPAL_AUTH_MASTER_TOKEN=8MHfUU2rzRB59pdOHNNVVw3XLe3gl9YNw7vIXxJZNJo 48 | ``` 49 | 50 | #### 3) Run the container (local run example) 51 | 52 | ``` 53 | docker run -it \ 54 | --env OPAL_BROADCAST_URI \ 55 | --env UVICORN_NUM_WORKERS \ 56 | --env OPAL_POLICY_REPO_URL \ 57 | --env OPAL_POLICY_REPO_WEBHOOK_SECRET \ 58 | --env OPAL_DATA_CONFIG_SOURCES \ 59 | --env OPAL_AUTH_PRIVATE_KEY \ 60 | --env OPAL_AUTH_PUBLIC_KEY \ 61 | --env OPAL_AUTH_MASTER_TOKEN \ 62 | -p 7002:7002 \ 63 | permitio/opal-server 64 | ``` 65 | 66 | #### 4) Run the container in production 67 | 68 | As we mentioned before, in production you will not use `docker run`. 69 | 70 | Deployment looks somewhat like this: 71 | 72 | - Declare your container configuration in code, e.g: AWS ECS task definition file, Helm chart, etc. 73 | - All the secrets and sensitive vars should be fetched from a secrets store. 74 | - Deploy your task / helm chart, etc to your cloud environment. 75 | - Expose the server to the internet with HTTPS (i.e: use a valid SSL/TLS certificate). 76 | - Keep your master token in a safe location (you will need it shortly to generate identity tokens). 77 | 78 | ## How to run OPAL Client 79 | 80 | Great! we have OPAL Server up and running. Let's continue and explains how to run OPAL Client. 81 | -------------------------------------------------------------------------------- /documentation/docs/getting-started/tldr.mdx: -------------------------------------------------------------------------------- 1 | # OPAL TL;DR 2 | 3 | OPAL is an advanced piece of software with many capabilities and configuration options, hence it has a lot of docs; but if you want just the gist of it - this is the article for you. 4 | 5 | ## How OPAL works 6 | 7 | The OPAL server sends instructions to the OPAL-clients (via pub/sub subscriptions over websockets) to load policy and data into their managed policy-agents (e.g. OPA, Cedar-agent, AWS AVP) 8 | 9 | ### Policy 10 | 11 | OPAL tracks [policies from Git](/tutorials/track_a_git_repo) or from [API bundle servers](/tutorials/track_an_api_bundle_server). 12 | 13 | With Git - directories with policy-code (e.g. `.rego` or `.cedar` files) are automatically mapped to topics - which a client can subscribe to with `OPAL_POLICY_SUBSCRIPTION_DIRS` 14 | Every time you push a change, the OPAL server will notify the subscribing OPAL-clients to load the new policy. 15 | 16 | ### Data 17 | 18 | OPAL tracks data from various sources via webhooks and [Fetch-Providers](/tutorials/write_your_own_fetch_provider) (extensible python modules that teach it to load data from sources). 19 | 20 | [Initial data is indicated by the server](getting-started/running-opal/run-opal-server/data-sources) based on `OPAL_DATA_CONFIG_SOURCES`. 21 | Subsequent data updates are triggered via [the data update webhook](/tutorials/trigger_data_updates). 22 | Every time the policy agent (or it's managing OPAL-client) restarts, the data and policy are loaded from scratch. 23 | 24 | #### Data as part of policy bundle 25 | 26 | Data can also be loaded with the policy as part of `data.json` files, located in the folders next to the policy file. 27 | 28 | :::note 29 | The **folder path** is used as the **key path** in the policy engine cache. 30 | In order to avoid race conditions between policy data updates and regular data updates, make sure the key paths used by your policy-data and the ones used by your data-updates are different. 31 | ::: 32 | -------------------------------------------------------------------------------- /documentation/docs/opal-plus/deploy.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 2 3 | title: Deploy OPAL+ 4 | --- 5 | 6 | With OPAL+, you get access to private Docker images that include additional features and capabilities. 7 | To apply for Permit OPAL+, [fill in the form available here](https://hello.permit.io/opal-plus) 8 | 9 | In order to access the OPAL+ Docker images, you need to have Docker Hub credentials with an access token. 10 | Those should be received from your Customer Success manager. 11 | Reach out to us [on Slack](https://bit.ly/permit-slack) if you need assistance. 12 | 13 | ## Accessing the OPAL+ Docker Images 14 | 15 | To access the OPAL+ Docker images, you need to log in to Docker Hub with your credentials. 16 | You can do this by running the [docker login](https://docs.docker.com/reference/cli/docker/login/) command: 17 | 18 | ```bash 19 | docker login -u -p 20 | ``` 21 | 22 | If you are using Kubernetes, check out the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) on how to pull images from a private registry. 23 | 24 | After logging in, you can pull the OPAL+ Docker images using the following commands: 25 | 26 | ```bash 27 | docker pull permitio/opal-plus:latest 28 | ``` 29 | 30 | ## Running the OPAL+ Docker Images 31 | 32 | Running the OPAL+ Docker images is similar to running the open-source OPAL images. 33 | 34 | Check out the [OPAL Docker documentation](/getting-started/running-opal/run-docker-containers) for more information. 35 | -------------------------------------------------------------------------------- /documentation/docs/opal-plus/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 1 3 | title: Introduction 4 | --- 5 | 6 | 7 |
15 | {" "} 16 | {" "} 17 |
18 | 19 | :::note 20 | OPAL is and will always be an open-source project free for all. 21 | OPAL+ is a way for enterprise users to get more out of OPAL when needed; and is a product of OPAL users approaching us and asking for additional capabilities on top of those provided by OPAL. 22 | 23 | If you just need a hosted version of OPAL; or you're building application-level permissions consider simply using [Permit.io's PRO tier](https://www.permit.io/pricing). 24 | ::: 25 | 26 | ## What is Permit OPAL+ 27 | 28 | Permit OPAL+ is an enterprise software service package provided by Permit Inc. the company behind Permit.io and OPAL. 29 | The package can include: 30 | 31 | - A special license for an internal OPAL version used by Permit ( early access to features before they are opened sourced) 32 | - Hosted OPAL servers 33 | - Hosted policy decision points (OPAL-client + OPA / Cedar) 34 | - Direct access and impact on the project road-map 35 | - Dedicated support channel 36 | - Custom SLA 37 | - Professional services for OPAL 38 | - [Custom data-fetcher providers](/tutorials/write_your_own_fetch_provider), including NRE. 39 | 40 | See more about the [features of OPAL+](/opal-plus/features). 41 | 42 | ## Joining the OPAL+ Program 43 | 44 | The OPAL+ program is available for select enterprises, who apply for access; 45 | Currently the program can accept up to 5 enterprises. Applications are reviewed and considered on a first-come-first-served basis. 46 | 47 | To apply for Permit OPAL+, [fill in the form available here](https://hello.permit.io/opal-plus) 48 | -------------------------------------------------------------------------------- /documentation/docs/opal-plus/troubleshooting.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 10 3 | title: Troubleshooting 4 | --- 5 | 6 | When something goes wrong, we are here to help. 7 | 8 | Feel free to reach out to us on Slack on our [community channel](https://bit.ly/opal-slack) 9 | or in your dedicated OPAL+ support channel, and we will do our best to assist you quickly. 10 | 11 | ### Seeking Support 12 | 13 | When seeking support, please provide as much information as possible to help us understand the issue and resolve it faster. 14 | This includes: 15 | * Full description of the problem 16 | * Steps to reproduce the issue 17 | * Environment details (OS, Docker, Kubernetes, etc.) 18 | 19 | You can also provide the following information: 20 | 21 | #### Extract Logs 22 | 23 | On production, we advise you to connect OPAL+ to your logging system to collect logs. 24 | Configure the [OPAL_LOG_SERIALIZE](/getting-started/configuration) environment variable to `true` to serialize logs in JSON format. 25 | 26 | When running OPAL+ locally or in a development environment, you can extract logs from the console. 27 | Alternatively, you can enable logging to file by setting the [OPAL_LOG_TO_FILE](/getting-started/configuration) to `true`. 28 | 29 | You can also enable debug logs by setting the [OPAL_LOG_LEVEL](/getting-started/configuration) environment variable to `debug`. 30 | 31 | #### Export Configuration 32 | 33 | You can export your configuration by running the following command: 34 | 35 | ```bash 36 | opal-client print-config 37 | ``` 38 | 39 | Make sure to censor any sensitive information, like passwords or API keys, before sharing the configuration. 40 | 41 | ## Common Issues 42 | 43 | More common issues and their solutions are available in the [OPAL documentation](/getting-started/running-opal/troubleshooting). 44 | -------------------------------------------------------------------------------- /documentation/docs/overview/_security.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 5 3 | title: Security 4 | --- 5 | 6 | 7 | 8 | 9 | 10 | **Coming Soon** - otherwise - OPAL is **ready for production** security-wise. 11 | 12 | 26 | -------------------------------------------------------------------------------- /documentation/docs/overview/modules.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 4 3 | title: Core Modules 4 | --- 5 | 6 | 7 | - ## OPAL-Server 8 | - Policy 9 | - git-webhook 10 | - git-watcher 11 | - Data 12 | - DataUpdatePublisher 13 | - PubSub 14 | - Publisher 15 | - API 16 | 17 | 18 | 19 | 20 | - ## OPAL-Client 21 | - OPA-runner 22 | - PolicyStoreClient 23 | - Policy 24 | - PolicyUpdater 25 | - PolicyFetcher 26 | - Data 27 | - DataUpdater 28 | - DataFetcher 29 | - FetchingEngine 30 | - FetchingProviders 31 | - API 32 | -------------------------------------------------------------------------------- /documentation/docs/tutorials/_configure_backbone_pubsub.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 1 3 | title: Configure Backbone Pubsub 4 | --- 5 | 6 | **Coming Soon** 7 | -------------------------------------------------------------------------------- /documentation/docs/tutorials/cedar.mdx: -------------------------------------------------------------------------------- 1 | # Cedar-Agent and Cedar 2 | 3 | Cedar is an open-source engine and language created by AWS. 4 | [Cedar agent](https://github.com/permitio/cedar-agent) is an OSS project from Permit.io - which provides the ability to run Cedar as a standalone agent (Similar to how one would use OPA) which can then be powered by [OPAL](https://github.com/permitio/opal). 5 | Cedar agent is the easiest way to deploy and run Cedar. 6 | 7 | :::info Demo 8 | Check out our [demo app that uses Cedar-Agent and OPAL here](https://github.com/permitio/tinytodo). 9 | ::: 10 | 11 | OPAL can run Cedar instead of OPA. To launch an example configuration with Docker Compose, do: 12 | ``` 13 | git clone https://github.com/permitio/opal.git 14 | cd opal 15 | docker compose -f docker/docker-compose-example-cedar.yml up -d 16 | ``` 17 | 18 | You'll then have Cedar's dev web interface at [http://localhost:8180/rapidoc/](http://localhost:8180/rapidoc/), where you can call Cedar-Agent's API routes. 19 | 20 | You can show data with GET on **/data**, policy with GET on **/policies**, and you can POST the following authorization to **/is_authorized** request to perform an authorization check: 21 | ``` 22 | { 23 | "principal": "User::\"someone@permit.io\"", 24 | "action": "Action::\"document:write\"", 25 | "resource": "ResourceType::\"document\"" 26 | } 27 | ``` 28 | 29 | To show how the policy affects the request, set a policy with fewer permissions with a PUT on **/policies**: 30 | ``` 31 | [ 32 | { 33 | "id": "policy.cedar", 34 | "content": "permit(\n principal in Role::\"Editor\",\n action in [Action::\"document:read\",Action::\"document:delete\"],\n resource in ResourceType::\"document\"\n) when {\n true\n};" 35 | } 36 | ] 37 | ``` 38 | Then restore the correct policy: 39 | ``` 40 | [ 41 | { 42 | "id": "policy.cedar", 43 | "content": "permit(\n principal in Role::\"Editor\",\n action in [Action::\"document:read\",Action::\"document:write\",Action::\"document:delete\"],\n resource in ResourceType::\"document\"\n) when {\n true\n};" 44 | } 45 | ] 46 | ``` 47 | Alternatively, you can also change the Docker compose config and set your own policy git repo (the **OPAL_POLICY_REPO_URL** variable), and change it on the fly. 48 | 49 | 50 | If you want to see OPAL's logs, you can do: 51 | ``` 52 | docker compose -f docker/docker-compose-example-cedar.yml logs opal_server 53 | ``` 54 | and 55 | ``` 56 | docker compose -f docker/docker-compose-example-cedar.yml logs opal_client 57 | ``` 58 | For the server and client, respectively. 59 | -------------------------------------------------------------------------------- /documentation/docs/tutorials/configure_opal.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 3 3 | title: Configure OPAL 4 | --- 5 | 6 | # Configure OPAL 7 | 8 | Both OPAL-Server and OPAL client provide multiple ways to load configuration: 9 | 10 | - via environment variables (prefixed with 'OPAL\_') 11 | - via command line values 12 | - via a '.env' or '.ini' file. 13 | 14 | You can combine configuration sources; be aware of the override rules: 15 | default < .env file < env variable < command line 16 | i.e. the settings in the env file override the defaults, but both env-variables and command line values override them. 17 | 18 | ## What are the configuration variables available 19 | 20 | To view the configuration settings for both OPAL-SERVER and OPAL-CLIENT you can do one of: 21 | 22 | - Run the server or client as a CLI - and use the '--help' command. 23 | - using the '--help' on specific commands such as 'run' will provide more information 24 | - Look at the configuration code itself: 25 | - [Common config](https://github.com/permitio/opal/blob/master/packages/opal-common/opal_common/config.py) : values available for both server and client 26 | - [Server config](https://github.com/permitio/opal/blob/master/packages/opal-server/opal_server/config.py) : values available only for the server 27 | - [Client config](https://github.com/permitio/opal/blob/master/packages/opal-client/opal_client/config.py) : values available only for the client 28 | 29 | ## Configuration architecture 30 | 31 | OPAL's configuration is based on our very own `Confi` module, which in turn is based on [Decouple](https://pypi.org/project/python-decouple/), and adds complex value parsing with Pydantic, and command line arguments via Typer/Click. 32 | 33 | ## Configuring logs 34 | 35 | OPAL supports to log out puts - STDERR/STDOUT and a log file review the variables prefixed with `LOG_` for the options. 36 | -------------------------------------------------------------------------------- /documentation/docs/tutorials/install_as_python_packages.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 5 3 | title: Run OPAL as Python Packages 4 | --- 5 | 6 | # Run OPAL as Python packages / CLI utilities 7 | 8 | ## CLI execution 9 | 10 | Python version **3.7 or greater** is required. 11 | 12 | ### Server 13 | 14 | - install 15 | ```sh 16 | pip install opal-server 17 | ``` 18 | - run 19 | ```sh 20 | # get CLI help 21 | opal-server --help 22 | # run server as daemon 23 | opal-server run 24 | ``` 25 | 26 | ### Client 27 | 28 | - install 29 | ``` 30 | pip install opal-client 31 | ``` 32 | - run 33 | ```sh 34 | # get CLI help 35 | opal-client --help 36 | # run the client as a daemon 37 | opal-client run 38 | ``` 39 | -------------------------------------------------------------------------------- /documentation/docs/tutorials/monitoring_opal.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 11 3 | title: Monitoring OPAL 4 | --- 5 | 6 | # Monitoring OPAL 7 | 8 | There are multiple ways you can monitor your OPAL deployment: 9 | 10 | - **Logs** - Using the structured logs outputted to stderr by both the OPAL-servers and OPAL-clients 11 | - **Health-checks** - OPAL exposes HTTP health check endpoints ([See below](##health-checks)) 12 | - [**Callbacks**](/tutorials/healthcheck_policy_and_update_callbacks#-data-update-callbacks) - Using the callback webhooks feature - having OPAL-clients report their updates 13 | - **Statistics** - Using the built-in statistics feature in OPAL ([See below](##opal-statistics)) 14 | 15 | ## Health checks 16 | 17 | ### OPAL Server 18 | 19 | opal-server exposes http health check endpoints on `/` & `/healthcheck`.
20 | Currently it returns `200 OK` as long as server is up. 21 | 22 | ### OPAL Client 23 | 24 | opal-client exposes 2 types of http health checks: 25 | 26 | - **Readiness** - available on `/ready`.
27 | Returns `200 OK` if the client have loaded policy & data to OPA at least once (from either server, or local backup file), otherwise `503 Unavailable` is returned. 28 | 29 | - **Liveness** - available on `/`, `/healthcheck` & `/healthy`.
30 | Returns `200 OK` if the last attempts to load policy & data to OPA were successful, otherwise `503 Unavailable` is returned 31 | 32 | **Notice:** if you don't except your opal-client to load any data into OPA, set `OPAL_DATA_UPDATER_ENABLED: False`, so opal-client could report being healthy. 33 | 34 | You can also configure opal-client to store dynamic health status as a document in OPA, [Learn more here](/tutorials/healthcheck_policy_and_update_callbacks) 35 | 36 | ## OPAL Statistics 37 | 38 | By enabling `OPAL_STATISTICS_ENABLED=true` (on both servers and clients), the OPAL-Server would start maintaining a unified state of all the clients and which topics they've subscribed to. 39 | The state can then be retrieved as a JSON object by calling the `/statistics` api route on the server 40 | 41 | ### Code Reference: 42 | 43 | - [opal_server/statistics.py](https://github.com/permitio/opal/blob/master/packages/opal-server/opal_server/statistics.py) 44 | 45 | 46 | ## OPAL Client tracker (EAP) 47 | 48 | Alternative implementation for the Statistics feature, OPAL-Server tracks all OPAL-clients connected through websocket. 49 | Gathered information includes connection details (client's source host and port), connection time, and subscribed topics. 50 | Available through `/pubsub_client_info` api route on the server. 51 | 52 | ### Caveats: 53 | - When `UVICORN_NUM_WORKERS > 1`, retrieved information would only include clients connected to the replying server process. 54 | - This is an early access feature and is likely to change. Backward compatibility is not garaunteed. 55 | -------------------------------------------------------------------------------- /documentation/docs/welcome.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 1 3 | slug: / 4 | title: Welcome to OPAL 👋 5 | --- 6 | 7 |
8 | opal 13 |
14 | 15 | ## **Open Policy Administration Layer** 16 | 17 | OPAL is an administration layer for Policy Engines such as [Open Policy Agent (OPA)](https://www.openpolicyagent.org), 18 | and [AWS' Cedar Agent](https://github.com/permitio/cedar-agent). OPAL detects changes to both policy and policy data in realtime, and pushes live 19 | updates to your agents - briging open-policy up to the speed needed by live applications. 20 | 21 | As your application state changes (whether it's via your APIs, DBs, git, S3 or 3rd-party SaaS services), 22 | OPAL will make sure your services are always in sync with the authorization data and policy they need. 23 | 24 | ### Getting Started 25 | 26 | OPAL is available both as python packages with a built-in CLI as well as pre-built docker images. 27 | 28 | - **[Try the OPAL live playground environment in docker-compose](https://docs.opal.ac/getting-started/quickstart/opal-playground/overview/)** 29 | - **[Getting Started Guide for Containers](https://docs.opal.ac/getting-started/running-opal/overview/)** 30 | - **[OPAL Kubernetes Helm Chart](https://github.com/permitio/opal-helm-chart)** 31 | - An in-depth introduction to OPAL **[is available here](https://www.permit.io/blog/introduction-to-opal)**. 32 | 33 | ### Need help? 34 | 35 | Come talk to us about OPAL, or authorization in general - we would love to hear from you ❤️ 36 | 37 | 57 | 58 | You can also ask questions and request features to be added to the road-map in our [**Github discussions**](https://github.com/permitio/opal/discussions). 59 | Issues should be reported in [**Github issues**](https://github.com/permitio/opal/issues). 60 | 61 | Want to support the project? **[Give us a ⭐️ on GitHub!](https://github.com/permitio/opal)** 62 | -------------------------------------------------------------------------------- /documentation/docusaurus.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | // Note: type annotations allow type checking and IDEs autocompletion 3 | 4 | 5 | /** @type {import('@docusaurus/types').Config} */ 6 | const config = { 7 | title: "OPAL", 8 | tagline: "Administration layer for the Open Policy Agent", 9 | url: "https://docs.opal.ac", 10 | baseUrl: "/", 11 | onBrokenLinks: "throw", 12 | onBrokenMarkdownLinks: "warn", 13 | favicon: "img/favicon.ico", 14 | organizationName: "permitio", // Usually your GitHub org/user name. 15 | projectName: "opal", // Usually your repo name. 16 | 17 | // Even if you don't use internalization, you can use this field to set useful 18 | // metadata like html lang. For example, if your site is Chinese, you may want 19 | // to replace "en" with "zh-Hans". 20 | i18n: { 21 | defaultLocale: "en", 22 | locales: ["en"], 23 | }, 24 | 25 | presets: [ 26 | [ 27 | "classic", 28 | /** @type {import('@docusaurus/preset-classic').Options} */ 29 | ({ 30 | docs: { 31 | sidebarPath: require.resolve("./sidebars.js"), 32 | routeBasePath: "/", 33 | // Please change this to your repo. 34 | // Remove this to remove the "edit this page" links. 35 | }, 36 | blog: false, // disabled docusaurus default blog 37 | theme: { 38 | customCss:[ require.resolve("./src/css/custom.scss") 39 | ] 40 | }, 41 | }), 42 | ], 43 | ], 44 | 45 | plugins: ["docusaurus-plugin-sass"], 46 | 47 | themeConfig: 48 | /** @type {import('@docusaurus/preset-classic').ThemeConfig} */ 49 | ({ 50 | navbar: { 51 | title: "OPAL", 52 | logo: { 53 | alt: "My Site Logo", 54 | src: "img/opal.png", 55 | }, 56 | items: [ 57 | { 58 | href: "https://github.com/permitio/opal", 59 | label: "GitHub", 60 | position: "right", 61 | }, 62 | ], 63 | }, 64 | footer: { 65 | style: "dark", 66 | links: [ 67 | { 68 | label: "Visit Permit.io", 69 | to: "https://permit.io", 70 | }, 71 | ], 72 | copyright: `Copyright © ${new Date().getFullYear()} Permit, Inc.`, 73 | }, 74 | prism: { 75 | theme: require('prism-react-renderer').themes.nightOwl, 76 | additionalLanguages: ['bash'] 77 | }, 78 | announcementBar: { 79 | id: "support_us", 80 | content: 81 | 'If you like OPAL, give us a ⭐️ on GitHub and follow us on Twitter', 82 | backgroundColor: "#6851ff", 83 | textColor: "#FFFFFF", 84 | isCloseable: true, 85 | }, 86 | }), 87 | }; 88 | 89 | module.exports = config; 90 | -------------------------------------------------------------------------------- /documentation/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "documentation", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "docusaurus": "docusaurus", 7 | "start": "docusaurus start", 8 | "build": "docusaurus build", 9 | "swizzle": "docusaurus swizzle", 10 | "deploy": "docusaurus deploy", 11 | "clear": "docusaurus clear", 12 | "serve": "docusaurus serve", 13 | "write-translations": "docusaurus write-translations", 14 | "write-heading-ids": "docusaurus write-heading-ids" 15 | }, 16 | "dependencies": { 17 | "@docusaurus/core": "3.0.0", 18 | "@docusaurus/preset-classic": "3.0.0", 19 | "@mdx-js/react": "^3.0.0", 20 | "clsx": "^1.2.1", 21 | "docusaurus-plugin-sass": "^0.2.5", 22 | "node": "^18.0.0", 23 | "prism-react-renderer": "^2.1.0", 24 | "react": "^18.3.0", 25 | "react-dom": "^18.3.0", 26 | "sass": "^1.71.1", 27 | "axios": "^1.7.5", 28 | "micromatch": "^4.0.8" 29 | }, 30 | "devDependencies": { 31 | "@docusaurus/module-type-aliases": "3.0.0" 32 | }, 33 | "browserslist": { 34 | "production": [ 35 | ">0.5%", 36 | "not dead", 37 | "not op_mini all" 38 | ], 39 | "development": [ 40 | "last 1 chrome version", 41 | "last 1 firefox version", 42 | "last 1 safari version" 43 | ] 44 | }, 45 | "engines": { 46 | "node": ">=18.0" 47 | }, 48 | "overrides": { 49 | "got": "11.8.5", 50 | "trim": "0.0.3" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /documentation/src/css/prism-theme.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | // Original: https://github.com/dracula/visual-studio-code 4 | // Converted automatically using ./tools/themeFromVsCode 5 | var theme = { 6 | plain: { 7 | color: "#fff", 8 | backgroundColor: "#000", 9 | }, 10 | styles: [ 11 | { 12 | types: ["prolog", "constant", "builtin"], 13 | style: { 14 | color: "#9bdbff", 15 | }, 16 | }, 17 | { 18 | types: ["inserted", "function"], 19 | style: { 20 | color: "#ee90a3", 21 | }, 22 | }, 23 | { 24 | types: ["deleted"], 25 | style: { 26 | color: "rgb(255, 85, 85)", 27 | }, 28 | }, 29 | { 30 | types: ["changed"], 31 | style: { 32 | color: "rgb(255, 184, 108)", 33 | }, 34 | }, 35 | { 36 | types: ["punctuation", "symbol"], 37 | style: { 38 | color: "rgb(248, 248, 242)", 39 | }, 40 | }, 41 | { 42 | types: ["string", "char", "tag", "selector"], 43 | style: { 44 | color: "rgb(96, 238, 164)", 45 | }, 46 | }, 47 | { 48 | types: ["keyword", "variable"], 49 | style: { 50 | color: "rgb(183, 168, 255)", 51 | }, 52 | }, 53 | { 54 | types: ["comment"], 55 | style: { 56 | color: "rgb(98, 114, 164)", 57 | }, 58 | }, 59 | { 60 | types: ["attr-name"], 61 | style: { 62 | color: "rgb(241, 250, 140)", 63 | }, 64 | }, 65 | ], 66 | }; 67 | 68 | module.exports = theme; 69 | -------------------------------------------------------------------------------- /documentation/static/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/documentation/static/.nojekyll -------------------------------------------------------------------------------- /documentation/static/img/FAQ-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/documentation/static/img/FAQ-1.png -------------------------------------------------------------------------------- /documentation/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/documentation/static/img/favicon.ico -------------------------------------------------------------------------------- /documentation/static/img/opal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/documentation/static/img/opal.png -------------------------------------------------------------------------------- /documentation/static/img/opal_plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/documentation/static/img/opal_plus.png -------------------------------------------------------------------------------- /packages/__packaging__.py: -------------------------------------------------------------------------------- 1 | """ 2 | OPAL - Open Policy Administration Layer 3 | 4 | OPAL is an administration layer for Open Policy Agent (OPA). It automatically discovers 5 | changes to your authorization policies and pushes live updates to your policy agents. 6 | 7 | Project homepage: https://github.com/permitio/opal 8 | """ 9 | 10 | import os 11 | 12 | VERSION = (0, 0, 0) # Placeholder, to be set by CI/CD 13 | VERSION_STRING = ".".join(map(str, VERSION)) 14 | 15 | __version__ = VERSION_STRING 16 | __author__ = "Or Weis, Asaf Cohen" 17 | __author_email__ = "or@permit.io" 18 | __license__ = "Apache 2.0" 19 | __copyright__ = "Copyright 2021 Or Weis and Asaf Cohen" 20 | 21 | 22 | def get_install_requires(here): 23 | """Gets the contents of install_requires from text file. 24 | 25 | Getting the minimum requirements from a text file allows us to pre- 26 | install them in docker, speeding up our docker builds and better 27 | utilizing the docker layer cache. 28 | 29 | The requirements in requires.txt are in fact the minimum set of 30 | packages you need to run OPAL (and are thus different from a 31 | "requirements.txt" file). 32 | """ 33 | with open(os.path.join(here, "requires.txt")) as fp: 34 | return [ 35 | line.strip() for line in fp.read().splitlines() if not line.startswith("#") 36 | ] 37 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/__init__.py: -------------------------------------------------------------------------------- 1 | from opal_client.client import OpalClient 2 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/callbacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-client/opal_client/callbacks/__init__.py -------------------------------------------------------------------------------- /packages/opal-client/opal_client/cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import typer 5 | from fastapi.applications import FastAPI 6 | from typer.main import Typer 7 | from typer.models import Context 8 | 9 | # Add parent path to use local src as package for tests 10 | root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) 11 | sys.path.append(root_dir) 12 | 13 | from opal_client.config import opal_client_config 14 | from opal_common.cli.docs import MainTexts 15 | from opal_common.cli.typer_app import get_typer_app 16 | from opal_common.config import opal_common_config 17 | 18 | app = get_typer_app() 19 | 20 | 21 | @app.command() 22 | def run(engine_type: str = typer.Option("uvicron", help="uvicorn or gunicorn")): 23 | """Run the client as a daemon.""" 24 | typer.echo(f"-- Starting OPAL client (with {engine_type}) --") 25 | from opal_common.corn_utils import run_gunicorn, run_uvicorn 26 | 27 | if engine_type == "gunicorn": 28 | app: FastAPI 29 | from opal_client.main import app 30 | 31 | run_gunicorn( 32 | app, 33 | opal_client_config.CLIENT_API_SERVER_WORKER_COUNT, 34 | host=opal_client_config.CLIENT_API_SERVER_HOST, 35 | port=opal_client_config.CLIENT_API_SERVER_PORT, 36 | ) 37 | else: 38 | run_uvicorn( 39 | "opal_client.main:app", 40 | workers=opal_client_config.CLIENT_API_SERVER_WORKER_COUNT, 41 | host=opal_client_config.CLIENT_API_SERVER_HOST, 42 | port=opal_client_config.CLIENT_API_SERVER_PORT, 43 | ) 44 | 45 | 46 | @app.command() 47 | def print_config(): 48 | """To test config values, print the configuration parsed from ENV and 49 | CMD.""" 50 | typer.echo("Printing configuration values") 51 | typer.echo(str(opal_client_config)) 52 | typer.echo(str(opal_common_config)) 53 | 54 | 55 | def cli(): 56 | main_texts = MainTexts("OPAL-CLIENT", "client") 57 | 58 | def on_start(ctx: Context, **kwargs): 59 | if ctx.invoked_subcommand is None or ctx.invoked_subcommand == "run": 60 | typer.secho(main_texts.header, bold=True, fg=typer.colors.MAGENTA) 61 | if ctx.invoked_subcommand is None: 62 | typer.echo(ctx.get_usage()) 63 | typer.echo(main_texts.docs) 64 | 65 | opal_client_config.cli( 66 | [opal_common_config], 67 | typer_app=app, 68 | help=main_texts.docs, 69 | on_start=on_start, 70 | ) 71 | 72 | 73 | if __name__ == "__main__": 74 | cli() 75 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-client/opal_client/data/__init__.py -------------------------------------------------------------------------------- /packages/opal-client/opal_client/data/api.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from fastapi import APIRouter, HTTPException, status 4 | from opal_client.data.updater import DataUpdater 5 | from opal_common.logger import logger 6 | 7 | 8 | def init_data_router(data_updater: Optional[DataUpdater]): 9 | router = APIRouter() 10 | 11 | @router.post("/data-updater/trigger", status_code=status.HTTP_200_OK) 12 | async def trigger_policy_data_update(): 13 | logger.info("triggered policy data update from api") 14 | if data_updater: 15 | await data_updater.get_base_policy_data( 16 | data_fetch_reason="request from sdk" 17 | ) 18 | return {"status": "ok"} 19 | else: 20 | raise HTTPException( 21 | status_code=status.HTTP_503_SERVICE_UNAVAILABLE, 22 | detail="Data Updater is currently disabled. Dynamic data updates are not available.", 23 | ) 24 | 25 | return router 26 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/data/rpc.py: -------------------------------------------------------------------------------- 1 | from fastapi_websocket_pubsub.rpc_event_methods import RpcEventClientMethods 2 | from opal_client.logger import logger 3 | 4 | 5 | class TenantAwareRpcEventClientMethods(RpcEventClientMethods): 6 | """Use this methods class when the server uses 7 | `TenantAwareRpcEventServerMethods`.""" 8 | 9 | TOPIC_SEPARATOR = "::" 10 | 11 | async def notify(self, subscription=None, data=None): 12 | topic = subscription["topic"] 13 | logger.info( 14 | "Received notification of event: {topic}", 15 | topic=topic, 16 | subscription=subscription, 17 | data=data, 18 | ) 19 | if self.TOPIC_SEPARATOR in topic: 20 | topic_parts = topic.split(self.TOPIC_SEPARATOR) 21 | if len(topic_parts) > 1: 22 | topic = topic_parts[1] # index 0 holds the app id 23 | await self.client.trigger_topic(topic=topic, data=data) 24 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/engine/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-client/opal_client/engine/__init__.py -------------------------------------------------------------------------------- /packages/opal-client/opal_client/engine/healthcheck/example-transaction.json: -------------------------------------------------------------------------------- 1 | { 2 | "system": { 3 | "opal": { 4 | "transactions": [ 5 | { 6 | "id": "674bd92f62420242d6fb5b5d1e0c831f29f6fe27", 7 | "actions": [ 8 | "set_policies" 9 | ], 10 | "success": false 11 | }, 12 | { 13 | "id": "9a20ca2e08ce379497142310cb8bbf64f9879271", 14 | "actions": [ 15 | "set_policies" 16 | ], 17 | "success": true 18 | }, 19 | { 20 | "id": "83e97b8d08f142c294460cf4dd64a544", 21 | "actions": [ 22 | "set_policy_data" 23 | ], 24 | "success": true 25 | }, 26 | { 27 | "id": "83e97b8d08f142c294460c555d64a544", 28 | "actions": [ 29 | "set_policy_data" 30 | ], 31 | "success": true 32 | } 33 | ] 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/engine/healthcheck/opal.rego: -------------------------------------------------------------------------------- 1 | package system.opal 2 | 3 | # ----------------------------------------------------------------------------------- 4 | # Ready rule - opal successfully loaded at least one policy bundle and data update 5 | # ----------------------------------------------------------------------------------- 6 | default ready = {ready} 7 | 8 | # ----------------------------------------------------------------------------------- 9 | # Healthy rule - the last policy-write and data-write transactions were successful. 10 | # 11 | # Note: 12 | # At the moment we make an (inaccurate but simplified) assumption that successful 13 | # transactions reset the bad state (going out of sync) caused by failed transactions. 14 | # ----------------------------------------------------------------------------------- 15 | default healthy = {healthy} 16 | 17 | last_policy_transaction := {last_policy_transaction} 18 | last_data_transaction := {last_data_transaction} 19 | last_failed_policy_transaction := {last_failed_policy_transaction} 20 | last_failed_data_transaction := {last_failed_data_transaction} 21 | 22 | transaction_data_statistics := {transaction_data_statistics} 23 | transaction_policy_statistics := {transaction_policy_statistics} 24 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/limiter.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | from fastapi import HTTPException, status 3 | from opal_client.config import opal_client_config 4 | from opal_client.logger import logger 5 | from opal_common.security.sslcontext import get_custom_ssl_context 6 | from opal_common.utils import get_authorization_header, tuple_to_dict 7 | from tenacity import retry, stop, wait_random_exponential 8 | 9 | 10 | class StartupLoadLimiter: 11 | """Validates OPAL server is not too loaded before starting up.""" 12 | 13 | def __init__(self, backend_url=None, token=None): 14 | """ 15 | Args: 16 | backend_url (str): Defaults to opal_client_config.SERVER_URL. 17 | token ([type], optional): [description]. Defaults to opal_client_config.CLIENT_TOKEN. 18 | """ 19 | self._backend_url = backend_url or opal_client_config.SERVER_URL 20 | self._loadlimit_endpoint_url = f"{self._backend_url}/loadlimit" 21 | 22 | self._token = token or opal_client_config.CLIENT_TOKEN 23 | self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) 24 | self._custom_ssl_context = get_custom_ssl_context() 25 | self._ssl_context_kwargs = ( 26 | {"ssl": self._custom_ssl_context} 27 | if self._custom_ssl_context is not None 28 | else {} 29 | ) 30 | 31 | @retry(wait=wait_random_exponential(max=10), stop=stop.stop_never, reraise=True) 32 | async def wait_for_server_ready(self): 33 | logger.info("Trying to get server's load limit pass") 34 | async with aiohttp.ClientSession() as session: 35 | try: 36 | async with session.get( 37 | self._loadlimit_endpoint_url, 38 | headers={"content-type": "text/plain", **self._auth_headers}, 39 | **self._ssl_context_kwargs, 40 | ) as response: 41 | if response.status != status.HTTP_200_OK: 42 | logger.warning( 43 | f"loadlimit endpoint returned status {response.status}" 44 | ) 45 | raise HTTPException(response.status) 46 | except aiohttp.ClientError as e: 47 | logger.warning("server connection error: {err}", err=repr(e)) 48 | raise 49 | 50 | def __call__(self): 51 | return self.wait_for_server_ready() 52 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/logger.py: -------------------------------------------------------------------------------- 1 | from opal_common.logger import * 2 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/main.py: -------------------------------------------------------------------------------- 1 | from opal_client.client import OpalClient 2 | 3 | client = OpalClient() 4 | # expose app for Uvicorn 5 | app = client.app 6 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/policy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-client/opal_client/policy/__init__.py -------------------------------------------------------------------------------- /packages/opal-client/opal_client/policy/api.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, status 2 | from opal_client.policy.updater import PolicyUpdater 3 | from opal_common.logger import logger 4 | 5 | 6 | def init_policy_router(policy_updater: PolicyUpdater): 7 | router = APIRouter() 8 | 9 | @router.post("/policy-updater/trigger", status_code=status.HTTP_200_OK) 10 | async def trigger_policy_update(): 11 | logger.info("triggered policy update from api") 12 | await policy_updater.trigger_update_policy(force_full_update=True) 13 | return {"status": "ok"} 14 | 15 | return router 16 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/policy/options.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional 3 | 4 | from pydantic import BaseModel, Field 5 | from tenacity import ( 6 | _utils, 7 | stop_after_attempt, 8 | wait_exponential, 9 | wait_fixed, 10 | wait_random_exponential, 11 | ) 12 | 13 | 14 | class WaitStrategy(str, Enum): 15 | # Fixed-time waiting between each retry (see https://tenacity.readthedocs.io/en/latest/api.html#tenacity.wait.wait_fixed) 16 | fixed = "fixed" 17 | # Exponential backoff (see https://tenacity.readthedocs.io/en/latest/api.html#tenacity.wait.wait_exponential) 18 | exponential = "exponential" 19 | # Exponential backoff randomized (see https://tenacity.readthedocs.io/en/latest/api.html#tenacity.wait.wait_random_exponential) 20 | random_exponential = "random_exponential" 21 | 22 | 23 | class ConnRetryOptions(BaseModel): 24 | wait_strategy: WaitStrategy = Field( 25 | WaitStrategy.fixed, 26 | description="waiting strategy (e.g. fixed for fixed-time waiting, exponential for exponential back-off) (default fixed)", 27 | ) 28 | wait_time: float = Field( 29 | 2, 30 | description="waiting time in seconds (semantic depends on the waiting strategy) (default 2)", 31 | ) 32 | attempts: int = Field(2, description="number of attempts (default 2)") 33 | max_wait: float = Field( 34 | _utils.MAX_WAIT, 35 | description="max time to wait in total (for exponential strategies only)", 36 | ) 37 | 38 | def toTenacityConfig(self): 39 | if self.wait_strategy == WaitStrategy.exponential: 40 | wait = wait_exponential(multiplier=self.wait_time, max=self.max_wait) 41 | elif self.wait_strategy == WaitStrategy.random_exponential: 42 | wait = wait_random_exponential(multiplier=self.wait_time, max=self.max_wait) 43 | else: 44 | wait = wait_fixed(self.wait_time) 45 | return dict(wait=wait, stop=stop_after_attempt(self.attempts)) 46 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/policy/topics.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List 3 | 4 | from opal_client.config import opal_client_config 5 | from opal_common.paths import PathUtils 6 | 7 | 8 | def default_subscribed_policy_directories() -> List[str]: 9 | """Wraps the configured value of POLICY_SUBSCRIPTION_DIRS, but dedups 10 | intersecting dirs.""" 11 | subscription_directories = [ 12 | Path(d) for d in opal_client_config.POLICY_SUBSCRIPTION_DIRS 13 | ] 14 | non_intersecting_directories = PathUtils.non_intersecting_directories( 15 | subscription_directories 16 | ) 17 | return [str(directory) for directory in non_intersecting_directories] 18 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/policy_store/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-client/opal_client/policy_store/__init__.py -------------------------------------------------------------------------------- /packages/opal-client/opal_client/policy_store/api.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends 2 | from opal_client.config import opal_client_config 3 | from opal_client.policy_store.schemas import PolicyStoreAuth, PolicyStoreDetails 4 | from opal_common.authentication.authz import require_peer_type 5 | from opal_common.authentication.deps import JWTAuthenticator 6 | from opal_common.authentication.types import JWTClaims 7 | from opal_common.authentication.verifier import Unauthorized 8 | from opal_common.logger import logger 9 | from opal_common.schemas.security import PeerType 10 | 11 | 12 | def init_policy_store_router(authenticator: JWTAuthenticator): 13 | router = APIRouter() 14 | 15 | @router.get( 16 | "/policy-store/config", 17 | response_model=PolicyStoreDetails, 18 | response_model_exclude_none=True, 19 | # Deprecating this route 20 | deprecated=True, 21 | ) 22 | async def get_policy_store_details(claims: JWTClaims = Depends(authenticator)): 23 | try: 24 | require_peer_type( 25 | authenticator, claims, PeerType.listener 26 | ) # may throw Unauthorized 27 | except Unauthorized as e: 28 | logger.error(f"Unauthorized to publish update: {repr(e)}") 29 | raise 30 | 31 | token = None 32 | oauth_client_secret = None 33 | if not opal_client_config.EXCLUDE_POLICY_STORE_SECRETS: 34 | token = opal_client_config.POLICY_STORE_AUTH_TOKEN 35 | oauth_client_secret = ( 36 | opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET 37 | ) 38 | return PolicyStoreDetails( 39 | url=opal_client_config.POLICY_STORE_URL, 40 | token=token or None, 41 | auth_type=opal_client_config.POLICY_STORE_AUTH_TYPE or PolicyStoreAuth.NONE, 42 | oauth_client_id=opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_ID 43 | or None, 44 | oauth_client_secret=oauth_client_secret or None, 45 | oauth_server=opal_client_config.POLICY_STORE_AUTH_OAUTH_SERVER or None, 46 | ) 47 | 48 | return router 49 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/policy_store/schemas.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional 3 | 4 | from pydantic import BaseModel, Field, validator 5 | 6 | 7 | class PolicyStoreTypes(Enum): 8 | OPA = "OPA" 9 | CEDAR = "CEDAR" 10 | MOCK = "MOCK" 11 | 12 | 13 | class PolicyStoreAuth(Enum): 14 | NONE = "none" 15 | TOKEN = "token" 16 | OAUTH = "oauth" 17 | TLS = "tls" 18 | 19 | 20 | class PolicyStoreDetails(BaseModel): 21 | """ 22 | represents a policy store endpoint - contains the policy store's: 23 | - location (url) 24 | - type 25 | - credentials 26 | """ 27 | 28 | type: PolicyStoreTypes = Field( 29 | PolicyStoreTypes.OPA, 30 | description="the type of policy store, currently only OPA is officially supported", 31 | ) 32 | url: str = Field( 33 | ..., 34 | description="the url that OPA can be found in. if localhost is the host - " 35 | "it means OPA is on the same hostname as OPAL client.", 36 | ) 37 | token: Optional[str] = Field( 38 | None, description="optional access token required by the policy store" 39 | ) 40 | 41 | auth_type: PolicyStoreAuth = Field( 42 | PolicyStoreAuth.NONE, 43 | description="the type of authentication is supported for the policy store.", 44 | ) 45 | 46 | oauth_client_id: Optional[str] = Field( 47 | None, description="optional OAuth client id required by the policy store" 48 | ) 49 | oauth_client_secret: Optional[str] = Field( 50 | None, description="optional OAuth client secret required by the policy store" 51 | ) 52 | oauth_server: Optional[str] = Field( 53 | None, description="optional OAuth server required by the policy store" 54 | ) 55 | 56 | @validator("type") 57 | def force_enum(cls, v): 58 | if isinstance(v, str): 59 | return PolicyStoreTypes(v) 60 | if isinstance(v, PolicyStoreTypes): 61 | return v 62 | raise ValueError(f"invalid value: {v}") 63 | 64 | class Config: 65 | use_enum_values = True 66 | allow_population_by_field_name = True 67 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-client/opal_client/tests/__init__.py -------------------------------------------------------------------------------- /packages/opal-client/opal_client/tests/engine_runner_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from opal_client.engine.options import AuthenticationScheme, CedarServerOptions 3 | 4 | 5 | @pytest.mark.parametrize( 6 | "options,expected", 7 | [ 8 | (CedarServerOptions(), "--addr 0.0.0.0 --port 8180"), 9 | (CedarServerOptions(addr=":1234"), "--addr 0.0.0.0 --port 1234"), 10 | (CedarServerOptions(addr="1.2.3.4:1234"), "--addr 1.2.3.4 --port 1234"), 11 | ( 12 | CedarServerOptions( 13 | authentication=AuthenticationScheme.token, 14 | authentication_token="mytoken", 15 | ), 16 | "-a mytoken --addr 0.0.0.0 --port 8180", 17 | ), 18 | ], 19 | ) 20 | def test_cedar_arguments(options: CedarServerOptions, expected: str): 21 | expected_args = expected.split(" ") 22 | assert list(options.get_args()) == expected_args 23 | -------------------------------------------------------------------------------- /packages/opal-client/opal_client/utils.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | from fastapi import Response 3 | from fastapi.encoders import jsonable_encoder 4 | 5 | 6 | async def proxy_response(response: aiohttp.ClientResponse) -> Response: 7 | content = await response.text() 8 | return Response( 9 | content=content, 10 | status_code=response.status, 11 | headers=dict(response.headers), 12 | media_type="application/json", 13 | ) 14 | 15 | 16 | def exclude_none_fields(data): 17 | # remove default values from the pydatic model with a None value and also 18 | # convert the model to a valid JSON serializable type using jsonable_encoder 19 | return jsonable_encoder(data, exclude_none=True) 20 | -------------------------------------------------------------------------------- /packages/opal-client/requires.txt: -------------------------------------------------------------------------------- 1 | aiofiles>=0.8.0,<1 2 | aiohttp>=3.9.2,<4 3 | psutil>=5.9.1,<6 4 | tenacity>=8.0.1,<9 5 | dpath>=2.1.5,<3 6 | jsonpatch>=1.33,<2 7 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/authentication/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/authentication/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/authentication/authz.py: -------------------------------------------------------------------------------- 1 | from opal_common.authentication.deps import JWTAuthenticator 2 | from opal_common.authentication.types import JWTClaims 3 | from opal_common.authentication.verifier import Unauthorized 4 | from opal_common.schemas.data import DataUpdate 5 | from opal_common.schemas.security import PeerType 6 | 7 | 8 | def require_peer_type( 9 | authenticator: JWTAuthenticator, claims: JWTClaims, required_type: PeerType 10 | ): 11 | if not authenticator.enabled: 12 | return 13 | 14 | peer_type = claims.get("peer_type", None) 15 | if peer_type is None: 16 | raise Unauthorized(description="Missing 'peer_type' claim for OPAL jwt token") 17 | try: 18 | type = PeerType(peer_type) 19 | except ValueError: 20 | raise Unauthorized( 21 | description=f"Invalid 'peer_type' claim for OPAL jwt token: {peer_type}" 22 | ) 23 | 24 | if type != required_type: 25 | raise Unauthorized( 26 | description=f"Incorrect 'peer_type' claim for OPAL jwt token: {str(type)}, expected: {str(required_type)}" 27 | ) 28 | 29 | 30 | def restrict_optional_topics_to_publish( 31 | authenticator: JWTAuthenticator, claims: JWTClaims, update: DataUpdate 32 | ): 33 | if not authenticator.enabled: 34 | return 35 | 36 | if "permitted_topics" not in claims: 37 | return 38 | 39 | for entry in update.entries: 40 | unauthorized_topics = set(entry.topics).difference(claims["permitted_topics"]) 41 | if unauthorized_topics: 42 | raise Unauthorized( 43 | description=f"Invalid 'topics' to publish {unauthorized_topics}" 44 | ) 45 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/authentication/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/authentication/tests/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/authentication/types.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Any, Dict 3 | 4 | from cryptography.hazmat.primitives.asymmetric.types import ( 5 | PrivateKeyTypes, 6 | PublicKeyTypes, 7 | ) 8 | from jwt.algorithms import get_default_algorithms 9 | 10 | # custom types 11 | PrivateKey = PrivateKeyTypes 12 | PublicKey = PublicKeyTypes 13 | JWTClaims = Dict[str, Any] 14 | 15 | 16 | class EncryptionKeyFormat(str, Enum): 17 | """Represent the supported formats for storing encryption keys. 18 | 19 | - PEM (https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) 20 | - SSH (RFC4716) or short format (RFC4253, section-6.6, explained here: https://coolaj86.com/articles/the-ssh-public-key-format/) 21 | - DER (https://en.wikipedia.org/wiki/X.690#DER_encoding) 22 | """ 23 | 24 | pem = "pem" 25 | ssh = "ssh" 26 | der = "der" 27 | 28 | 29 | # dynamic enum because pyjwt does not define one 30 | # see: https://pyjwt.readthedocs.io/en/stable/algorithms.html for possible values 31 | JWTAlgorithm = Enum("JWTAlgorithm", [(k, k) for k in get_default_algorithms().keys()]) 32 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/cli/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/cli/docs.py: -------------------------------------------------------------------------------- 1 | class MainTexts: 2 | def __init__(self, first_line, name): 3 | self.header = f"""\b 4 | {first_line} 5 | Open-Policy Administration Layer - {name}\b\f""" 6 | 7 | self.docs = f"""\b 8 | Config top level options: 9 | - Use env-vars (same as cmd options) but uppercase 10 | and with "_" instead of "-"; all prefixed with "OPAL_" 11 | - Use command line options as detailed by '--help' 12 | - Use .env or .ini files 13 | 14 | \b 15 | Examples: 16 | - opal-{name} --help Detailed help on CLI 17 | - opal-{name} run --help Help on run command 18 | - opal-{name} run --engine-type gunicorn Run {name} with gunicorn 19 | \b 20 | """ 21 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/cli/typer_app.py: -------------------------------------------------------------------------------- 1 | import typer 2 | from opal_common.cli.commands import all_commands 3 | 4 | 5 | def get_typer_app(): 6 | app = typer.Typer() 7 | for cmd in all_commands: 8 | app.command()(cmd) 9 | return app 10 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/confi/__init__.py: -------------------------------------------------------------------------------- 1 | from opal_common.confi.confi import * 2 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/confi/cli.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Dict, List 2 | 3 | import click 4 | import typer 5 | from opal_common.confi.types import ConfiEntry 6 | from typer.main import Typer 7 | 8 | 9 | def create_click_cli(confi_entries: Dict[str, ConfiEntry], callback: Callable): 10 | cli = callback 11 | for key, entry in confi_entries.items(): 12 | option_kwargs = entry.get_cli_option_kwargs() 13 | # make the key fit cmd-style (i.e. kebab-case) 14 | adjusted_key = entry.key.lower().replace("_", "-") 15 | keys = [f"--{adjusted_key}", entry.key] 16 | # add flag if given (i.e '-t' option) 17 | if entry.flags is not None: 18 | keys.extend(entry.flags) 19 | # use lower case as the key, and as is (no prefix, and no case altering) as the name 20 | # see https://click.palletsprojects.com/en/7.x/options/#name-your-options 21 | cli = click.option(*keys, **option_kwargs)(cli) 22 | # pass context 23 | cli = click.pass_context(cli) 24 | # wrap in group 25 | cli = click.group(invoke_without_command=True)(cli) 26 | return cli 27 | 28 | 29 | def get_cli_object_for_config_objects( 30 | config_objects: list, 31 | typer_app: Typer = None, 32 | help: str = None, 33 | on_start: Callable = None, 34 | ): 35 | # callback to save CLI results back to objects 36 | def callback(ctx, **kwargs): 37 | if callable(on_start): 38 | on_start(ctx, **kwargs) 39 | 40 | for key, value in kwargs.items(): 41 | # find the confi-object which the key belongs to and ... 42 | for config_obj in config_objects: 43 | if key in config_obj.entries: 44 | # ... update that object with the new value 45 | setattr(config_obj, key, value) 46 | config_obj._entries[key].value = value 47 | 48 | if help is not None: 49 | callback.__doc__ = help 50 | # Create a merged config-entires map 51 | entries = {} 52 | for config_obj in config_objects: 53 | entries.update(config_obj.entries) 54 | # convert to a click-cli group 55 | click_group = create_click_cli(entries, callback) 56 | # add the typer app into our click group 57 | if typer_app is not None: 58 | typer_click_object = typer.main.get_command(typer_app) 59 | # add the app commands directly to out click app 60 | for name, cmd in typer_click_object.commands.items(): 61 | click_group.add_command(cmd, name) 62 | return click_group 63 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/corn_utils.py: -------------------------------------------------------------------------------- 1 | """Utilities to run UVICORN / GUNICORN.""" 2 | import multiprocessing 3 | from typing import Dict 4 | 5 | import gunicorn.app.base 6 | 7 | 8 | def calc_default_number_of_workers(): 9 | return (multiprocessing.cpu_count() * 2) + 1 10 | 11 | 12 | class GunicornApp(gunicorn.app.base.BaseApplication): 13 | def __init__(self, app, options: Dict[str, str] = None): 14 | self.options = options or {} 15 | self.application = app 16 | super().__init__() 17 | 18 | def load_config(self): 19 | config = { 20 | key: value 21 | for key, value in self.options.items() 22 | if key in self.cfg.settings and value is not None 23 | } 24 | for key, value in config.items(): 25 | self.cfg.set(key.lower(), value) 26 | 27 | def load(self): 28 | return self.application 29 | 30 | 31 | def run_gunicorn(app, number_of_workers=None, host=None, port=None, **kwargs): 32 | options = { 33 | "bind": "%s:%s" % (host or "127.0.0.1", port or "8080"), 34 | "workers": number_of_workers or calc_default_number_of_workers(), 35 | "worker_class": "uvicorn.workers.UvicornWorker", 36 | } 37 | options.update(kwargs) 38 | GunicornApp(app, options).run() 39 | 40 | 41 | def run_uvicorn( 42 | app_path, number_of_workers=None, host=None, port=None, reload=False, **kwargs 43 | ): 44 | options = { 45 | "host": host or "127.0.0.1", 46 | "port": port or "8080", 47 | "reload": reload, 48 | "workers": number_of_workers or calc_default_number_of_workers(), 49 | } 50 | options.update(kwargs) 51 | import uvicorn 52 | 53 | uvicorn.run(app_path, **options) 54 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/__init__.py: -------------------------------------------------------------------------------- 1 | from opal_common.engine.parsing import get_rego_package 2 | from opal_common.engine.paths import is_data_module, is_policy_module 3 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/parsing.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Optional 3 | 4 | # This regex matches the package declaration at the top of a valid .rego file 5 | REGO_PACKAGE_DECLARATION = re.compile(r"^package\s+([a-zA-Z0-9\.\"\[\]]+)$") 6 | 7 | 8 | def get_rego_package(contents: str) -> Optional[str]: 9 | """Try to parse the package name from rego file contents. 10 | 11 | return None if failed to parse (probably invalid .rego file) 12 | """ 13 | lines = contents.splitlines() 14 | for line in lines: 15 | match = REGO_PACKAGE_DECLARATION.match(line) 16 | if match is not None: 17 | return match.group(1) 18 | return None 19 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/paths.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from opal_common.config import opal_common_config 4 | 5 | 6 | def is_data_module(path: Path) -> bool: 7 | """Only json files named `data.json` can be included in official OPA 8 | bundles as static data files. 9 | 10 | checks if a given path points to such file. 11 | """ 12 | return path.name == "data.json" 13 | 14 | 15 | def is_policy_module(path: Path) -> bool: 16 | """Checks if a given path points to a rego file (extension == .rego). 17 | 18 | Only rego files are allowed in official OPA bundles as policy files. 19 | """ 20 | return path.suffix in opal_common_config.POLICY_REPO_POLICY_EXTENSIONS 21 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/engine/py.typed -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/tests/fixtures/invalid-package.rego: -------------------------------------------------------------------------------- 1 | # URL Extraction 2 | # -------------- 3 | # 4 | # This example allows users to read their own profiles. This example shows how to: 5 | # 6 | # * Perform pattern matching on JSON values in Rego. 7 | # * Use Rego built-in functions to parse base64 encoded strings. 8 | # * Use parsed inputs provided by the OPA-Istio/Envoy integration. 9 | # 10 | # For more information see: 11 | # 12 | # * Rego Cheat Sheet: https://www.openpolicyagent.org/docs/latest/policy-cheatsheet/ 13 | # * Rego Built-in Functions: https://www.openpolicyagent.org/docs/latest/policy-reference/ 14 | # * Rego Unification: https://www.openpolicyagent.org/docs/latest/policy-language/#unification 15 | # * OPA-Istio/Envoy Integration: https://github.com/open-policy-agent/opa-envoy-plugin 16 | 17 | # modified the package name with invalid character 18 | # (the "=" char) on purpose for testing purposes 19 | package envoy.http.urlextract= 20 | 21 | default allow = false 22 | 23 | allow { 24 | # The `some` keyword declares local variables. This example declares a local 25 | # variable called `user_name` (used below). 26 | some user_name 27 | 28 | input.attributes.request.http.method == "GET" 29 | 30 | # The `=` operator in Rego performs pattern matching/unification. OPA finds 31 | # variable assignments that satisfy this expression (as well as all of the other 32 | # expressions in the same rule.) 33 | input.parsed_path = ["users", "profile", user_name] 34 | 35 | # Check if the `user_name` from path is the same as the username from the 36 | # credentials. 37 | user_name == basic_auth.user_name 38 | } 39 | 40 | basic_auth := {"user_name": user_name, "password": password} { 41 | v := input.attributes.request.http.headers.authorization 42 | startswith(v, "Basic ") 43 | s := substring(v, count("Basic "), -1) 44 | [user_name, password] := split(base64url.decode(s), ":") 45 | } 46 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/tests/fixtures/jwt.rego: -------------------------------------------------------------------------------- 1 | # JWT Decoding 2 | # ------------ 3 | # 4 | # The example allows a user "alice" to create new dogs in a 'pet store' API. 5 | # 6 | # This example show show to: 7 | # 8 | # * Extract and decode a JSON Web Token (JWT). 9 | # * Verify signatures on JWT using built-in functions in Rego. 10 | # * Define helper rules that provide useful abstractions. 11 | # 12 | # For more information see: 13 | # 14 | # * Rego JWT decoding and verification functions: https://www.openpolicyagent.org/docs/latest/policy-reference/#token-verification 15 | # 16 | # Hint: When you click Evaluate, you see values for `allow`, `is_post`, `is_dogs`, 17 | # `claims` and `bearer_token` because by default the playground evaluates all of 18 | # the rules in the current package. You can evaluate specific rules by selecting 19 | # the rule name (e.g., `claims`) and clicking Evaluate Selection. 20 | package envoy.http.jwt 21 | 22 | default allow = false 23 | 24 | allow { 25 | is_post 26 | is_dogs 27 | claims.username == "alice" 28 | } 29 | 30 | is_post { 31 | input.attributes.request.http.method == "POST" 32 | } 33 | 34 | is_dogs { 35 | input.attributes.request.http.path == "/pets/dogs" 36 | } 37 | 38 | claims := payload { 39 | # Verify the signature on the Bearer token. In this example the secret is 40 | # hardcoded into the policy however it could also be loaded via data or 41 | # an environment variable. Environment variables can be accessed using 42 | # the `opa.runtime()` built-in function. 43 | io.jwt.verify_hs256(bearer_token, "B41BD5F462719C6D6118E673A2389") 44 | 45 | # This statement invokes the built-in function `io.jwt.decode` passing the 46 | # parsed bearer_token as a parameter. The `io.jwt.decode` function returns an 47 | # array: 48 | # 49 | # [header, payload, signature] 50 | # 51 | # In Rego, you can pattern match values using the `=` and `:=` operators. This 52 | # example pattern matches on the result to obtain the JWT payload. 53 | [_, payload, _] := io.jwt.decode(bearer_token) 54 | } 55 | 56 | bearer_token := t { 57 | # Bearer tokens are contained inside of the HTTP Authorization header. This rule 58 | # parses the header and extracts the Bearer token value. If no Bearer token is 59 | # provided, the `bearer_token` value is undefined. 60 | v := input.attributes.request.http.headers.authorization 61 | startswith(v, "Bearer ") 62 | t := substring(v, count("Bearer "), -1) 63 | } 64 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/tests/fixtures/no-package.rego: -------------------------------------------------------------------------------- 1 | # Hello World 2 | # ----------- 3 | # 4 | # This example grants public HTTP access to "/", full access to "charlie", and 5 | # blocks everything else. This example shows how to: 6 | # 7 | # * Construct a simple whitelist/deny-by-default HTTP API authorization policy. 8 | # * Refer to the data sent by Envoy in External Authorization messages. 9 | # 10 | # For more information see: 11 | # 12 | # * Rego Rules: https://www.openpolicyagent.org/docs/latest/#rules 13 | # * Envoy External Authorization: https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto 14 | 15 | # removed the package name on purpose for testing purposes (was: package envoy.http.public) 16 | 17 | # If neither of the rules below match, `allow` is `false`. 18 | default allow = false 19 | 20 | # `allow` is a "rule". The simplest kind of rules in Rego are "if-then" statements 21 | # that assign a single value to a variable. If the value is omitted, it defaults to `true`. 22 | # In other words, this rule is equivalent to: 23 | # 24 | # allow = true { 25 | # input.attributes.request.http.method == "GET" 26 | # input.attributes.request.http.path == "/" 27 | # } 28 | # 29 | # Since statements like `X = true { ... }` are so common, Rego lets you omit the `= true` bit. 30 | # 31 | # This rule says (in English): 32 | # 33 | # allow is true if... 34 | # method is "GET", and... 35 | # path is "/" 36 | # 37 | # The statements in the body of the rule are AND-ed together. 38 | allow { 39 | input.attributes.request.http.method == "GET" 40 | input.attributes.request.http.path == "/" 41 | } 42 | 43 | # In Rego, logical OR is expressed by defining multiple rules with the same name. 44 | # 45 | # This rule says (in English): 46 | # 47 | # allow is true if... 48 | # authorization is "Basic charlie" 49 | allow { 50 | input.attributes.request.http.headers.authorization == "Basic charlie" 51 | } 52 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/tests/fixtures/play.rego: -------------------------------------------------------------------------------- 1 | package play 2 | 3 | # Welcome to the Rego playground! Rego (pronounced "ray-go") is OPA's policy language. 4 | # 5 | # Try it out: 6 | # 7 | # 1. Click Evaluate. Note: 'hello' is 'true' 8 | # 2. Change "world" to "hello" in the INPUT panel. Click Evaluate. Note: 'hello' is 'false' 9 | # 3. Change "world" to "hello" on line 25 in the editor. Click Evaluate. Note: 'hello' is 'true' 10 | # 11 | # Features: 12 | # 13 | # Examples browse a collection of example policies 14 | # Coverage view the policy statements that were executed 15 | # Evaluate execute the policy with INPUT and DATA 16 | # Publish share your playground and experiment with local deployment 17 | # INPUT edit the JSON value your policy sees under the 'input' global variable 18 | # (resize) DATA edit the JSON value your policy sees under the 'data' global variable 19 | # OUTPUT view the result of policy execution 20 | 21 | default hello = false 22 | 23 | hello { 24 | m := input.message 25 | m == "world" 26 | } 27 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/tests/fixtures/rbac.rego: -------------------------------------------------------------------------------- 1 | # Role-based Access Control (RBAC) 2 | # -------------------------------- 3 | # 4 | # This example defines an RBAC model for a Pet Store API. The Pet Store API allows 5 | # users to look at pets, adopt them, update their stats, and so on. The policy 6 | # controls which users can perform actions on which resources. The policy implements 7 | # a classic Role-based Access Control model where users are assigned to roles and 8 | # roles are granted the ability to perform some action(s) on some type of resource. 9 | # 10 | # This example shows how to: 11 | # 12 | # * Define an RBAC model in Rego that interprets role mappings represented in JSON. 13 | # * Iterate/search across JSON data structures (e.g., role mappings) 14 | # 15 | # For more information see: 16 | # 17 | # * Rego comparison to other systems: https://www.openpolicyagent.org/docs/latest/comparison-to-other-systems/ 18 | # * Rego Iteration: https://www.openpolicyagent.org/docs/latest/#iteration 19 | 20 | package app.rbac 21 | 22 | # By default, deny requests. 23 | default allow = false 24 | 25 | # Allow admins to do anything. 26 | allow { 27 | user_is_admin 28 | } 29 | 30 | # Allow the action if the user is granted permission to perform the action. 31 | allow { 32 | # Find grants for the user. 33 | some grant 34 | user_is_granted[grant] 35 | 36 | # Check if the grant permits the action. 37 | input.action == grant.action 38 | input.type == grant.type 39 | } 40 | 41 | # user_is_admin is true if... 42 | user_is_admin { 43 | 44 | # for some `i`... 45 | some i 46 | 47 | # "admin" is the `i`-th element in the user->role mappings for the identified user. 48 | data.user_roles[input.user][i] == "admin" 49 | } 50 | 51 | # user_is_granted is a set of grants for the user identified in the request. 52 | # The `grant` will be contained if the set `user_is_granted` for every... 53 | user_is_granted[grant] { 54 | some i, j 55 | 56 | # `role` assigned an element of the user_roles for this user... 57 | role := data.user_roles[input.user][i] 58 | 59 | # `grant` assigned a single grant from the grants list for 'role'... 60 | grant := data.role_grants[role][j] 61 | } 62 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/tests/parsing_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import pytest 5 | 6 | # Add root opal dir to use local src as package for tests (i.e, no need for python -m pytest) 7 | root_dir = os.path.abspath( 8 | os.path.join( 9 | os.path.dirname(__file__), 10 | os.path.pardir, 11 | os.path.pardir, 12 | os.path.pardir, 13 | ) 14 | ) 15 | sys.path.append(root_dir) 16 | 17 | from opal_common.engine.parsing import get_rego_package 18 | 19 | 20 | def test_can_extract_the_correct_package_name(): 21 | """The different variations of package names (with real examples from opa 22 | playground)""" 23 | # package in first line, no dots 24 | source_rego = os.path.join(os.path.dirname(__file__), "fixtures/play.rego") 25 | with open(source_rego, "r") as f: 26 | contents = f.read() 27 | assert get_rego_package(contents) == "play" 28 | 29 | # package after comments, two part name 30 | source_rego = os.path.join(os.path.dirname(__file__), "fixtures/rbac.rego") 31 | with open(source_rego, "r") as f: 32 | contents = f.read() 33 | assert get_rego_package(contents) == "app.rbac" 34 | 35 | # package after comments, three part name 36 | source_rego = os.path.join(os.path.dirname(__file__), "fixtures/jwt.rego") 37 | with open(source_rego, "r") as f: 38 | contents = f.read() 39 | assert get_rego_package(contents) == "envoy.http.jwt" 40 | 41 | 42 | def test_no_package_name_in_file(): 43 | """Test no package name in module or invalid package.""" 44 | # package line was removed 45 | source_rego = os.path.join(os.path.dirname(__file__), "fixtures/no-package.rego") 46 | with open(source_rego, "r") as f: 47 | contents = f.read() 48 | assert get_rego_package(contents) is None 49 | 50 | # package line with invalid contents 51 | source_rego = os.path.join( 52 | os.path.dirname(__file__), "fixtures/invalid-package.rego" 53 | ) 54 | with open(source_rego, "r") as f: 55 | contents = f.read() 56 | assert get_rego_package(contents) is None 57 | 58 | # empty file 59 | assert get_rego_package("") is None 60 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/engine/tests/paths_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import pytest 5 | 6 | # Add root opal dir to use local src as package for tests (i.e, no need for python -m pytest) 7 | root_dir = os.path.abspath( 8 | os.path.join( 9 | os.path.dirname(__file__), 10 | os.path.pardir, 11 | os.path.pardir, 12 | os.path.pardir, 13 | ) 14 | ) 15 | sys.path.append(root_dir) 16 | 17 | from pathlib import Path 18 | 19 | from opal_common.engine.paths import is_data_module, is_policy_module 20 | 21 | 22 | def test_is_data_module(): 23 | """Test is_data_module() on different paths.""" 24 | # files that are named data.json are data modules 25 | assert is_data_module(Path("data.json")) == True 26 | assert is_data_module(Path("some/dir/to/data.json")) == True 27 | 28 | # json files that are not named data.json are not data modules 29 | assert is_data_module(Path("other.json")) == False 30 | assert is_data_module(Path("some/dir/to/other.json")) == False 31 | 32 | # files with other extensions are not data modules 33 | assert is_data_module(Path("data.txt")) == False 34 | 35 | # directories are not data modules 36 | assert is_data_module(Path(".")) == False 37 | assert is_data_module(Path("some/dir/to")) == False 38 | 39 | 40 | def test_is_policy_module(): 41 | """Test is_policy_module() on different paths.""" 42 | # files with rego extension are rego modules 43 | assert is_policy_module(Path("some/dir/to/file.rego")) == True 44 | assert is_policy_module(Path("rbac.rego")) == True 45 | 46 | # files with other extensions are not rego modules 47 | assert is_policy_module(Path("rbac.json")) == False 48 | 49 | # directories are not data modules 50 | assert is_policy_module(Path(".")) == False 51 | assert is_policy_module(Path("some/dir/to")) == False 52 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/__init__.py: -------------------------------------------------------------------------------- 1 | from opal_common.fetcher.engine.fetching_engine import FetchingEngine 2 | from opal_common.fetcher.events import FetcherConfig, FetchEvent 3 | from opal_common.fetcher.fetcher_register import FetcherRegister 4 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/engine/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/fetcher/engine/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/engine/core_callbacks.py: -------------------------------------------------------------------------------- 1 | from opal_common.fetcher.events import FetchEvent 2 | 3 | 4 | # Callback signatures 5 | async def OnFetchFailureCallback(exception: Exception, event: FetchEvent): 6 | """ 7 | Args: 8 | exception (Exception): The exception thrown causing the failure 9 | event (FetchEvent): the queued event which failed 10 | """ 11 | pass 12 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/engine/fetch_worker.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Coroutine 3 | 4 | from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine 5 | from opal_common.fetcher.events import FetchEvent 6 | from opal_common.fetcher.fetcher_register import FetcherRegister 7 | from opal_common.fetcher.logger import get_logger 8 | 9 | logger = get_logger("fetch_worker") 10 | 11 | 12 | async def fetch_worker(queue: asyncio.Queue, engine): 13 | """The worker task performing items added to the Engine's Queue. 14 | 15 | Args: 16 | queue (asyncio.Queue): The Queue 17 | engine (BaseFetchingEngine): The engine itself 18 | """ 19 | engine: BaseFetchingEngine 20 | register: FetcherRegister = engine.register 21 | while True: 22 | # types 23 | event: FetchEvent 24 | callback: Coroutine 25 | # get a event from the queue 26 | event, callback = await queue.get() 27 | # take care of it 28 | try: 29 | # get fetcher for the event 30 | fetcher = register.get_fetcher_for_event(event) 31 | # fetch 32 | async with fetcher: 33 | res = await fetcher.fetch() 34 | data = await fetcher.process(res) 35 | # callback to event owner 36 | try: 37 | await callback(data) 38 | except Exception as err: 39 | logger.exception(f"Fetcher callback - {callback} failed") 40 | await engine._on_failure(err, event) 41 | except Exception as err: 42 | logger.exception("Failed to process fetch event") 43 | await engine._on_failure(err, event) 44 | finally: 45 | # Notify the queue that the "work item" has been processed. 46 | queue.task_done() 47 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/events.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from pydantic import BaseModel, Field 4 | 5 | 6 | class FetcherConfig(BaseModel): 7 | """The configuration of a fetcher, used as part of a FetchEvent Fetch 8 | Provider's have their own uniqueue events and configurations. 9 | 10 | Configurations 11 | """ 12 | 13 | fetcher: Optional[str] = Field( 14 | None, 15 | description="indicates to OPAL client that it should use a custom FetcherProvider to fetch the data", 16 | ) 17 | 18 | 19 | class FetchEvent(BaseModel): 20 | """Event used to describe an queue fetching tasks Design note - 21 | 22 | By using a Pydantic model - we can create a potentially transfer FetchEvents to be handled by other network nodes (perhaps via RPC) 23 | """ 24 | 25 | # Event id to be filled by the engine 26 | id: str = None 27 | # optional name of the specific event 28 | name: str = None 29 | # A string identifying the fetcher class to use (as registered in the fetcher register) 30 | fetcher: str 31 | # The url the event targets for fetching 32 | url: str 33 | # Specific fetcher configuration (overridden by deriving event classes (FetcherConfig) 34 | config: dict = None 35 | # Tenacity.retry - Override default retry configuration for this event 36 | retry: dict = None 37 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger("opal.fetcher") 4 | 5 | 6 | def get_logger(name): 7 | return logger.getChild(name) 8 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/providers/__init__.py: -------------------------------------------------------------------------------- 1 | from opal_common.emport import dynamic_all 2 | 3 | __all__ = dynamic_all(__file__) 4 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py: -------------------------------------------------------------------------------- 1 | """Simple HTTP get data fetcher using requests supports.""" 2 | 3 | from fastapi_websocket_rpc.rpc_methods import RpcMethodsBase 4 | from fastapi_websocket_rpc.websocket_rpc_client import WebSocketRpcClient 5 | from opal_common.fetcher.events import FetcherConfig, FetchEvent 6 | from opal_common.fetcher.fetch_provider import BaseFetchProvider 7 | from opal_common.fetcher.logger import get_logger 8 | 9 | logger = get_logger("rpc_fetch_provider") 10 | 11 | 12 | class FastApiRpcFetchConfig(FetcherConfig): 13 | """Config for FastApiRpcFetchConfig's Adding HTTP headers.""" 14 | 15 | rpc_method_name: str 16 | rpc_arguments: dict 17 | 18 | 19 | class FastApiRpcFetchEvent(FetchEvent): 20 | fetcher: str = "FastApiRpcFetchProvider" 21 | config: FastApiRpcFetchConfig 22 | 23 | 24 | class FastApiRpcFetchProvider(BaseFetchProvider): 25 | def __init__(self, event: FastApiRpcFetchEvent) -> None: 26 | self._event: FastApiRpcFetchEvent 27 | super().__init__(event) 28 | 29 | def parse_event(self, event: FetchEvent) -> FastApiRpcFetchEvent: 30 | return FastApiRpcFetchEvent( 31 | **event.dict(exclude={"config"}), config=event.config 32 | ) 33 | 34 | async def _fetch_(self): 35 | assert ( 36 | self._event is not None 37 | ), "FastApiRpcFetchEvent not provided for FastApiRpcFetchProvider" 38 | args = self._event.config.rpc_arguments 39 | method = self._event.config.rpc_method_name 40 | result = None 41 | logger.info( 42 | f"{self.__class__.__name__} fetching from {self._url} with RPC call {method}({args})" 43 | ) 44 | async with WebSocketRpcClient( 45 | self._url, 46 | # we don't expose anything to the server 47 | RpcMethodsBase(), 48 | default_response_timeout=4, 49 | ) as client: 50 | result = await client.call(method, args) 51 | return result 52 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/fetcher/tests/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/tests/failure_handler_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import aiohttp 5 | 6 | # Add parent path to use local src as package for tests 7 | root_dir = os.path.abspath( 8 | os.path.join( 9 | os.path.dirname(__file__), os.path.pardir, os.path.pardir, os.path.pardir 10 | ) 11 | ) 12 | print(root_dir) 13 | sys.path.append(root_dir) 14 | 15 | import asyncio 16 | 17 | import pytest 18 | import tenacity 19 | from opal_common.fetcher import FetchEvent, FetchingEngine 20 | from opal_common.fetcher.providers.http_fetch_provider import ( 21 | HttpFetchEvent, 22 | HttpFetchProvider, 23 | ) 24 | 25 | # Configurable 26 | PORT = int(os.environ.get("PORT") or "9110") 27 | BASE_URL = f"http://localhost:{PORT}" 28 | DATA_ROUTE = f"/data" 29 | DATA_KEY = "Hello" 30 | DATA_VALUE = "World" 31 | DATA_SECRET_VALUE = "SecretWorld" 32 | 33 | 34 | @pytest.mark.asyncio 35 | async def test_retry_failure(): 36 | """Test callback on failure.""" 37 | got_data_event = asyncio.Event() 38 | got_error = asyncio.Event() 39 | 40 | async with FetchingEngine() as engine: 41 | # callback to handle failure 42 | async def error_callback(error: Exception, event: FetchEvent): 43 | # check we got the exception we expected 44 | assert isinstance(error, aiohttp.client_exceptions.ClientConnectorError) 45 | got_error.set() 46 | 47 | # register the callback 48 | engine.register_failure_handler(error_callback) 49 | 50 | # callback for success - shouldn't eb called in this test 51 | async def callback(result): 52 | got_data_event.set() 53 | 54 | # Use an event on an invalid port - and only to attempts 55 | retry_config = HttpFetchProvider.DEFAULT_RETRY_CONFIG.copy() 56 | retry_config["stop"] = tenacity.stop.stop_after_attempt(2) 57 | event = HttpFetchEvent(url=f"http://localhost:25", retry=retry_config) 58 | # queue the event 59 | await engine.queue_fetch_event(event, callback) 60 | # wait for the failure callback 61 | await asyncio.wait_for(got_error.wait(), 25) 62 | assert not got_data_event.is_set() 63 | assert got_error.is_set() 64 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/fetcher/tests/rpc_fetch_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # Add parent path to use local src as package for tests 5 | root_dir = os.path.abspath( 6 | os.path.join( 7 | os.path.dirname(__file__), os.path.pardir, os.path.pardir, os.path.pardir 8 | ) 9 | ) 10 | sys.path.append(root_dir) 11 | 12 | import asyncio 13 | from multiprocessing import Process 14 | 15 | import pytest 16 | import uvicorn 17 | from fastapi import FastAPI 18 | from fastapi_websocket_rpc import RpcMethodsBase, WebsocketRPCEndpoint 19 | from opal_common.fetcher import FetchingEngine 20 | from opal_common.fetcher.providers.fastapi_rpc_fetch_provider import ( 21 | FastApiRpcFetchConfig, 22 | FastApiRpcFetchEvent, 23 | FastApiRpcFetchProvider, 24 | ) 25 | 26 | # Configurable 27 | PORT = int(os.environ.get("PORT") or "9110") 28 | uri = f"ws://localhost:{PORT}/rpc" 29 | DATA_PREFIX = "I AM DATA - HEAR ME ROAR" 30 | SUFFIX = " - Magic!" 31 | 32 | 33 | class RpcData(RpcMethodsBase): 34 | async def get_data(self, suffix: str) -> str: 35 | return DATA_PREFIX + suffix 36 | 37 | 38 | def setup_server(): 39 | app = FastAPI() 40 | endpoint = WebsocketRPCEndpoint(RpcData()) 41 | endpoint.register_route(app, "/rpc") 42 | 43 | uvicorn.run(app, port=PORT) 44 | 45 | 46 | @pytest.fixture(scope="module") 47 | def server(): 48 | # Run the server as a separate process 49 | proc = Process(target=setup_server, args=(), daemon=True) 50 | proc.start() 51 | yield proc 52 | proc.kill() # Cleanup after test 53 | 54 | 55 | @pytest.mark.asyncio 56 | async def test_simple_rpc_fetch(server): 57 | """""" 58 | got_data_event = asyncio.Event() 59 | async with FetchingEngine() as engine: 60 | engine.register.register_fetcher( 61 | FastApiRpcFetchProvider.__name__, FastApiRpcFetchProvider 62 | ) 63 | # Event for RPC fetch 64 | fetch_event = FastApiRpcFetchEvent( 65 | url=uri, 66 | config=FastApiRpcFetchConfig( 67 | rpc_method_name="get_data", rpc_arguments={"suffix": SUFFIX} 68 | ), 69 | ) 70 | 71 | # Callback for event 72 | async def callback(result): 73 | data = result.result 74 | assert data == DATA_PREFIX + SUFFIX 75 | got_data_event.set() 76 | 77 | await engine.queue_fetch_event(fetch_event, callback) 78 | await asyncio.wait_for(got_data_event.wait(), 5) 79 | assert got_data_event.is_set() 80 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/git_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/git_utils/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/git_utils/bundle_utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List 3 | 4 | from opal_common.schemas.policy import DataModule, PolicyBundle, RegoModule 5 | 6 | 7 | class BundleUtils: 8 | MAX_INDEX = 10000 9 | 10 | @staticmethod 11 | def sorted_policy_modules_to_load( 12 | bundle: PolicyBundle, ignore=None 13 | ) -> List[RegoModule]: 14 | """Policy modules sorted according to manifest.""" 15 | manifest_paths = [Path(path) for path in bundle.manifest] 16 | 17 | def key_function(module: RegoModule) -> int: 18 | """This method reduces the module to a number that can be act as 19 | sorting key. 20 | 21 | the number is the index in the manifest list, so basically 22 | we sort according to manifest. 23 | """ 24 | try: 25 | return manifest_paths.index(Path(module.path)) 26 | except ValueError: 27 | return BundleUtils.MAX_INDEX 28 | 29 | return sorted(bundle.policy_modules, key=key_function) 30 | 31 | @staticmethod 32 | def sorted_data_modules_to_load(bundle: PolicyBundle) -> List[DataModule]: 33 | """Data modules sorted according to manifest.""" 34 | manifest_paths = [Path(path) for path in bundle.manifest] 35 | 36 | def key_function(module: DataModule) -> int: 37 | try: 38 | return manifest_paths.index(Path(module.path)) 39 | except ValueError: 40 | return BundleUtils.MAX_INDEX 41 | 42 | return sorted(bundle.data_modules, key=key_function) 43 | 44 | @staticmethod 45 | def sorted_policy_modules_to_delete(bundle: PolicyBundle) -> List[Path]: 46 | if bundle.deleted_files is None: 47 | return [] 48 | # already sorted 49 | return bundle.deleted_files.policy_modules 50 | 51 | @staticmethod 52 | def sorted_data_modules_to_delete(bundle: PolicyBundle) -> List[Path]: 53 | if bundle.deleted_files is None: 54 | return [] 55 | # already sorted 56 | return bundle.deleted_files.data_modules 57 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/git_utils/env.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | from opal_common.config import opal_common_config 5 | 6 | SSH_PREFIX = "ssh://" 7 | GIT_SSH_USER_PREFIX = "git@" 8 | 9 | 10 | def save_ssh_key_to_pem_file(key: str) -> Path: 11 | key = key.replace("_", "\n") 12 | if not key.endswith("\n"): 13 | key = key + "\n" # pem file must end with newline 14 | key_path = os.path.expanduser(opal_common_config.GIT_SSH_KEY_FILE) 15 | parent_directory = os.path.dirname(key_path) 16 | if not os.path.exists(parent_directory): 17 | os.makedirs(parent_directory, exist_ok=True) 18 | with open(key_path, "w") as f: 19 | f.write(key) 20 | os.chmod(key_path, 0o600) 21 | return Path(key_path) 22 | 23 | 24 | def is_ssh_repo_url(repo_url: str): 25 | """Return True if the repo url uses SSH authentication. 26 | 27 | (see: 28 | https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh) 29 | """ 30 | return repo_url.startswith(SSH_PREFIX) or repo_url.startswith(GIT_SSH_USER_PREFIX) 31 | 32 | 33 | def provide_git_ssh_environment(url: str, ssh_key: str): 34 | """Provides git SSH configuration via GIT_SSH_COMMAND. 35 | 36 | the git ssh config will be provided only if the following conditions are met: 37 | - the repo url is a git ssh url 38 | - an ssh private key is provided in Repo Cloner __init__ 39 | """ 40 | if not is_ssh_repo_url(url) or ssh_key is None: 41 | return {} # no ssh config 42 | git_ssh_identity_file = save_ssh_key_to_pem_file(ssh_key) 43 | return { 44 | "GIT_SSH_COMMAND": f"ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i {git_ssh_identity_file}", 45 | "GIT_TRACE": "1", 46 | "GIT_CURL_VERBOSE": "1", 47 | } 48 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/git_utils/exceptions.py: -------------------------------------------------------------------------------- 1 | class GitFailed(Exception): 2 | """An exception we throw on git failures that are caused by wrong 3 | assumptions. 4 | 5 | i.e: we want to track a non-existing branch, or git url is not 6 | valid. 7 | """ 8 | 9 | def __init__(self, exc: Exception): 10 | self._original_exc = exc 11 | super().__init__() 12 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/http_utils.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | import aiohttp 4 | import httpx 5 | 6 | 7 | def is_http_error_response( 8 | response: Union[aiohttp.ClientResponse, httpx.Response] 9 | ) -> bool: 10 | """HTTP 400 and above are considered error responses.""" 11 | status: int = ( 12 | response.status 13 | if isinstance(response, aiohttp.ClientResponse) 14 | else response.status_code 15 | ) 16 | 17 | return status >= 400 18 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | from loguru import logger 5 | from opal_common.config import opal_common_config 6 | from opal_common.logging_utils.filter import ModuleFilter 7 | from opal_common.logging_utils.formatter import Formatter 8 | from opal_common.logging_utils.intercept import InterceptHandler 9 | from opal_common.logging_utils.thirdparty import hijack_uvicorn_logs 10 | from opal_common.monitoring.apm import fix_ddtrace_logging 11 | 12 | 13 | def configure_logs(): 14 | """Takeover process logs and create a logger with Loguru according to the 15 | configuration.""" 16 | fix_ddtrace_logging() 17 | 18 | intercept_handler = InterceptHandler() 19 | formatter = Formatter(opal_common_config.LOG_FORMAT) 20 | filter = ModuleFilter( 21 | include_list=opal_common_config.LOG_MODULE_INCLUDE_LIST, 22 | exclude_list=opal_common_config.LOG_MODULE_EXCLUDE_LIST, 23 | ) 24 | logging.basicConfig(handlers=[intercept_handler], level=0, force=True) 25 | 26 | if opal_common_config.LOG_PATCH_UVICORN_LOGS: 27 | # Monkey patch UVICORN to use our logger 28 | hijack_uvicorn_logs(intercept_handler) 29 | # Clean slate 30 | logger.remove() 31 | # Logger configuration 32 | logger.add( 33 | sys.stderr, 34 | filter=filter.filter, 35 | format=formatter.format, 36 | level=opal_common_config.LOG_LEVEL, 37 | backtrace=opal_common_config.LOG_TRACEBACK, 38 | diagnose=opal_common_config.LOG_DIAGNOSE, 39 | colorize=opal_common_config.LOG_COLORIZE, 40 | serialize=opal_common_config.LOG_SERIALIZE, 41 | ) 42 | # log to a file 43 | if opal_common_config.LOG_TO_FILE: 44 | logger.add( 45 | opal_common_config.LOG_FILE_PATH, 46 | compression=opal_common_config.LOG_FILE_COMPRESSION, 47 | retention=opal_common_config.LOG_FILE_RETENTION, 48 | rotation=opal_common_config.LOG_FILE_ROTATION, 49 | serialize=opal_common_config.LOG_FILE_SERIALIZE, 50 | level=opal_common_config.LOG_FILE_LEVEL, 51 | ) 52 | 53 | 54 | def get_logger(name=""): 55 | """Backward compatibility to old get_logger.""" 56 | return logger 57 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/logging_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/logging_utils/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/logging_utils/decorators.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import logging 3 | 4 | 5 | def log_exception(logger=logging.getLogger(), rethrow=True): 6 | def deco(func): 7 | @functools.wraps(func) 8 | def wrapper(*args, **kwargs): 9 | try: 10 | return func(*args, **kwargs) 11 | except Exception as e: 12 | logger.exception(e) 13 | if rethrow: 14 | raise 15 | 16 | return wrapper 17 | 18 | return deco 19 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/logging_utils/filter.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | 4 | class ModuleFilter: 5 | """Filter logs by module name.""" 6 | 7 | def __init__( 8 | self, exclude_list: List[str] = None, include_list: List[str] = None 9 | ) -> None: 10 | """[summary] 11 | 12 | Args: 13 | exclude_list (List[str], optional): module name (prefixes) to reject. Defaults to []. 14 | include_list (List[str], optional): module name (prefixes) to include (even if higher form is excluded). Defaults to []. 15 | 16 | Usage: 17 | ModuleFilter(["uvicorn"]) # exclude all logs coming from module name starting with "uvicorn" 18 | ModuleFilter(["uvicorn"], ["uvicorn.access]) # exclude all logs coming from module name starting with "uvicorn" except ones starting with "uvicorn.access") 19 | """ 20 | self._exclude_list = exclude_list or [] 21 | self._include_list = include_list or [] 22 | 23 | def filter(self, record): 24 | name: str = record["name"] 25 | for module in self._include_list: 26 | if name.startswith(module): 27 | return True 28 | for module in self._exclude_list: 29 | if name.startswith(module): 30 | return False 31 | return True 32 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/logging_utils/formatter.py: -------------------------------------------------------------------------------- 1 | class Formatter: 2 | MAX_FIELD_LEN = 25 3 | 4 | def __init__(self, format_string: str): 5 | self.fmt = format_string 6 | 7 | def limit_len(self, record, field, length=MAX_FIELD_LEN): 8 | # Shorten field content 9 | content = record[field] 10 | if len(content) > length: 11 | parts = content.split(".") 12 | if len(parts) > 2: 13 | content = f"{parts[0]}...{parts[-1]}" 14 | if len(content) > length: 15 | content = f"{content[:length-3]}..." 16 | record[field] = content 17 | 18 | def format(self, record): 19 | self.limit_len(record, "name", 40) 20 | return self.fmt 21 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/logging_utils/intercept.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from loguru import logger 4 | 5 | 6 | class InterceptHandler(logging.Handler): 7 | def emit(self, record): 8 | # Get corresponding Loguru level if it exists 9 | try: 10 | level = logger.level(record.levelname).name 11 | except ValueError: 12 | level = record.levelno 13 | 14 | # Find caller from where originated the logged message 15 | frame, depth = logging.currentframe(), 2 16 | while frame.f_code.co_filename == logging.__file__: 17 | if frame.f_back is None: 18 | break 19 | frame = frame.f_back 20 | depth += 1 21 | 22 | logger.opt(depth=depth, exception=record.exc_info).log( 23 | level, record.getMessage() 24 | ) 25 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/logging_utils/thirdparty.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def hijack_uvicorn_logs(intercept_handler: logging.Handler): 5 | """Uvicorn loggers are configured to use special handlers. 6 | 7 | Adding an intercept handler to the root logger manages to intercept logs from uvicorn, however, the log messages are duplicated. 8 | This is happening because uvicorn loggers are propagated by default - we get a log message once for the "uvicorn" / "uvicorn.error" 9 | logger and once for the root logger). Another stupid issue is that the "uvicorn.error" logger is not just for errors, which is confusing. 10 | 11 | This method is doing 2 things for each uvicorn logger: 12 | 1) remove all existing handlers and replace them with the intercept handler (i.e: will be logged via loguru) 13 | 2) cancel propagation - which will mean messages will not propagate to the root logger (which also has an InterceptHandler), fixing the duplication 14 | """ 15 | # get loggers directly from uvicorn config - if they will change something - we will know. 16 | from uvicorn.config import LOGGING_CONFIG 17 | 18 | uvicorn_logger_names = list(LOGGING_CONFIG.get("loggers", {}).keys()) or [ 19 | "uvicorn", 20 | "uvicorn.access", 21 | "uvicorn.error", 22 | ] 23 | for logger_name in uvicorn_logger_names: 24 | logger = logging.getLogger(logger_name) 25 | logger.handlers = [intercept_handler] 26 | logger.propagate = False 27 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/monitoring/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/monitoring/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/monitoring/apm.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Optional 3 | from urllib.parse import urlparse 4 | 5 | from ddtrace import Span, config, patch, tracer 6 | from ddtrace.filters import TraceFilter 7 | from loguru import logger 8 | 9 | 10 | def configure_apm(enable_apm: bool, service_name: str): 11 | """Optionally enable datadog APM / profiler.""" 12 | if enable_apm: 13 | logger.info("Enabling DataDog APM") 14 | # logging.getLogger("ddtrace").propagate = False 15 | 16 | class FilterRootPathTraces(TraceFilter): 17 | def process_trace(self, trace: list[Span]) -> Optional[list[Span]]: 18 | for span in trace: 19 | if span.parent_id is not None: 20 | return trace 21 | 22 | if url := span.get_tag("http.url"): 23 | parsed_url = urlparse(url) 24 | 25 | if parsed_url.path == "/": 26 | return None 27 | 28 | return trace 29 | 30 | patch( 31 | fastapi=True, 32 | redis=True, 33 | asyncpg=True, 34 | aiohttp=True, 35 | loguru=True, 36 | ) 37 | tracer.configure( 38 | settings={ 39 | "FILTERS": [ 40 | FilterRootPathTraces(), 41 | ] 42 | } 43 | ) 44 | 45 | else: 46 | logger.info("DataDog APM disabled") 47 | tracer.configure(enabled=False) 48 | 49 | 50 | def fix_ddtrace_logging(): 51 | logging.getLogger("ddtrace").setLevel(logging.WARNING) 52 | 53 | ddtrace_logger = logging.getLogger("ddtrace") 54 | for handler in ddtrace_logger.handlers: 55 | ddtrace_logger.removeHandler(handler) 56 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/monitoring/metrics.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Optional 3 | 4 | import datadog 5 | from loguru import logger 6 | 7 | 8 | def configure_metrics( 9 | enable_metrics: bool, statsd_host: str, statsd_port: int, namespace: str = "" 10 | ): 11 | if not enable_metrics: 12 | logger.info("DogStatsD metrics disabled") 13 | return 14 | else: 15 | logger.info( 16 | "DogStatsD metrics enabled; statsd: {host}:{port}", 17 | host=statsd_host, 18 | port=statsd_port, 19 | ) 20 | 21 | if not namespace: 22 | namespace = os.environ.get("DD_SERVICE", "") 23 | 24 | namespace = namespace.lower().replace("-", "_") 25 | datadog.initialize( 26 | statsd_host=statsd_host, 27 | statsd_port=statsd_port, 28 | statsd_namespace=f"permit.{namespace}", 29 | ) 30 | 31 | 32 | def _format_tags(tags: Optional[dict[str, str]]) -> Optional[list[str]]: 33 | if not tags: 34 | return None 35 | 36 | return [f"{k}:{v}" for k, v in tags.items()] 37 | 38 | 39 | def increment(metric: str, tags: Optional[dict[str, str]] = None): 40 | datadog.statsd.increment(metric, tags=_format_tags(tags)) 41 | 42 | 43 | def decrement(metric: str, tags: Optional[dict[str, str]] = None): 44 | datadog.statsd.decrement(metric, tags=_format_tags(tags)) 45 | 46 | 47 | def gauge(metric: str, value: float, tags: Optional[dict[str, str]] = None): 48 | datadog.statsd.gauge(metric, value, tags=_format_tags(tags)) 49 | 50 | 51 | def event(title: str, message: str, tags: Optional[dict[str, str]] = None): 52 | datadog.statsd.event(title=title, message=message, tags=_format_tags(tags)) 53 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/schemas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/schemas/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/schemas/policy.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List, Optional 3 | 4 | from pydantic import BaseModel, Field 5 | 6 | 7 | class BaseSchema(BaseModel): 8 | class Config: 9 | orm_mode = True 10 | 11 | 12 | class DataModule(BaseSchema): 13 | path: str = Field( 14 | ..., description="where to place the data module relative to opa data root" 15 | ) 16 | data: str = Field(..., description="data module file contents (json)") 17 | 18 | 19 | class RegoModule(BaseSchema): 20 | path: str = Field( 21 | ..., 22 | description="path of policy module on disk, will be used to generate policy id", 23 | ) 24 | package_name: str = Field(..., description="opa module package name") 25 | rego: str = Field(..., description="rego module file contents (text)") 26 | 27 | 28 | class DeletedFiles(BaseSchema): 29 | data_modules: List[Path] = [] 30 | policy_modules: List[Path] = [] 31 | 32 | 33 | class PolicyBundle(BaseSchema): 34 | manifest: List[str] 35 | hash: str = Field(..., description="commit hash (debug version)") 36 | old_hash: Optional[str] = Field( 37 | None, description="old commit hash (in diff bundles)" 38 | ) 39 | data_modules: List[DataModule] 40 | policy_modules: List[RegoModule] 41 | deleted_files: Optional[DeletedFiles] 42 | 43 | 44 | class PolicyUpdateMessage(BaseSchema): 45 | old_policy_hash: str 46 | new_policy_hash: str 47 | changed_directories: List[str] 48 | 49 | 50 | class PolicyUpdateMessageNotification(BaseSchema): 51 | update: PolicyUpdateMessage 52 | topics: List[str] 53 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/schemas/policy_source.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional, Union 2 | 3 | try: 4 | from typing import Literal 5 | except ImportError: 6 | # Py<3.8 7 | from typing_extensions import Literal 8 | 9 | from opal_common.schemas.policy import BaseSchema 10 | from pydantic import Field 11 | 12 | 13 | class NoAuthData(BaseSchema): 14 | auth_type: Literal["none"] = "none" 15 | 16 | 17 | class SSHAuthData(BaseSchema): 18 | auth_type: Literal["ssh"] = "ssh" 19 | username: str = Field(..., description="SSH username") 20 | public_key: Optional[str] = Field(None, description="SSH public key") 21 | private_key: str = Field(..., description="SSH private key") 22 | 23 | 24 | class GitHubTokenAuthData(BaseSchema): 25 | auth_type: Literal["github_token"] = "github_token" 26 | token: str = Field(..., description="Github Personal Access Token (PAI)") 27 | 28 | 29 | class UserPassAuthData(BaseSchema): 30 | auth_type: Literal["userpass"] = "userpass" 31 | username: str = Field(..., description="Username") 32 | password: str = Field(..., description="Password") 33 | 34 | 35 | class BasePolicyScopeSource(BaseSchema): 36 | source_type: str 37 | url: str 38 | auth: Union[NoAuthData, SSHAuthData, GitHubTokenAuthData, UserPassAuthData] = Field( 39 | ..., discriminator="auth_type" 40 | ) 41 | directories: List[str] = Field(["."], description="Directories to include") 42 | extensions: List[str] = Field( 43 | [".rego", ".json"], description="File extensions to use" 44 | ) 45 | bundle_ignore: Optional[List[str]] = Field( 46 | None, description="glob paths to omit from bundle" 47 | ) 48 | manifest: str = Field(".manifest", description="path to manifest file") 49 | poll_updates: bool = Field( 50 | False, description="Whether OPAL should check for updates periodically" 51 | ) 52 | 53 | 54 | class GitPolicyScopeSource(BasePolicyScopeSource): 55 | branch: str = Field("main", description="Git branch to track") 56 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/schemas/scopes.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from opal_common.schemas.data import DataSourceConfig 4 | from opal_common.schemas.policy import BaseSchema 5 | from opal_common.schemas.policy_source import GitPolicyScopeSource 6 | from pydantic import Field 7 | 8 | 9 | class Scope(BaseSchema): 10 | scope_id: str = Field(..., description="Scope ID") 11 | policy: Union[GitPolicyScopeSource] = Field(..., description="Policy source") 12 | data: DataSourceConfig = Field( 13 | DataSourceConfig(entries=[]), description="Data source configuration" 14 | ) 15 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/schemas/security.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from enum import Enum 3 | from typing import Optional 4 | from uuid import UUID, uuid4 5 | 6 | from pydantic import BaseModel, Field, validator 7 | 8 | PEER_TYPE_DESCRIPTION = ( 9 | "The peer type we generate access token for, i.e: opal client, data provider, etc." 10 | ) 11 | TTL_DESCRIPTION = ( 12 | "Token lifetime (timedelta), can accept duration in seconds or ISO_8601 format." 13 | + " see: https://en.wikipedia.org/wiki/ISO_8601#Durations" 14 | ) 15 | CLAIMS_DESCRIPTION = "extra claims to attach to the jwt" 16 | 17 | 18 | class PeerType(str, Enum): 19 | client = "client" 20 | datasource = "datasource" 21 | listener = "listener" 22 | 23 | 24 | class AccessTokenRequest(BaseModel): 25 | """A request to generate an access token to opal server.""" 26 | 27 | id: UUID = Field(default_factory=uuid4) 28 | type: PeerType = Field(PeerType.client, description=PEER_TYPE_DESCRIPTION) 29 | ttl: timedelta = Field(timedelta(days=365), description=TTL_DESCRIPTION) 30 | claims: dict = Field({}, description=CLAIMS_DESCRIPTION) 31 | 32 | @validator("type") 33 | def force_enum(cls, v): 34 | if isinstance(v, str): 35 | return PeerType(v) 36 | if isinstance(v, PeerType): 37 | return v 38 | raise ValueError(f"invalid value: {v}") 39 | 40 | class Config: 41 | use_enum_values = True 42 | allow_population_by_field_name = True 43 | 44 | 45 | class TokenDetails(BaseModel): 46 | id: UUID 47 | type: PeerType = Field(PeerType.client, description=PEER_TYPE_DESCRIPTION) 48 | expired: datetime 49 | claims: dict 50 | 51 | 52 | class AccessToken(BaseModel): 53 | token: str 54 | type: str = "bearer" 55 | details: Optional[TokenDetails] 56 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/schemas/store.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from enum import Enum 3 | from typing import Any, Dict, List, Optional 4 | 5 | from pydantic import BaseModel, Field, root_validator 6 | 7 | 8 | class TransactionType(str, Enum): 9 | policy = "policy" 10 | data = "data" 11 | 12 | 13 | class RemoteStatus(BaseModel): 14 | remote_url: str = Field(None, description="Url of remote data/policy source") 15 | succeed: bool = Field(True, description="Is request succeed") 16 | error: str = Field(None, description="If failed contains the type of exception") 17 | 18 | 19 | class StoreTransaction(BaseModel): 20 | """Represents a transaction of policy or data to OPA.""" 21 | 22 | id: str = Field(..., description="The id of the transaction") 23 | actions: List[str] = Field( 24 | ..., description="The write actions performed as part of the transaction" 25 | ) 26 | transaction_type: TransactionType = Field( 27 | None, description="Type of transaction,is it data/policy transaction" 28 | ) 29 | success: bool = Field( 30 | False, description="Whether or not the transaction was successful" 31 | ) 32 | error: str = Field( 33 | "", description="Error message in case of failure, defaults to empty string" 34 | ) 35 | creation_time: str = Field( 36 | None, description="Creation time for this store transaction" 37 | ) 38 | end_time: str = Field(None, description="Finish time for this store transaction") 39 | remotes_status: List[RemoteStatus] = Field( 40 | None, 41 | description="List of the remote sources for this transaction and their status", 42 | ) 43 | 44 | 45 | class JSONPatchAction(BaseModel): 46 | """Abstract base class for JSON patch actions (RFC 6902)""" 47 | 48 | op: str = Field(..., description="patch action to perform") 49 | path: str = Field(..., description="target location in modified json") 50 | value: Optional[Any] = Field( 51 | None, description="json document, the operand of the action" 52 | ) 53 | from_field: Optional[str] = Field( 54 | None, description="source location in json", alias="from" 55 | ) 56 | 57 | @root_validator 58 | def value_must_be_present(cls, values): 59 | if values.get("op") in ["add", "replace"] and values.get("value") is None: 60 | raise TypeError("'value' must be present when op is either add or replace") 61 | return values 62 | 63 | 64 | class ArrayAppendAction(JSONPatchAction): 65 | op: str = Field("add", description="add action -> adds to the array") 66 | path: str = Field("-", description="dash marks the last index of an array") 67 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/schemas/webhook.py: -------------------------------------------------------------------------------- 1 | import typing 2 | from enum import Enum 3 | from typing import Union 4 | 5 | from opal_common.schemas.policy import BaseSchema 6 | from pydantic import Field 7 | 8 | 9 | class SecretTypeEnum(str, Enum): 10 | """Is the passed secret in the webhook a token or a signature on the 11 | request body.""" 12 | 13 | token = "token" 14 | signature = "signature" 15 | 16 | 17 | class GitWebhookRequestParams(BaseSchema): 18 | secret_header_name: str = Field( 19 | ..., 20 | description="The HTTP header holding the secret", 21 | ) 22 | secret_type: SecretTypeEnum = Field( 23 | ..., 24 | description=SecretTypeEnum.__doc__, 25 | ) 26 | secret_parsing_regex: str = Field( 27 | ..., 28 | description="The regex used to parse out the actual signature from the header. Use '(.*)' for the entire value", 29 | ) 30 | event_header_name: typing.Optional[str] = Field( 31 | default=None, 32 | description="The HTTP header holding the event information (used instead of event_request_key)", 33 | ) 34 | event_request_key: typing.Optional[str] = Field( 35 | default=None, 36 | description="The JSON object key holding the event information (used instead of event_header_name)", 37 | ) 38 | push_event_value: str = Field( 39 | ..., 40 | description="The event value indicating a Git push", 41 | ) 42 | match_sender_url: bool = Field( 43 | True, 44 | description="Should OPAL verify that the sender url matches the tracked repo URL, and drop the webhook request otherwise?", 45 | ) 46 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/security/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/security/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/security/sslcontext.py: -------------------------------------------------------------------------------- 1 | import os 2 | import ssl 3 | from typing import Optional 4 | 5 | from opal_common.config import opal_common_config 6 | 7 | 8 | def get_custom_ssl_context() -> Optional[ssl.SSLContext]: 9 | """Potentially (if enabled), returns a custom ssl context that respect 10 | self-signed certificates. 11 | 12 | More accurately, may return an ssl context that respects a local CA 13 | as a valid issuer. 14 | """ 15 | if not opal_common_config.CLIENT_SELF_SIGNED_CERTIFICATES_ALLOWED: 16 | return None 17 | 18 | ca_file: Optional[str] = opal_common_config.CLIENT_SSL_CONTEXT_TRUSTED_CA_FILE 19 | 20 | if ca_file is None: 21 | return None 22 | 23 | if not ca_file: 24 | return None 25 | 26 | ca_file_path = os.path.expanduser(ca_file) 27 | if not os.path.isfile(ca_file_path): 28 | return None 29 | 30 | return ssl.create_default_context(cafile=ca_file_path) 31 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/sources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/sources/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/synchronization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/synchronization/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/synchronization/expiring_redis_lock.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import redis.asyncio as redis 4 | from opal_common.logger import logger 5 | 6 | 7 | async def run_locked( 8 | _redis: redis.Redis, lock_name: str, coro: asyncio.coroutine, timeout: int = 10 9 | ): 10 | """This function runs a coroutine wrapped in a redis lock, in a way that 11 | prevents hanging locks. Hanging locks can happen when a process crashes 12 | while holding a lock. 13 | 14 | This function sets a redis enforced timeout, and reacquires the lock 15 | every timeout * 0.8 (as long as it runs) 16 | """ 17 | lock = _redis.lock(lock_name, timeout=timeout) 18 | try: 19 | logger.debug(f"Trying to acquire redis lock: {lock_name}") 20 | await lock.acquire() 21 | logger.debug(f"Acquired lock: {lock_name}") 22 | 23 | locked_task = asyncio.create_task(coro) 24 | 25 | while True: 26 | done, _ = await asyncio.wait( 27 | (locked_task,), 28 | timeout=timeout * 0.8, 29 | return_when=asyncio.FIRST_COMPLETED, 30 | ) 31 | if locked_task in done: 32 | break 33 | else: 34 | # Extend lock timeout as long as the coroutine is still running 35 | await lock.reacquire() 36 | logger.debug(f"Reacquired lock: {lock_name}") 37 | 38 | finally: 39 | await lock.release() 40 | logger.debug(f"Released lock: {lock_name}") 41 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/tests/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/tests/test_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from opal_client.config import opal_client_config 3 | from opal_common.config import opal_common_config 4 | from opal_server.config import opal_server_config 5 | 6 | 7 | def test_opal_common_config_descriptions(): 8 | for name, entry in opal_common_config.entries.items(): 9 | assert entry.description is not None, f"{name} is missing a description" 10 | 11 | 12 | def test_opal_client_config_descriptions(): 13 | for name, entry in opal_client_config.entries.items(): 14 | assert entry.description is not None, f"{name} is missing a description" 15 | 16 | 17 | def test_opal_server_config_descriptions(): 18 | for name, entry in opal_server_config.entries.items(): 19 | assert entry.description is not None, f"{name} is missing a description" 20 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import requests 4 | 5 | 6 | def wait_for_server(port: int, timeout: int = 2): 7 | """Waits for the http server (of either the server or the client) to be 8 | available.""" 9 | start = time.time() 10 | while time.time() - start < timeout: 11 | try: 12 | # Assumes both server and client have "/" route 13 | requests.get(f"http://localhost:{port}/") 14 | return 15 | except requests.exceptions.ConnectionError: 16 | time.sleep(0.1) 17 | raise TimeoutError(f"Server did not start within {timeout} seconds") 18 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/tests/url_utils_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import pytest 5 | 6 | # Add root opal dir to use local src as package for tests (i.e, no need for python -m pytest) 7 | root_dir = os.path.abspath( 8 | os.path.join( 9 | os.path.dirname(__file__), 10 | os.path.pardir, 11 | os.path.pardir, 12 | ) 13 | ) 14 | sys.path.append(root_dir) 15 | 16 | from pathlib import Path 17 | from typing import List 18 | 19 | from opal_common.urls import set_url_query_param 20 | 21 | 22 | def test_set_url_query_param(): 23 | base_url = "api.permit.io/opal/data/config" 24 | 25 | # https scheme, query string not empty 26 | assert ( 27 | set_url_query_param( 28 | f"https://{base_url}?some=val&other=val2", "token", "secret" 29 | ) 30 | == f"https://{base_url}?some=val&other=val2&token=secret" 31 | ) 32 | 33 | # http scheme, query string empty 34 | assert ( 35 | set_url_query_param(f"http://{base_url}", "token", "secret") 36 | == f"http://{base_url}?token=secret" 37 | ) 38 | 39 | # no scheme, query string empty 40 | assert ( 41 | set_url_query_param(f"{base_url}", "token", "secret") 42 | == f"{base_url}?token=secret" 43 | ) 44 | 45 | # no scheme, query string not empty 46 | assert ( 47 | set_url_query_param(f"{base_url}?some=val", "token", "secret") 48 | == f"{base_url}?some=val&token=secret" 49 | ) 50 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/topics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-common/opal_common/topics/__init__.py -------------------------------------------------------------------------------- /packages/opal-common/opal_common/topics/listener.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Coroutine 2 | 3 | try: 4 | from typing import Protocol 5 | except ImportError: 6 | from typing_extensions import Protocol 7 | 8 | from fastapi_websocket_pubsub import PubSubClient, Topic, TopicList 9 | from opal_common.logger import logger 10 | 11 | 12 | class TopicCallback(Protocol): 13 | def __call__(self, topic: Topic, data: Any) -> Coroutine: 14 | ... 15 | 16 | 17 | class TopicListener: 18 | """A simple wrapper around a PubSubClient that listens on a topic and runs 19 | a callback when messages arrive for that topic. 20 | 21 | Provides start() and stop() shortcuts that helps treat this client 22 | as a separate "process" or task that runs in the background. 23 | """ 24 | 25 | def __init__( 26 | self, 27 | client: PubSubClient, 28 | server_uri: str, 29 | topics: TopicList = None, 30 | callback: TopicCallback = None, 31 | ): 32 | """[summary] 33 | 34 | Args: 35 | client (PubSubClient): a configured not-yet-started pub sub client 36 | server_uri (str): the URI of the pub sub server we subscribe to 37 | topics (TopicList): the topic(s) we subscribe to 38 | callback (TopicCallback): the (async) callback to run when a message 39 | arrive on one of the subsribed topics 40 | """ 41 | self._client = client 42 | self._server_uri = server_uri 43 | self._topics = topics 44 | self._callback = callback 45 | 46 | async def __aenter__(self): 47 | self.start() 48 | return self 49 | 50 | async def __aexit__(self, exc_type, exc, tb): 51 | await self.stop() 52 | 53 | def start(self): 54 | """Starts the pub/sub client and subscribes to the predefined topic. 55 | 56 | the client will attempt to connect to the pubsub server until 57 | successful. 58 | """ 59 | logger.info("started topic listener, topics={topics}", topics=self._topics) 60 | for topic in self._topics: 61 | self._client.subscribe(topic, self._callback) 62 | self._client.start_client(f"{self._server_uri}") 63 | 64 | async def stop(self): 65 | """Stops the pubsub client.""" 66 | await self._client.disconnect() 67 | logger.info("stopped topic listener", topics=self._topics) 68 | 69 | async def wait_until_done(self): 70 | """When the listener is a used as a context manager, this method waits 71 | until the client is done (i.e: terminated) to prevent exiting the 72 | context.""" 73 | return await self._client.wait_until_done() 74 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/topics/utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List 3 | 4 | from opal_common.paths import PathUtils 5 | 6 | POLICY_PREFIX = "policy:" 7 | 8 | 9 | def policy_topics(paths: List[Path]) -> List[str]: 10 | """Prefixes a list of directories with the policy topic prefix.""" 11 | return ["{}{}".format(POLICY_PREFIX, str(path)) for path in paths] 12 | 13 | 14 | def remove_prefix(topic: str, prefix: str = POLICY_PREFIX): 15 | """Removes the policy topic prefix to get the path (directory) encoded in 16 | the topic.""" 17 | if topic.startswith(prefix): 18 | return topic[len(prefix) :] 19 | return topic 20 | 21 | 22 | def pubsub_topics_from_directories(dirs: List[str]) -> List[str]: 23 | """Converts a list of directories on the policy repository that the client 24 | wants to subscribe to into a list of topics. 25 | 26 | this method also ensures the client only subscribes to non- 27 | intersecting directories by dedupping directories that are 28 | descendents of one another. 29 | """ 30 | policy_directories = PathUtils.non_intersecting_directories([Path(d) for d in dirs]) 31 | return policy_topics(policy_directories) 32 | -------------------------------------------------------------------------------- /packages/opal-common/opal_common/urls.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import ParseResult, parse_qsl, urlencode, urlparse, urlunparse 2 | 3 | 4 | def set_url_query_param(url: str, param_name: str, param_value: str): 5 | """Given a url, set or replace a query parameter and return the modified 6 | url. 7 | 8 | >> set_url_query_param('https://api.permit.io/opal/data/config', 'token', 'secret') 9 | 'https://api.permit.io/opal/data/config?token=secret' 10 | 11 | >> set_url_query_param('https://api.permit.io/opal/data/config?some=var', 'token', 'secret') 12 | 'https://api.permit.io/opal/data/config?some=var&token=secret' 13 | """ 14 | parsed_url: ParseResult = urlparse(url) 15 | 16 | query_params: dict = dict(parse_qsl(parsed_url.query)) 17 | query_params[param_name] = param_value 18 | new_query_string = urlencode(query_params) 19 | 20 | return urlunparse( 21 | ( 22 | parsed_url.scheme, 23 | parsed_url.netloc, 24 | parsed_url.path, 25 | parsed_url.params, 26 | new_query_string, 27 | parsed_url.fragment, 28 | ) 29 | ) 30 | -------------------------------------------------------------------------------- /packages/opal-common/requires.txt: -------------------------------------------------------------------------------- 1 | aiohttp>=3.9.2,<4 2 | click>=8.1.3,<9 3 | cryptography>=42.0.4,<43 4 | gitpython>=3.1.32,<4 5 | loguru>=0.6.0,<1 6 | pyjwt[crypto]>=2.4.0,<3 7 | python-decouple>=3.6,<4 8 | tenacity>=8.0.1,<9 9 | datadog>=0.44.0, <1 10 | ddtrace>=2.8.1,<3 11 | certifi>=2023.7.22 # not directly required, pinned by Snyk to avoid a vulnerability 12 | requests>=2.32.0 # not directly required, pinned by Snyk to avoid a vulnerability 13 | httpx>=0.27.0 14 | urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability 15 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-server/opal_server/__init__.py -------------------------------------------------------------------------------- /packages/opal-server/opal_server/cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import typer 5 | from click.core import Context 6 | from fastapi.applications import FastAPI 7 | from typer.main import Typer 8 | 9 | # Add parent path to use local src as package for tests 10 | root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) 11 | sys.path.append(root_dir) 12 | 13 | from opal_common.cli.docs import MainTexts 14 | from opal_common.cli.typer_app import get_typer_app 15 | from opal_common.config import opal_common_config 16 | from opal_common.corn_utils import run_gunicorn, run_uvicorn 17 | from opal_server.config import opal_server_config 18 | 19 | app = get_typer_app() 20 | 21 | 22 | @app.command() 23 | def run(engine_type: str = typer.Option("uvicron", help="uvicorn or gunicorn")): 24 | """Run the server as a daemon.""" 25 | typer.echo(f"-- Starting OPAL Server (with {engine_type}) --") 26 | 27 | if engine_type == "gunicorn": 28 | app: FastAPI 29 | from opal_server.main import app 30 | 31 | run_gunicorn( 32 | app, 33 | opal_server_config.SERVER_WORKER_COUNT, 34 | host=opal_server_config.SERVER_HOST, 35 | port=opal_server_config.SERVER_BIND_PORT, 36 | ) 37 | else: 38 | run_uvicorn( 39 | "opal_server.main:app", 40 | workers=opal_server_config.SERVER_WORKER_COUNT, 41 | host=opal_server_config.SERVER_HOST, 42 | port=opal_server_config.SERVER_BIND_PORT, 43 | ) 44 | 45 | 46 | @app.command() 47 | def print_config(): 48 | """To test config values, print the configuration parsed from ENV and 49 | CMD.""" 50 | typer.echo("Printing configuration values") 51 | typer.echo(str(opal_server_config)) 52 | typer.echo(str(opal_common_config)) 53 | 54 | 55 | def cli(): 56 | main_texts = MainTexts("💎 OPAL-SERVER 💎", "server") 57 | 58 | def on_start(ctx: Context, **kwargs): 59 | if ctx.invoked_subcommand is None or ctx.invoked_subcommand == "run": 60 | typer.secho(main_texts.header, bold=True, fg=typer.colors.MAGENTA) 61 | if ctx.invoked_subcommand is None: 62 | typer.echo(ctx.get_usage()) 63 | typer.echo(main_texts.docs) 64 | 65 | opal_server_config.cli( 66 | [opal_common_config], typer_app=app, help=main_texts.docs, on_start=on_start 67 | ) 68 | 69 | 70 | if __name__ == "__main__": 71 | cli() 72 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-server/opal_server/data/__init__.py -------------------------------------------------------------------------------- /packages/opal-server/opal_server/data/tests/test_data_update_publisher.py: -------------------------------------------------------------------------------- 1 | from opal_server.data.data_update_publisher import DataUpdatePublisher 2 | 3 | 4 | def test_topic_combos(): 5 | get_topic_combos = DataUpdatePublisher.get_topic_combos 6 | 7 | assert set(get_topic_combos("a/b/c")) == {"a", "a/b", "a/b/c"} 8 | assert set(get_topic_combos("x:a/b/c")) == {"x:a", "x:a/b", "x:a/b/c"} 9 | assert set(get_topic_combos("x:y:a/b/c")) == {"x:y:a", "x:y:a/b", "x:y:a/b/c"} 10 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/loadlimiting.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Request 2 | from opal_common.logger import logger 3 | from slowapi import Limiter 4 | 5 | 6 | def init_loadlimit_router(loadlimit_notation: str = None): 7 | """Initializes a route where a client (or any other network peer) can 8 | inquire what opal clients are currently connected to the server and on what 9 | topics are they registered. 10 | 11 | If the OPAL server does not have statistics enabled, the route will 12 | return 501 Not Implemented 13 | """ 14 | router = APIRouter() 15 | 16 | # We want to globally limit the endpoint, not per client 17 | limiter = Limiter(key_func=lambda: "global") 18 | 19 | if loadlimit_notation: 20 | logger.info(f"rate limiting is on, configured limit: {loadlimit_notation}") 21 | 22 | @router.get("/loadlimit") 23 | @limiter.limit(loadlimit_notation) 24 | async def loadlimit(request: Request): 25 | return 26 | 27 | else: 28 | 29 | @router.get("/loadlimit") 30 | async def loadlimit(request: Request): 31 | return 32 | 33 | return router 34 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/main.py: -------------------------------------------------------------------------------- 1 | def create_app(*args, **kwargs): 2 | from opal_server.server import OpalServer 3 | 4 | server = OpalServer(*args, **kwargs) 5 | return server.app 6 | 7 | 8 | app = create_app() 9 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/policy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-server/opal_server/policy/__init__.py -------------------------------------------------------------------------------- /packages/opal-server/opal_server/policy/bundles/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-server/opal_server/policy/bundles/__init__.py -------------------------------------------------------------------------------- /packages/opal-server/opal_server/policy/watcher/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-server/opal_server/policy/watcher/__init__.py -------------------------------------------------------------------------------- /packages/opal-server/opal_server/policy/webhook/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-server/opal_server/policy/webhook/__init__.py -------------------------------------------------------------------------------- /packages/opal-server/opal_server/policy/webhook/listener.py: -------------------------------------------------------------------------------- 1 | """The webhook listener listens on the `webhook` topic. the reason we need it, 2 | is because the uvicorn worker that serves the webhook HTTP request from github 3 | is not necessarily the leader worker that runs the repo watcher. 4 | 5 | The solution is quite simply: the worker that serves the request simply 6 | publishes on the `webhook` topic, and the listener's callback is 7 | triggered. 8 | """ 9 | 10 | from fastapi_websocket_pubsub.pub_sub_client import PubSubClient, Topic 11 | from opal_common.confi.confi import load_conf_if_none 12 | from opal_common.topics.listener import TopicCallback, TopicListener 13 | from opal_common.utils import get_authorization_header 14 | from opal_server.config import opal_server_config 15 | 16 | 17 | def setup_webhook_listener( 18 | callback: TopicCallback, 19 | server_uri: str = None, 20 | server_token: str = None, 21 | topic: Topic = "webhook", 22 | ) -> TopicListener: 23 | # load defaults 24 | server_uri = load_conf_if_none(server_uri, opal_server_config.OPAL_WS_LOCAL_URL) 25 | server_token = load_conf_if_none(server_token, opal_server_config.OPAL_WS_TOKEN) 26 | 27 | return TopicListener( 28 | client=PubSubClient(extra_headers=[get_authorization_header(server_token)]), 29 | server_uri=server_uri, 30 | topics=[topic], 31 | callback=callback, 32 | ) 33 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/publisher.py: -------------------------------------------------------------------------------- 1 | from fastapi_websocket_pubsub import PubSubClient, Topic 2 | from opal_common.confi.confi import load_conf_if_none 3 | from opal_common.topics.publisher import ( 4 | ClientSideTopicPublisher, 5 | PeriodicPublisher, 6 | ServerSideTopicPublisher, 7 | TopicPublisher, 8 | ) 9 | from opal_common.utils import get_authorization_header 10 | from opal_server.config import opal_server_config 11 | 12 | 13 | def setup_publisher_task( 14 | server_uri: str = None, 15 | server_token: str = None, 16 | ) -> TopicPublisher: 17 | server_uri = load_conf_if_none( 18 | server_uri, 19 | opal_server_config.OPAL_WS_LOCAL_URL, 20 | ) 21 | server_token = load_conf_if_none( 22 | server_token, 23 | opal_server_config.OPAL_WS_TOKEN, 24 | ) 25 | return ClientSideTopicPublisher( 26 | client=PubSubClient(extra_headers=[get_authorization_header(server_token)]), 27 | server_uri=server_uri, 28 | ) 29 | 30 | 31 | def setup_broadcaster_keepalive_task( 32 | publisher: ServerSideTopicPublisher, 33 | time_interval: int, 34 | topic: Topic = "__broadcast_session_keepalive__", 35 | ) -> PeriodicPublisher: 36 | """A periodic publisher with the intent to trigger messages on the 37 | broadcast channel, so that the session to the backbone won't become idle 38 | and close on the backbone end.""" 39 | return PeriodicPublisher( 40 | publisher, time_interval, topic, task_name="broadcaster keepalive task" 41 | ) 42 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/redis_utils.py: -------------------------------------------------------------------------------- 1 | from typing import Generator 2 | 3 | import redis.asyncio as redis 4 | from opal_common.logger import logger 5 | from pydantic import BaseModel 6 | 7 | 8 | class RedisDB: 9 | """Small utility class to persist objects in Redis.""" 10 | 11 | def __init__(self, redis_url): 12 | self._url = redis_url 13 | logger.debug("Connecting to Redis: {url}", url=self._url) 14 | 15 | self._redis = redis.Redis.from_url(self._url) 16 | 17 | @property 18 | def redis_connection(self) -> redis.Redis: 19 | return self._redis 20 | 21 | async def set(self, key: str, value: BaseModel): 22 | await self._redis.set(key, self._serialize(value)) 23 | 24 | async def set_if_not_exists(self, key: str, value: BaseModel) -> bool: 25 | """:param key: 26 | :param value: 27 | :return: True if created, False if key already exists 28 | """ 29 | 30 | return await self._redis.set(key, self._serialize(value), nx=True) 31 | 32 | async def get(self, key: str) -> bytes: 33 | return await self._redis.get(key) 34 | 35 | async def scan(self, pattern: str) -> Generator[bytes, None, None]: 36 | cur = b"0" 37 | while cur: 38 | cur, keys = await self._redis.scan(cur, match=pattern) 39 | 40 | for key in keys: 41 | value = await self._redis.get(key) 42 | yield value 43 | 44 | async def delete(self, key: str): 45 | await self._redis.delete(key) 46 | 47 | def _serialize(self, value: BaseModel) -> str: 48 | return value.json() 49 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/scopes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-server/opal_server/scopes/__init__.py -------------------------------------------------------------------------------- /packages/opal-server/opal_server/scopes/loader.py: -------------------------------------------------------------------------------- 1 | from opal_common.logger import logger 2 | from opal_common.schemas.policy_source import ( 3 | GitPolicyScopeSource, 4 | NoAuthData, 5 | SSHAuthData, 6 | ) 7 | from opal_common.schemas.scopes import Scope 8 | from opal_server.config import ServerRole, opal_server_config 9 | from opal_server.scopes.scope_repository import ScopeRepository 10 | 11 | DEFAULT_SCOPE_ID = "default" 12 | 13 | 14 | async def load_scopes(repo: ScopeRepository): 15 | logger.info("Server is primary, loading default scope.") 16 | await _load_env_scope(repo) 17 | 18 | 19 | async def _load_env_scope(repo: ScopeRepository): 20 | # backwards compatible opal scope 21 | if opal_server_config.POLICY_REPO_URL is not None: 22 | logger.info( 23 | "Adding default scope from env: {url}", 24 | url=opal_server_config.POLICY_REPO_URL, 25 | ) 26 | 27 | auth = NoAuthData() 28 | 29 | if opal_server_config.POLICY_REPO_SSH_KEY is not None: 30 | private_ssh_key = opal_server_config.POLICY_REPO_SSH_KEY 31 | private_ssh_key = private_ssh_key.replace("_", "\n") 32 | 33 | if not private_ssh_key.endswith("\n"): 34 | private_ssh_key += "\n" 35 | 36 | auth = SSHAuthData(username="git", private_key=private_ssh_key) 37 | 38 | scope = Scope( 39 | scope_id=DEFAULT_SCOPE_ID, 40 | policy=GitPolicyScopeSource( 41 | source_type=opal_server_config.POLICY_SOURCE_TYPE.lower(), 42 | url=opal_server_config.POLICY_REPO_URL, 43 | manifest=opal_server_config.POLICY_REPO_MANIFEST_PATH, 44 | branch=opal_server_config.POLICY_REPO_MAIN_BRANCH, 45 | auth=auth, 46 | ), 47 | ) 48 | 49 | await repo.put(scope) 50 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/scopes/scope_repository.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from opal_common.schemas.scopes import Scope 4 | from opal_server.redis_utils import RedisDB 5 | 6 | 7 | class ScopeNotFoundError(Exception): 8 | def __init__(self, id: str): 9 | self._id = id 10 | 11 | def __str__(self) -> str: 12 | return f"Scope {self._id} not found" 13 | 14 | 15 | class ScopeRepository: 16 | def __init__(self, redis_db: RedisDB): 17 | self._redis_db = redis_db 18 | self._prefix = "permit.io/Scope" 19 | 20 | @property 21 | def db(self) -> RedisDB: 22 | return self._redis_db 23 | 24 | async def all(self) -> List[Scope]: 25 | scopes = [] 26 | 27 | async for value in self._redis_db.scan(f"{self._prefix}:*"): 28 | scope = Scope.parse_raw(value) 29 | scopes.append(scope) 30 | 31 | return scopes 32 | 33 | async def get(self, scope_id: str) -> Scope: 34 | key = self._redis_key(scope_id) 35 | value = await self._redis_db.get(key) 36 | 37 | if value: 38 | return Scope.parse_raw(value) 39 | else: 40 | raise ScopeNotFoundError(scope_id) 41 | 42 | async def put(self, scope: Scope): 43 | key = self._redis_key(scope.scope_id) 44 | await self._redis_db.set(key, scope) 45 | 46 | async def delete(self, scope_id: str): 47 | key = self._redis_key(scope_id) 48 | await self._redis_db.delete(key) 49 | 50 | def _redis_key(self, scope_id: str): 51 | return f"{self._prefix}:{scope_id}" 52 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/security/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/permitio/opal/3ac430e12b75cf770655562a1edd6f99581ca20a/packages/opal-server/opal_server/security/__init__.py -------------------------------------------------------------------------------- /packages/opal-server/opal_server/security/api.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from fastapi import APIRouter, Depends, HTTPException, status 4 | from opal_common.authentication.deps import StaticBearerAuthenticator 5 | from opal_common.authentication.signer import JWTSigner 6 | from opal_common.logger import logger 7 | from opal_common.schemas.security import AccessToken, AccessTokenRequest, TokenDetails 8 | 9 | 10 | def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthenticator): 11 | router = APIRouter() 12 | 13 | @router.post( 14 | "/token", 15 | status_code=status.HTTP_200_OK, 16 | response_model=AccessToken, 17 | dependencies=[Depends(authenticator)], 18 | ) 19 | async def generate_new_access_token(req: AccessTokenRequest): 20 | if not signer.enabled: 21 | raise HTTPException( 22 | status_code=status.HTTP_503_SERVICE_UNAVAILABLE, 23 | detail="opal server was not configured with security, cannot generate tokens!", 24 | ) 25 | 26 | claims = {"peer_type": req.type.value, **req.claims} 27 | token = signer.sign(sub=req.id, token_lifetime=req.ttl, custom_claims=claims) 28 | logger.info(f"Generated opal token: peer_type={req.type.value}") 29 | return AccessToken( 30 | token=token, 31 | details=TokenDetails( 32 | id=req.id, 33 | type=req.type, 34 | expired=datetime.utcnow() + req.ttl, 35 | claims=claims, 36 | ), 37 | ) 38 | 39 | return router 40 | -------------------------------------------------------------------------------- /packages/opal-server/opal_server/security/jwks.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | 4 | from fastapi import FastAPI 5 | from fastapi.staticfiles import StaticFiles 6 | from opal_common.authentication.signer import JWTSigner 7 | 8 | 9 | class JwksStaticEndpoint: 10 | """Configure a static files endpoint on a fastapi app, exposing JWKs.""" 11 | 12 | def __init__( 13 | self, 14 | signer: JWTSigner, 15 | jwks_url: str, 16 | jwks_static_dir: str, 17 | ): 18 | self._signer = signer 19 | self._jwks_url = Path(jwks_url) 20 | self._jwks_static_dir = Path(jwks_static_dir) 21 | 22 | def configure_app(self, app: FastAPI): 23 | # create the directory in which the jwks.json file should sit 24 | self._jwks_static_dir.mkdir(parents=True, exist_ok=True) 25 | 26 | # get the jwks contents from the signer 27 | jwks_contents = {} 28 | if self._signer.enabled: 29 | jwk = json.loads(self._signer.get_jwk()) 30 | jwks_contents = {"keys": [jwk]} 31 | 32 | # write the jwks.json file 33 | filename = self._jwks_static_dir / self._jwks_url.name 34 | with open(filename, "w") as f: 35 | f.write(json.dumps(jwks_contents)) 36 | 37 | route_url = str(self._jwks_url.parent) 38 | app.mount( 39 | route_url, 40 | StaticFiles(directory=str(self._jwks_static_dir)), 41 | name="jwks_dir", 42 | ) 43 | -------------------------------------------------------------------------------- /packages/opal-server/requires.txt: -------------------------------------------------------------------------------- 1 | click>=8.1.3,<9 2 | permit-broadcaster[postgres,redis,kafka]==0.2.6 3 | gitpython>=3.1.32,<4 4 | pyjwt[crypto]>=2.1.0,<3 5 | slowapi>=0.1.5,<1 6 | # slowapi is stuck on and old `redis`, so fix that and switch from aioredis to redis 7 | pygit2>=1.14.1,<1.15 8 | asgiref>=3.5.2,<4 9 | redis>=4.3.4,<5 10 | -------------------------------------------------------------------------------- /packages/requires.txt: -------------------------------------------------------------------------------- 1 | idna>=3.3,<4 2 | typer>=0.4.1,<1 3 | fastapi>=0.109.1,<1 4 | fastapi_websocket_pubsub==0.3.7 5 | fastapi_websocket_rpc==0.1.28 6 | websockets>=10.3,<14 7 | gunicorn>=22.0.0,<23 8 | pydantic[email]>=1.9.1,<2 9 | typing-extensions;python_version<'3.8' 10 | starlette>=0.40.0,<1 11 | uvicorn[standard]>=0.17.6,<1 12 | fastapi-utils>=0.2.1,<1 13 | setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability 14 | anyio>=4.4.0 # not directly required, pinned by Snyk to avoid a vulnerability 15 | starlette>=0.40.0 # not directly required, pinned by Snyk to avoid a vulnerability 16 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # Handling DeprecationWarning 'asyncio_mode' default value 2 | [pytest] 3 | asyncio_mode = strict 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e ./packages/opal-common 2 | -e ./packages/opal-client 3 | -e ./packages/opal-server 4 | ipython>=8.10.0 5 | pytest 6 | pytest-asyncio 7 | pytest-rerunfailures 8 | wheel>=0.38.0 9 | twine 10 | setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability 11 | zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability 12 | -------------------------------------------------------------------------------- /scripts/gunicorn_conf.py: -------------------------------------------------------------------------------- 1 | from opal_common.logger import logger 2 | 3 | 4 | def post_fork(server, worker): 5 | """This hook takes effect if we are using gunicorn to run OPAL.""" 6 | pass 7 | 8 | 9 | def when_ready(server): 10 | try: 11 | import opal_server.scopes.task 12 | except ImportError: 13 | # Not opal server 14 | return 15 | 16 | opal_server.scopes.task.ScopesPolicyWatcherTask.preload_scopes() 17 | logger.warning("Finished pre loading scopes...") 18 | -------------------------------------------------------------------------------- /scripts/start.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | export GUNICORN_CONF=${GUNICORN_CONF:-./gunicorn_conf.py} 5 | export GUNICORN_TIMEOUT=${GUNICORN_TIMEOUT:-30} 6 | export GUNICORN_KEEP_ALIVE_TIMEOUT=${GUNICORN_KEEP_ALIVE_TIMEOUT:-5} 7 | 8 | if [[ -z "${OPAL_BROADCAST_URI}" && "${UVICORN_NUM_WORKERS}" != "1" ]]; then 9 | echo "OPAL_BROADCAST_URI must be set when having multiple workers" 10 | exit 1 11 | fi 12 | 13 | prefix="" 14 | # Start Gunicorn 15 | if [[ -z "${OPAL_ENABLE_DATADOG_APM}" && "${OPAL_ENABLE_DATADOG_APM}" = "true" ]]; then 16 | prefix=ddtrace-run 17 | fi 18 | (set -x; exec $prefix gunicorn -b 0.0.0.0:${UVICORN_PORT} -k uvicorn.workers.UvicornWorker --workers=${UVICORN_NUM_WORKERS} -c ${GUNICORN_CONF} ${UVICORN_ASGI_APP} -t ${GUNICORN_TIMEOUT} --keep-alive ${GUNICORN_KEEP_ALIVE_TIMEOUT}) 19 | -------------------------------------------------------------------------------- /semver2pypi.py: -------------------------------------------------------------------------------- 1 | """Converts a semver version into a version for PyPI package 2 | https://peps.python.org/pep-0440/ 3 | 4 | A semver prerelease will be converted into prerelease of PyPI. 5 | A semver build will be converted into a development part of PyPI 6 | 7 | Usage: 8 | python semver2pypi.py 0.1.0-rc1 9 | 0.1.0rc1 10 | python semver2pypi.py 0.1.0-dev1 11 | 0.1.0.dev1 12 | python semver2pypi.py 0.1.0 13 | 0.1.0 14 | """ 15 | 16 | import sys 17 | 18 | from packaging.version import Version as PyPIVersion 19 | from semver import Version as SemVerVersion 20 | 21 | semver_version = SemVerVersion.parse(sys.argv[1]) 22 | finalized_version = semver_version.finalize_version() 23 | prerelease = semver_version.prerelease or "" 24 | build = semver_version.build or "" 25 | pypi_version = PyPIVersion(f"{finalized_version}{prerelease}{build}") 26 | print(pypi_version) 27 | --------------------------------------------------------------------------------