├── docs ├── template.j2 └── post_render.py ├── tests ├── __init__.py ├── test_query.py └── test_httpclient.py ├── pan_cortex_data_lake ├── adapters │ ├── __init__.py │ ├── adapter.py │ └── tinydb_adapter.py ├── __init__.py ├── utils.py ├── exceptions.py ├── query.py ├── httpclient.py └── credentials.py ├── .editorconfig ├── tox.ini ├── .github ├── ISSUE_TEMPLATE.md └── workflows │ ├── pypi_publish.yml │ ├── test.yml │ ├── pydocs-publish.yml │ └── pydocs-preview.yml ├── LICENSE ├── SUPPORT.md ├── .gitignore ├── examples ├── credentials_remove.py ├── query_all.py └── credentials_generate.py ├── pydoc-markdown.yml ├── pyproject.toml ├── CONTRIBUTING.md └── README.md /docs/template.j2: -------------------------------------------------------------------------------- 1 | custom_edit_url: {{ custom_edit_url }} -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Unit test package for pancloud.""" 4 | -------------------------------------------------------------------------------- /pan_cortex_data_lake/adapters/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Adapters package.""" 4 | 5 | from .adapter import StorageAdapter # noqa: F401 6 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.py] 14 | max_line_length = 79 15 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35, py36, py37, py38, pypy3 flake8 3 | skipsdist = True 4 | 5 | [testenv:flake8] 6 | basepython=python 7 | deps=flake8 8 | commands=flake8 pan_cortex_data_lake 9 | 10 | [testenv] 11 | setenv = 12 | PYTHONPATH = {toxinidir} 13 | commands = 14 | pip install -U pip 15 | pip install .[test] 16 | py.test -v ./tests --basetemp={envtmpdir} 17 | 18 | [flake8] 19 | ignore = E501, E402 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | - Cortex Data Lake Python SDK version: 2 | - Python version: 3 | - Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /pan_cortex_data_lake/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Python idiomatic SDK for Cortex™ Data Lake.""" 4 | 5 | __author__ = "Palo Alto Networks" 6 | __version__ = "2.0.0b1" 7 | 8 | from .exceptions import ( # noqa: F401 9 | CortexError, 10 | HTTPError, 11 | UnexpectedKwargsError, 12 | RequiredKwargsError, 13 | ) 14 | from .httpclient import HTTPClient # noqa: F401 15 | from .credentials import Credentials # noqa: F401 16 | from .query import QueryService # noqa: F401 17 | -------------------------------------------------------------------------------- /.github/workflows/pypi_publish.yml: -------------------------------------------------------------------------------- 1 | name: PyPI Publish 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | deploy: 9 | if: github.repository_owner == 'PaloAltoNetworks' 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Set up Python 14 | uses: actions/setup-python@v1 15 | with: 16 | python-version: '3.x' 17 | - name: Install dependencies 18 | run: | 19 | python -m pip install --upgrade pip 20 | pip install flit 21 | - name: Build and publish 22 | env: 23 | FLIT_USERNAME: ${{ secrets.PYPI_USERNAME }} 24 | FLIT_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 25 | run: | 26 | flit publish 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | ISC License 3 | 4 | Copyright (c) 2020, Palo Alto Networks 5 | 6 | Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 9 | 10 | -------------------------------------------------------------------------------- /docs/post_render.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | from editfrontmatter import EditFrontMatter 6 | import os 7 | import glob 8 | 9 | 10 | def main(): 11 | template_str = "".join(open(os.path.abspath("docs/template.j2"), "r").readlines()) 12 | md_files = glob.glob("./docs/**/*.md", recursive=True) 13 | for path in md_files: 14 | github_base_url = "https://github.com/PaloAltoNetworks/pan-cortex-data-lake-python/blob/master/pan_cortex_data_lake" 15 | github_file_path = path.split("/reference")[1].replace(".md", ".py") 16 | custom_edit_url = f"{github_base_url}{github_file_path}" # noqa: E999 17 | proc = EditFrontMatter(file_path=path, template_str=template_str) 18 | proc.run({"custom_edit_url": custom_edit_url}) 19 | proc.writeFile(path) 20 | 21 | 22 | if __name__ == "__main__": 23 | main() 24 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | Community Supported 2 | 3 | The software and templates in the repo are released under an as-is, best effort, 4 | support policy. This software should be seen as community supported and Palo 5 | Alto Networks will contribute our expertise as and when possible. We do not 6 | provide technical support or help in using or troubleshooting the components of 7 | the project through our normal support options such as Palo Alto Networks 8 | support teams, or ASC (Authorized Support Centers) partners and backline support 9 | options. The underlying product used (the VM-Series firewall) by the scripts or 10 | templates are still supported, but the support is only for the product 11 | functionality and not for help in deploying or using the template or script 12 | itself. Unless explicitly tagged, all projects or work posted in our GitHub 13 | repository (at https://github.com/PaloAltoNetworks) or sites other than our 14 | official Downloads page on https://support.paloaltonetworks.com are provided 15 | under the best effort policy. 16 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request_target: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | build: 13 | if: github.repository_owner == 'PaloAltoNetworks' 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | python-version: [3.5, 3.6, 3.7, 3.8, 3.9] 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v1 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | pip install .[test] 29 | - name: Lint with flake8 30 | run: | 31 | # stop the build if there are Python syntax errors or undefined names 32 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 33 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 34 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 35 | - name: Test with pytest 36 | run: | 37 | pytest 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | .pytest_cache/ 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | 56 | # Sphinx documentation 57 | docs/_build/ 58 | 59 | # PyBuilder 60 | target/ 61 | 62 | # pyenv python configuration file 63 | .python-version 64 | 65 | # user-defined stuff 66 | *.p12 67 | .DS_Store 68 | .envrc 69 | 70 | # IDE stuff 71 | .idea/* 72 | .vscode/* 73 | 74 | # pydoc-markdown 75 | docs/**/*.md -------------------------------------------------------------------------------- /pan_cortex_data_lake/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | :::info 5 | PAN CDL Python SDK utilities. 6 | ::: 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | 12 | import logging # noqa: F401 13 | 14 | 15 | class ApiStats(dict): 16 | """Object for storing, updating and retrieving API stats.""" 17 | 18 | def __init__(self, *args, **kwargs): 19 | super(ApiStats, self).__init__(*args, **kwargs) 20 | self.transactions = 0 21 | for arg in args: 22 | if isinstance(arg, dict): 23 | for k, v in arg.items(): 24 | self[k] = v 25 | 26 | if kwargs: 27 | for k, v in kwargs.items(): 28 | self[k] = v 29 | 30 | def __getattr__(self, attr): 31 | return self.get(attr) 32 | 33 | def __setattr__(self, key, value): 34 | self.__setitem__(key, value) 35 | 36 | def __setitem__(self, key, value): 37 | super(ApiStats, self).__setitem__(key, value) 38 | self.__dict__.update({key: value}) 39 | 40 | def __delattr__(self, item): 41 | self.__delitem__(item) 42 | 43 | def __delitem__(self, key): 44 | super(ApiStats, self).__delitem__(key) 45 | del self.__dict__[key] 46 | -------------------------------------------------------------------------------- /tests/test_query.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Tests for LoggingService.""" 5 | 6 | import os 7 | import sys 8 | 9 | import pytest 10 | 11 | curpath = os.path.dirname(os.path.abspath(__file__)) 12 | sys.path[:0] = [os.path.join(curpath, os.pardir)] 13 | 14 | from pan_cortex_data_lake.query import QueryService 15 | from pan_cortex_data_lake.httpclient import HTTPClient 16 | from pan_cortex_data_lake.exceptions import UnexpectedKwargsError 17 | 18 | 19 | HTTPBIN = os.environ.get("HTTPBIN_URL", "http://httpbin.org") 20 | TARPIT = os.environ.get("TARPIT", "http://10.255.255.1") 21 | 22 | 23 | class TestQueryService: 24 | def test_entry_points(self): 25 | 26 | QueryService(url=TARPIT).session 27 | QueryService(url=TARPIT).kwargs 28 | QueryService(url=TARPIT).url 29 | QueryService(url=TARPIT).cancel_job 30 | QueryService(url=TARPIT).create_query 31 | QueryService(url=TARPIT).get_job 32 | QueryService(url=TARPIT).get_job_results 33 | QueryService(url=TARPIT).iter_job_results 34 | QueryService(url=TARPIT).list_jobs 35 | 36 | def test_unexpected_kwargs(self): 37 | with pytest.raises(UnexpectedKwargsError): 38 | QueryService(url=TARPIT, foo="foo") 39 | 40 | def test_session(self): 41 | session = HTTPClient(url=TARPIT) 42 | QueryService(session=session) 43 | -------------------------------------------------------------------------------- /examples/credentials_remove.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Remove credentials profile.""" 5 | 6 | import os 7 | import sys 8 | 9 | from builtins import input 10 | 11 | curpath = os.path.dirname(os.path.abspath(__file__)) 12 | sys.path[:0] = [os.path.join(curpath, os.pardir)] 13 | 14 | from pan_cortex_data_lake import Credentials 15 | 16 | 17 | def confirm_delete(profile): 18 | """Prompt user to enter Y or N (case-insensitive) to continue.""" 19 | answer = "" 20 | while answer not in ["y", "n"]: 21 | answer = input("Delete PROFILE '%s' [Y/N]? " % profile).lower() 22 | return answer == "y" 23 | 24 | 25 | def main(): 26 | try: 27 | profile = input("PROFILE to remove: ") or None 28 | if profile is not None: 29 | c = Credentials(profile=profile) 30 | if confirm_delete(profile): 31 | print("Removing PROFILE '%s'..." % profile) 32 | op = c.remove_profile(profile) 33 | if len(op) > 0: 34 | print("\nPROFILE '%s' successfully removed.\n" % profile) 35 | else: 36 | print("\nPROFILE '%s' not found.\n" % profile) 37 | else: 38 | print("\nRemove PROFILE operation aborted.\n") 39 | else: 40 | print("\nMust specify a PROFILE to remove.\n") 41 | except KeyboardInterrupt: 42 | print("Exiting...") 43 | 44 | 45 | if __name__ == "__main__": 46 | main() 47 | -------------------------------------------------------------------------------- /examples/query_all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Query Service example SDK usage.""" 5 | 6 | import os 7 | import sys 8 | import time 9 | import logging 10 | 11 | # Necessary to reference cortex package in relative path 12 | curpath = os.path.dirname(os.path.abspath(__file__)) 13 | sys.path[:0] = [os.path.join(curpath, os.pardir)] 14 | 15 | from pan_cortex_data_lake import Credentials, QueryService 16 | 17 | url = "https://api.us.cdl.paloaltonetworks.com" # prod us 18 | 19 | # Create Credentials instance 20 | # export PAN_DEVELOPER_TOKEN for quick access 21 | c = Credentials() 22 | 23 | # Create Query Service instance 24 | qs = QueryService(url=url, force_trace=True, credentials=c) 25 | 26 | # SQL = 'SELECT * FROM `2020001.firewall.traffic` LIMIT 100' 27 | SQL = "SELECT * FROM `4199400902993631660.firewall.traffic` LIMIT 1" 28 | 29 | # Generate new 'query' 30 | query_params = {"query": SQL} 31 | 32 | q = qs.create_query(query_params=query_params) 33 | 34 | print("QUERY Params: {}\n".format(query_params)) 35 | 36 | print("QUERY HTTP STATUS CODE: {}\n".format(q.status_code)) 37 | 38 | print("QUERY Response: {}\n".format(q.text)) 39 | 40 | job_id = q.json()["jobId"] # access 'jobId' from 'query' response 41 | 42 | # Iterate through job results (pages) 43 | print("Iterate through job results: \n") 44 | for p in qs.iter_job_results(job_id=job_id, result_format="valuesDictionary"): 45 | print("RESULTS: {}\n".format(p.text)) 46 | 47 | print("STATS: {}".format(qs.stats)) 48 | -------------------------------------------------------------------------------- /pydoc-markdown.yml: -------------------------------------------------------------------------------- 1 | loaders: 2 | - type: python 3 | search_path: [pan_cortex_data_lake] 4 | processors: 5 | - type: filter 6 | skip_empty_modules: true 7 | - type: smart 8 | - type: crossref 9 | renderer: 10 | type: docusaurus 11 | docs_base_path: docs 12 | relative_output_path: develop/reference 13 | relative_sidebar_path: sidebar.json 14 | sidebar_top_level_label: SDK Reference 15 | markdown: 16 | render_module_header_template: | 17 | --- 18 | sidebar_label: {relative_module_name} 19 | title: {module_name} 20 | hide_title: true 21 | --- 22 | header_level_by_type: 23 | Class: 2 24 | Function: 3 25 | Method: 3 26 | Module: 1 27 | Data: 3 28 | code_headers: false 29 | descriptive_class_title: false 30 | descriptive_module_title: false 31 | add_module_prefix: false 32 | add_method_class_prefix: false 33 | add_member_class_prefix: false 34 | add_full_prefix: false 35 | data_code_block: true 36 | classdef_code_block: false 37 | classdef_with_decorators: false 38 | signature_in_header: false 39 | signature_python_help_style: false 40 | signature_with_vertical_bar: false 41 | signature_with_def: false 42 | signature_class_prefix: false 43 | docstrings_as_blockquote: false 44 | render_module_header: true 45 | classdef_render_init_signature_if_needed: false 46 | escape_html_in_docstring: false 47 | hooks: 48 | post-render: 49 | - ./docs/post_render.py 50 | 51 | -------------------------------------------------------------------------------- /examples/credentials_generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Generate credentials file.""" 5 | 6 | import getpass 7 | import os 8 | import sys 9 | 10 | from builtins import input 11 | 12 | curpath = os.path.dirname(os.path.abspath(__file__)) 13 | sys.path[:0] = [os.path.join(curpath, os.pardir)] 14 | 15 | from pan_cortex_data_lake import Credentials 16 | 17 | 18 | def confirm_write(profile): 19 | """Prompt user to enter Y or N (case-insensitive) to continue.""" 20 | answer = "" 21 | while answer not in ["y", "n"]: 22 | answer = input("\nWrite credentials to PROFILE '%s' [Y/N]? " % profile).lower() 23 | return answer == "y" 24 | 25 | 26 | def main(): 27 | try: 28 | print("\nCollecting info needed to generate credentials file...\n") 29 | client_id = input("CLIENT_ID: ") 30 | client_secret = getpass.getpass(prompt="CLIENT_SECRET: ") 31 | refresh_token = getpass.getpass(prompt="REFRESH_TOKEN: ") 32 | profile = input("PROFILE [default]: ") or None 33 | c = Credentials( 34 | client_id=client_id, 35 | client_secret=client_secret, 36 | refresh_token=refresh_token, 37 | profile=profile, 38 | ) 39 | if confirm_write(profile): 40 | print("Writing credentials file...") 41 | c.write_credentials() 42 | print("Done!\n") 43 | else: 44 | print("\nWrite credentials operation aborted.\n") 45 | except KeyboardInterrupt: 46 | print("Exiting...") 47 | 48 | 49 | if __name__ == "__main__": 50 | main() 51 | -------------------------------------------------------------------------------- /pan_cortex_data_lake/adapters/adapter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | :::info 5 | Base adapter class. 6 | ::: 7 | """ 8 | from __future__ import absolute_import 9 | 10 | from abc import ABCMeta, abstractmethod 11 | 12 | # Python 2.7 and 3.5+ compatibility 13 | ABC = ABCMeta("ABC", (object,), {"__slots__": ()}) 14 | 15 | 16 | class StorageAdapter(ABC): # enforce StorageAdapter interface 17 | """A storage adapter abstract base class.""" 18 | 19 | @abstractmethod 20 | def fetch_credential(self, credential=None, profile=None): 21 | """Fetch credential from store. 22 | 23 | Args: 24 | credential (str): Credential to fetch. 25 | profile (str): Credentials profile. Defaults to 'default'. 26 | 27 | """ 28 | pass 29 | 30 | @abstractmethod 31 | def init_store(self): 32 | """Initialize credentials store.""" 33 | pass 34 | 35 | @abstractmethod 36 | def remove_profile(self, profile=None): 37 | """Remove profile from store. 38 | 39 | Args: 40 | profile (str): Credentials profile to remove. 41 | 42 | """ 43 | pass 44 | 45 | @abstractmethod 46 | def write_credentials(self, credentials=None, profile=None, cache_token=None): 47 | """Write credentials. 48 | 49 | :::info 50 | Write credentials to store. 51 | ::: 52 | 53 | Args: 54 | cache_token (bool): If `True`, stores `access_token` in token store. Defaults to `True`. 55 | credentials (class): Read-only credentials. 56 | profile (str): Credentials profile. Defaults to 'default'. 57 | 58 | """ 59 | pass 60 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=3.2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [project] 6 | name = "pan-cortex-data-lake" 7 | keywords = [ 8 | "cortex", 9 | "data", 10 | "lake", 11 | "datalake", 12 | "sdk", 13 | "api", 14 | "palo alto networks", 15 | ] 16 | authors = [ 17 | {name = "Steven Serrata", email = "sserrata@paloaltonetworks.com"}, 18 | ] 19 | maintainers = [ 20 | {name = "Steven Serrata", email = "sserrata@paloaltonetworks.com"}, 21 | {name = "Developer Relations", email = "devrel@paloaltonetworks.com"}, 22 | ] 23 | readme = "README.md" 24 | classifiers = [ 25 | "Development Status :: 4 - Beta", 26 | "Intended Audience :: Developers", 27 | "License :: OSI Approved :: ISC License (ISCL)", 28 | "Natural Language :: English", 29 | "Programming Language :: Python :: 3", 30 | "Programming Language :: Python :: 3.5", 31 | "Programming Language :: Python :: 3.6", 32 | "Programming Language :: Python :: 3.7", 33 | "Programming Language :: Python :: 3.8", 34 | "Programming Language :: Python :: 3.9", 35 | ] 36 | license = {file = "LICENSE"} 37 | requires-python = ">=3.5" 38 | dynamic = ["version", "description"] 39 | dependencies = [ 40 | "requests >=2", 41 | "tinydb >=3" 42 | ] 43 | 44 | [project.optional-dependencies] 45 | test = [ 46 | "pytest >=2.7.3", 47 | "pytest-cov", 48 | "flake8", 49 | "tox", 50 | "coverage", 51 | ] 52 | 53 | [project.urls] 54 | Home = "https://cortex.pan.dev" 55 | Source = "https://github.com/PaloAltoNetworks/pan-cortex-data-lake-python" 56 | Documentation = "https://cortex.pan.dev/docs/develop/cdl_python_installation" 57 | 58 | [tool.flit.module] 59 | name = "pan_cortex_data_lake" 60 | 61 | -------------------------------------------------------------------------------- /pan_cortex_data_lake/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | :::info 5 | This module provides base classes for all errors raised by the PAN Cloud 6 | library. All other exceptions are raised and maintained by Python 7 | standard or nonstandard libraries. 8 | ::: 9 | 10 | """ 11 | 12 | 13 | class CortexError(Exception): 14 | """Base class for all exceptions raised by PAN Cloud library.""" 15 | 16 | def __init__(self, message): 17 | """Override the base class message attribute. 18 | 19 | Args: 20 | message (str): Exception message. 21 | 22 | """ 23 | super(CortexError, self).__init__(message) 24 | self.message = message 25 | 26 | 27 | class HTTPError(CortexError): 28 | """A pancloud HTTP error occurred.""" 29 | 30 | def __init__(self, inst): 31 | """Convert exception instance to string. 32 | 33 | Args: 34 | inst (class): Exception instance. 35 | 36 | """ 37 | CortexError.__init__(self, "{}".format(inst)) 38 | 39 | 40 | class PartialCredentialsError(CortexError): 41 | """The required credentials were not supplied.""" 42 | 43 | def __init__(self, inst): 44 | """Convert exception instance to string. 45 | 46 | Args: 47 | inst (class): Exception instance. 48 | 49 | """ 50 | CortexError.__init__(self, "{}".format(inst)) 51 | 52 | 53 | class RequiredKwargsError(CortexError): 54 | """A required keyword argument was not passed.""" 55 | 56 | def __init__(self, kwarg): 57 | """Capture missing key-word argument. 58 | 59 | Args: 60 | kwarg (str): Key-word argument. 61 | 62 | """ 63 | CortexError.__init__(self, "{}".format(kwarg)) 64 | 65 | 66 | class UnexpectedKwargsError(CortexError): 67 | """An unexpected keyword argument was passed.""" 68 | 69 | def __init__(self, kwargs): 70 | """Convert kwargs to CSV string. 71 | 72 | Args: 73 | kwargs (dict): Key-word arguments. 74 | 75 | """ 76 | CortexError.__init__(self, "{}".format(", ".join(kwargs.keys()))) 77 | -------------------------------------------------------------------------------- /tests/test_httpclient.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Tests for Requests HTTPClient wrapper.""" 5 | 6 | import os 7 | import sys 8 | 9 | import pytest 10 | 11 | curpath = os.path.dirname(os.path.abspath(__file__)) 12 | sys.path[:0] = [os.path.join(curpath, os.pardir)] 13 | 14 | from pan_cortex_data_lake.httpclient import HTTPClient 15 | from pan_cortex_data_lake.exceptions import ( 16 | HTTPError, 17 | UnexpectedKwargsError, 18 | CortexError, 19 | ) 20 | 21 | 22 | HTTPBIN = os.environ.get("HTTPBIN_URL", "http://httpbin.org") 23 | TARPIT = os.environ.get("TARPIT", "http://10.255.255.1") 24 | 25 | 26 | class TestHTTPClient: 27 | def test_entry_points(self): 28 | 29 | HTTPClient(url=TARPIT).request 30 | 31 | def test_invalid_url(self): 32 | with pytest.raises(HTTPError): 33 | HTTPClient(url="asdaksjhdakjsdh").request(method="GET") 34 | with pytest.raises(HTTPError): 35 | HTTPClient(url="http://").request(method="GET") 36 | 37 | def test_connection_timeout(self): 38 | with pytest.raises(HTTPError): 39 | HTTPClient(url=TARPIT).request(method="GET", timeout=(0.1, None)) 40 | 41 | def test_read_timeout(self): 42 | with pytest.raises(HTTPError): 43 | HTTPClient(url=HTTPBIN, port=80).request( 44 | method="GET", timeout=(None, 0.0001), endpoint="/" 45 | ) 46 | 47 | def test_httpclient_unexpected_kwargs(self): 48 | with pytest.raises(UnexpectedKwargsError): 49 | HTTPClient(url=TARPIT, foo="foo").request(method="GET") 50 | 51 | def test_request_unexpected_kwargs(self): 52 | with pytest.raises(UnexpectedKwargsError): 53 | HTTPClient(url=TARPIT).request(method="GET", foo="foo") 54 | 55 | def test_enforce_json(self): 56 | with pytest.raises(CortexError): 57 | HTTPClient( 58 | url=HTTPBIN, 59 | port=80, 60 | enforce_json=True, 61 | headers={"Accept": "application/json"}, 62 | ).request(method="GET", endpoint="/") 63 | 64 | def test_raise_for_status(self): 65 | with pytest.raises(HTTPError): 66 | HTTPClient(url=HTTPBIN, port=80, raise_for_status=True).request( 67 | method="GET", endpoint="/status/400" 68 | ) 69 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | :+1::tada: First off, thanks for taking the time to contribute! :tada::+1: 4 | 5 | It's people like you that make security open source such a force in preventing 6 | successful cyber-attacks. Following these guidelines helps keep the project 7 | maintainable, easy to contribute to, and more secure. Thank you for taking the 8 | time to follow this guide. 9 | 10 | ## Where to start 11 | 12 | There are many ways to contribute. You can fix a bug, improve the documentation, 13 | submit bug reports and feature requests, or take a first shot at a feature you 14 | need for yourself. 15 | 16 | Pull requests are necessary for all contributions of code or documentation. 17 | 18 | ## New to open source? 19 | 20 | If you're **new to open source** and not sure what a pull request is, welcome!! 21 | We're glad to have you! All of us once had a contribution to make and didn't 22 | know where to start. 23 | 24 | Even if you don't write code for your job, don't worry, the skills you learn 25 | during your first contribution to open source can be applied in so many ways, 26 | you'll wonder what you ever did before you had this knowledge. It's worth 27 | learning. 28 | 29 | [Learn how to make a pull request](https://github.com/PaloAltoNetworks/.github/blob/master/Learn-GitHub.md#learn-how-to-make-a-pull-request) 30 | 31 | ## Fixing a typo, or a one or two line fix 32 | 33 | Many fixes require little effort or review, such as: 34 | 35 | > - Spelling / grammar, typos, white space and formatting changes 36 | > - Comment clean up 37 | > - Change logging messages or debugging output 38 | 39 | These small changes can be made directly in GitHub if you like. 40 | 41 | Click the pencil icon in GitHub above the file to edit the file directly in 42 | GitHub. This will automatically create a fork and pull request with the change. 43 | See: 44 | [Make a small change with a Pull Request](https://www.freecodecamp.org/news/how-to-make-your-first-pull-request-on-github/) 45 | 46 | ## Bug fixes and features 47 | 48 | For something that is bigger than a one or two line fix, go through the process 49 | of making a fork and pull request yourself: 50 | 51 | > 1. Create your own fork of the code 52 | > 2. Clone the fork locally 53 | > 3. Make the changes in your local clone 54 | > 4. Push the changes from local to your fork 55 | > 5. Create a pull request to pull the changes from your fork back into the 56 | > upstream repository 57 | 58 | Please use clear commit messages so we can understand what each commit does. 59 | We'll review every PR and might offer feedback or request changes before 60 | merging. -------------------------------------------------------------------------------- /.github/workflows/pydocs-publish.yml: -------------------------------------------------------------------------------- 1 | name: PyDocs Publish 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | if: github.repository_owner == 'PaloAltoNetworks' 11 | name: Generate docs 12 | runs-on: ubuntu-latest 13 | permissions: 14 | contents: read 15 | 16 | steps: 17 | - name: Checkout repository 18 | uses: actions/checkout@v2 19 | 20 | - name: Set up Python 21 | uses: actions/setup-python@v2 22 | with: 23 | python-version: '3.9' 24 | architecture: 'x64' 25 | 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | python -m pip install pipx 30 | python -m pip install editfrontmatter 31 | pipx install 'pydoc-markdown>=4.0.0,<5.0.0' 32 | - name: Generate markdown docs 33 | run: | 34 | pydoc-markdown -vv 35 | - name: Upload markdown artifacts 36 | uses: actions/upload-artifact@v2 37 | with: 38 | name: docs-dir 39 | path: docs/ 40 | 41 | pull: 42 | name: Open pull request 43 | needs: build 44 | runs-on: ubuntu-latest 45 | 46 | steps: 47 | - name: Checkout docs repository 48 | uses: actions/checkout@v2 49 | with: 50 | repository: 'PaloAltoNetworks/cortex.pan.dev' 51 | 52 | - name: Download markdown artifacts 53 | uses: actions/download-artifact@v2 54 | with: 55 | name: docs-dir 56 | path: docs/ 57 | 58 | - name: Commit changes 59 | id: commit 60 | run: | 61 | git config --global user.email "sserrata@paloaltonetworks.com" 62 | git config --global user.name "Steven Serrata" 63 | git add . 64 | (git diff --exit-code || git diff --exit-code --cached) || git commit -m "update pydocs" 65 | git status 66 | - name: Create Pull Request 67 | id: pydocs 68 | uses: peter-evans/create-pull-request@v3 69 | with: 70 | token: ${{ secrets.PYDOC_TOKEN }} 71 | commit-message: 🐍 Update pydocs 72 | committer: GitHub 73 | author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> 74 | delete-branch: true 75 | title: '✅ [PyDocs Publish] Publish python library docs' 76 | body: | 77 | Update pydocs 78 | - Docs generated with `pydoc-markdown` 79 | 80 | > This PR includes changes ready to be merged to production. Please review and merge when ready. 81 | -------------------------------------------------------------------------------- /.github/workflows/pydocs-preview.yml: -------------------------------------------------------------------------------- 1 | name: PyDocs Preview 2 | 3 | on: 4 | pull_request_target: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | if: github.repository_owner == 'PaloAltoNetworks' 11 | name: Generate docs 12 | runs-on: ubuntu-latest 13 | permissions: 14 | contents: read 15 | 16 | steps: 17 | - name: Checkout repository 18 | uses: actions/checkout@v2 19 | with: 20 | ref: ${{ github.event.pull_request.head.sha }} 21 | 22 | - name: Set up Python 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: '3.9' 26 | architecture: 'x64' 27 | 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | python -m pip install pipx 32 | python -m pip install editfrontmatter 33 | pipx install 'pydoc-markdown>=4.0.0,<5.0.0' 34 | 35 | - name: Generate markdown docs 36 | run: | 37 | pydoc-markdown -vv 38 | 39 | - name: Upload markdown artifacts 40 | uses: actions/upload-artifact@v2 41 | with: 42 | name: docs-dir 43 | path: docs/ 44 | 45 | pull: 46 | name: Open pull request 47 | needs: build 48 | runs-on: ubuntu-latest 49 | 50 | steps: 51 | - name: Checkout docs repository 52 | uses: actions/checkout@v2 53 | with: 54 | repository: 'PaloAltoNetworks/cortex.pan.dev' 55 | 56 | - name: Download markdown artifacts 57 | uses: actions/download-artifact@v2 58 | with: 59 | name: docs-dir 60 | path: docs/ 61 | 62 | - name: Commit changes 63 | id: commit 64 | run: | 65 | git config --global user.email "sserrata@paloaltonetworks.com" 66 | git config --global user.name "Steven Serrata" 67 | git add . 68 | (git diff --exit-code || git diff --exit-code --cached) || git commit -m "update pydocs" 69 | git status 70 | 71 | - name: Create Pull Request 72 | id: pydocs 73 | uses: peter-evans/create-pull-request@v3 74 | with: 75 | token: ${{ secrets.PYDOC_TOKEN }} 76 | commit-message: Update pydocs 77 | committer: GitHub 78 | author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> 79 | delete-branch: true 80 | title: '🚫 [PyDocs Preview] Preview python library docs' 81 | labels: | 82 | don't merge 83 | body: | 84 | Update pydocs 85 | - Docs generated with `pydoc-markdown` 86 | 87 | > **For preview/review purposes only**, please close this PR and delete branch when done. A new PR 88 | will be opened (that can be merged) when upstream changes are merged to main branch. 89 | 90 | 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Tests](https://github.com/PaloAltoNetworks/pan-cortex-data-lake-python/workflows/Tests/badge.svg) ![PyPI upload](https://github.com/PaloAltoNetworks/pan-cortex-data-lake-python/workflows/PyPI%20upload/badge.svg?branch=master) [![PyPI version](https://badge.fury.io/py/pan-cortex-data-lake.svg)](https://badge.fury.io/py/pan-cortex-data-lake) 2 | 3 | # Palo Alto Networks Cortex™ Data Lake SDK 4 | 5 | Python idiomatic SDK for the Cortex™ Data Lake. 6 | 7 | The Palo Alto Networks Cortex Data Lake Python SDK was created to assist 8 | developers with programmatically interacting with the Palo Alto Networks 9 | Cortex™ Data Lake API. 10 | 11 | The primary goal is to provide full, low-level API coverage for the 12 | following Cortex™ Data Lake services: 13 | 14 | - Query Service 15 | 16 | The secondary goal is to provide coverage, in the form of helpers, for 17 | common tasks/operations. 18 | 19 | - Log/event pagination 20 | - OAuth 2.0 and token refreshing 21 | 22 | Resources: 23 | 24 | - Documentation: 25 | - Free software: [ISC license](https://choosealicense.com/licenses/isc/) 26 | 27 | --- 28 | 29 | ## Features 30 | 31 | - HTTP client wrapper for the popular Requests library with full access to its features. 32 | - Language bindings for Query Service. 33 | - Helper methods for performing common tasks, such as log/event pagination. 34 | - Support for OAuth 2.0 grant code authorization flow. 35 | - Library of example scripts illustrating how to leverage the SDK. 36 | - Support for API Explorer Developer Tokens for easier access to API! 37 | 38 | ## Status 39 | 40 | The Palo Alto Networks Cortex™ Data Lake Python SDK is considered **beta** at this time. 41 | 42 | ## Installation 43 | 44 | From PyPI: 45 | 46 | ```bash 47 | pip install pan-cortex-data-lake 48 | ``` 49 | 50 | From source: 51 | 52 | ```bash 53 | pip install . 54 | ``` 55 | 56 | To run tests: 57 | 58 | ```bash 59 | pip install .[test] 60 | ``` 61 | 62 | ## Obtaining and Using OAuth 2.0 Tokens 63 | 64 | If you're an app developer, work with your Developer Relations representative to obtain your OAuth2 credentials. API Explorer may optionally be used to generate a Developer Token, which can also be used to authenticate with the API. For details on API Explorer developer tokens, please visit . 65 | 66 | # Example 67 | 68 | ```python 69 | from pan_cortex_data_lake import Credentials, QueryService 70 | 71 | 72 | c = Credentials() 73 | qs = QueryService(credentials=c) 74 | query_params = { 75 | "query": "SELECT * FROM `1234567890.firewall.traffic` LIMIT 1", 76 | } 77 | q = qs.create_query(query_params=query_params) 78 | results = qs.get_job_results(job_id=q.json()['jobId']) 79 | print(results.json()) 80 | ``` 81 | 82 | # Contributors 83 | 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /pan_cortex_data_lake/adapters/tinydb_adapter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | :::info 5 | TinyDB storage adapter. 6 | ::: 7 | """ 8 | from __future__ import absolute_import 9 | 10 | import os 11 | from threading import RLock 12 | 13 | from tinydb import TinyDB, Query, __version__ 14 | from tinydb.storages import MemoryStorage 15 | 16 | from .. import CortexError 17 | from . import StorageAdapter 18 | 19 | 20 | class TinyDBStore(StorageAdapter): 21 | def __init__(self, **kwargs): 22 | self._storage_params = kwargs.get("storage_params") or {} 23 | self.dbfile = self._storage_params.get("dbfile") 24 | self.memory_storage = self._storage_params.get("memory_storage", False) 25 | self.query = Query() 26 | self.db = self.init_store() 27 | self.lock = RLock() 28 | 29 | def fetch_credential(self, credential=None, profile=None): 30 | """Fetch credential from credentials file. 31 | 32 | Args: 33 | credential (str): Credential to fetch. 34 | profile (str): Credentials profile. Defaults to ``'default'``. 35 | 36 | Returns: 37 | str, None: Fetched credential or ``None``. 38 | 39 | """ 40 | q = self.db.get(self.query.profile == profile) 41 | if q is not None: 42 | return q.get(credential) 43 | 44 | def init_store(self): 45 | if self.memory_storage is True: 46 | return TinyDB(storage=MemoryStorage) 47 | if self.dbfile: 48 | dbfile = self.dbfile 49 | elif os.getenv("PAN_CREDENTIALS_DBFILE"): 50 | dbfile = os.getenv("PAN_CREDENTIALS_DBFILE") 51 | else: 52 | dbfile = os.path.join( 53 | os.path.expanduser("~"), 54 | ".config", 55 | "pan_cortex_data_lake", 56 | "credentials.json", 57 | ) 58 | if not os.path.exists(os.path.dirname(dbfile)): 59 | try: 60 | os.makedirs(os.path.dirname(dbfile), 0o700) 61 | except OSError as e: 62 | raise CortexError("{}".format(e)) 63 | if __version__ >= "4.0.0": 64 | db = TinyDB(dbfile, sort_keys=True, indent=4) 65 | db.default_table_name = "profiles" 66 | else: 67 | db = TinyDB(dbfile, sort_keys=True, default_table="profiles", indent=4) 68 | return db 69 | 70 | def remove_profile(self, profile=None): 71 | """Remove profile from credentials file. 72 | 73 | Args: 74 | profile (str): Credentials profile to remove. 75 | 76 | Returns: 77 | list: List of affected document IDs. 78 | 79 | """ 80 | with self.db: 81 | return self.db.remove(self.query.profile == profile) 82 | 83 | def write_credentials(self, credentials=None, profile=None, cache_token=None): 84 | """Write credentials. 85 | 86 | :::info 87 | Write credentials to credentials file. Performs ``upsert``. 88 | ::: 89 | 90 | Args: 91 | cache_token (bool): If ``True``, stores ``access_token`` in token store. Defaults to ``True``. 92 | credentials (class): Read-only credentials. 93 | profile (str): Credentials profile. Defaults to ``'default'``. 94 | 95 | Returns: 96 | int: Affected document ID. 97 | 98 | """ 99 | d = { 100 | "profile": profile, 101 | "client_id": credentials.client_id, 102 | "client_secret": credentials.client_secret, 103 | "refresh_token": credentials.refresh_token, 104 | } 105 | if cache_token: 106 | d.update({"access_token": credentials.access_token}) 107 | with self.lock: 108 | return self.db.upsert(d, self.query.profile == profile) 109 | -------------------------------------------------------------------------------- /pan_cortex_data_lake/query.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | :::info 5 | The Query Service is a Palo Alto Networks cloud service which allows 6 | for the storage and retrieval of data stored in the Cortex Data Lake. 7 | Any type of textual data can be stored in the Cortex Data Lake. Palo 8 | Alto Networks firewalls and software can write data to this service, as 9 | can the software and services created by Palo Alto Network's various 10 | partners. 11 | ::: 12 | 13 | Examples: 14 | Refer to the [examples provided with this library](https://github.com/PaloAltoNetworks/pan-cortex-data-lake-python/tree/master/examples). 15 | 16 | """ 17 | 18 | from __future__ import absolute_import 19 | import logging 20 | import time 21 | 22 | from .exceptions import CortexError, HTTPError 23 | from .httpclient import HTTPClient 24 | from . import __version__ 25 | 26 | 27 | class QueryService(object): 28 | """A Cortex™ Query Service instance.""" 29 | 30 | def __init__(self, **kwargs): 31 | """ 32 | 33 | Parameters: 34 | session (HTTPClient): [HTTPClient](httpclient.md#httpclient) object. Defaults to `None`. 35 | url (str): URL to send API requests to. Later combined with `port` and `endpoint` parameter. 36 | 37 | Args: 38 | **kwargs: Supported [HTTPClient](httpclient.md#httpclient) parameters. 39 | 40 | """ 41 | self.kwargs = kwargs.copy() # used for __repr__ 42 | self.session = kwargs.pop("session", None) 43 | self._httpclient = self.session or HTTPClient(**kwargs) 44 | self._httpclient.stats.update( 45 | { 46 | "cancel_job": 0, 47 | "create_query": 0, 48 | "get_job": 0, 49 | "list_jobs": 0, 50 | "get_job_results": 0, 51 | "records": 0, 52 | } 53 | ) 54 | self.stats = self._httpclient.stats 55 | self.url = self._httpclient.url 56 | self._debug = logging.getLogger(__name__).debug 57 | 58 | def __repr__(self): 59 | for k in self.kwargs.get("headers", {}): 60 | if k.lower() == "authorization": 61 | x = dict(self.kwargs["headers"].items()) 62 | x[k] = "*" * 6 # starrify token 63 | return "{}({}, {})".format( 64 | self.__class__.__name__, 65 | ", ".join( 66 | "%s=%r" % (x, _) 67 | for x, _ in self.kwargs.items() 68 | if x != "headers" 69 | ), 70 | "headers=%r" % x, 71 | ) 72 | return "{}({})".format( 73 | self.__class__.__name__, ", ".join("%s=%r" % x for x in self.kwargs.items()) 74 | ) 75 | 76 | def cancel_job(self, job_id=None, **kwargs): 77 | """Cancel a query job. 78 | 79 | Args: 80 | job_id (str): Specifies the ID of the query job. 81 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 82 | 83 | Returns: 84 | requests.Response: Requests [Response()](https://docs.python-requests.org/en/latest/api/#requests.Response) object. 85 | 86 | Raises: 87 | 88 | 89 | """ 90 | endpoint = "/query/v2/jobs/{}".format(job_id) 91 | r = self._httpclient.request( 92 | method="DELETE", url=self.url, endpoint=endpoint, **kwargs 93 | ) 94 | self.stats.cancel_job += 1 95 | return r 96 | 97 | def create_query(self, job_id=None, query_params=None, **kwargs): 98 | """Create a search request. 99 | 100 | :::info 101 | When submission is successful, http status code of `201` (Created) 102 | is returned with a 'jobId' in response. Specifying a 'jobId' is 103 | optional. 104 | ::: 105 | 106 | Args: 107 | job_id (str): Specifies the ID of the query job. (optional) 108 | query_params (dict): Query parameters. 109 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 110 | 111 | Returns: 112 | requests.Response: Requests [Response()](https://docs.python-requests.org/en/latest/api/#requests.Response) object. 113 | 114 | """ 115 | json = kwargs.pop("json", {}) 116 | for name, value in [("jobId", job_id), ("params", query_params)]: 117 | if value is not None: 118 | json.update({name: value}) 119 | json.update( 120 | { 121 | "clientType": "cortex-data-lake-python", 122 | "clientVersion": "%s" % __version__, 123 | } 124 | ) 125 | endpoint = "/query/v2/jobs" 126 | r = self._httpclient.request( 127 | method="POST", url=self.url, json=json, endpoint=endpoint, **kwargs 128 | ) 129 | self.stats.create_query += 1 130 | return r 131 | 132 | def get_job(self, job_id=None, **kwargs): 133 | """Get specific job matching criteria. 134 | 135 | Args: 136 | job_id (str): Specifies the ID of the query job. 137 | params (dict): Payload/request dictionary. 138 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 139 | 140 | Returns: 141 | requests.Response: Requests [Response()](https://docs.python-requests.org/en/latest/api/#requests.Response) object. 142 | 143 | """ 144 | endpoint = "/query/v2/jobs/{}".format(job_id) 145 | r = self._httpclient.request( 146 | method="GET", url=self.url, endpoint=endpoint, **kwargs 147 | ) 148 | self.stats.get_job += 1 149 | return r 150 | 151 | def get_job_results( 152 | self, 153 | job_id=None, 154 | max_wait=None, 155 | offset=None, 156 | page_cursor=None, 157 | page_number=None, 158 | page_size=None, 159 | result_format=None, 160 | **kwargs 161 | ): 162 | """Get results for a specific job_id. 163 | 164 | Args: 165 | job_id (str): Specifies the ID of the query job. 166 | max_wait (int): How long to wait in ms for a job to complete. Max 2000. 167 | offset (int): Along with pageSize, offset can be used to page through result set. 168 | page_cursor (str): Token/handle that can be used to fetch more data. 169 | page_number (int): Return the nth page from the result set as specified by this parameter. 170 | page_size (int): If specified, limits the size of a batch of results to the specified value. If un-specified, backend picks a size that may provide best performance. 171 | result_format (str): valuesArray or valuesDictionary. 172 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 173 | 174 | Returns: 175 | requests.Response: Requests [Response()](https://docs.python-requests.org/en/latest/api/#requests.Response) object. 176 | 177 | """ 178 | params = kwargs.pop("params", {}) 179 | for name, value in [ 180 | ("maxWait", max_wait), 181 | ("offset", offset), 182 | ("pageCursor", page_cursor), 183 | ("pageNumber", page_number), 184 | ("pageSize", page_size), 185 | ("resultFormat", result_format), 186 | ]: 187 | if value is not None: 188 | params.update({name: value}) 189 | endpoint = "/query/v2/jobResults/{}".format(job_id) 190 | r = self._httpclient.request( 191 | method="GET", url=self.url, params=params, endpoint=endpoint, **kwargs 192 | ) 193 | self.stats.get_job_results += 1 194 | 195 | rows = r.json().get("rowsInPage") 196 | if rows is not None: 197 | self.stats.records += rows 198 | 199 | return r 200 | 201 | def iter_job_results( 202 | self, 203 | job_id=None, 204 | max_wait=None, 205 | offset=None, 206 | page_cursor=None, 207 | page_number=None, 208 | page_size=None, 209 | result_format=None, 210 | **kwargs 211 | ): 212 | """Retrieve results iteratively in a non-greedy manner using scroll token. 213 | 214 | Args: 215 | job_id (str): Specifies the ID of the query job. 216 | max_wait (int): How long to wait in ms for a job to complete. Max 2000. 217 | offset (int): Along with pageSize, offset can be used to page through result set. 218 | page_cursor (str): Token/handle that can be used to fetch more data. 219 | page_number (int): Return the nth page from the result set as specified by this parameter. 220 | page_size (int): If specified, limits the size of a batch of results to the specified value. If un-specified, backend picks a size that may provide best performance. 221 | result_format (str): valuesArray or valuesJson. 222 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 223 | 224 | Returns: 225 | requests.Response: Requests [Response()](https://docs.python-requests.org/en/latest/api/#requests.Response) object. 226 | 227 | """ 228 | params = kwargs.pop("params", {}) 229 | enforce_json = kwargs.pop("enforce_json", True) 230 | for name, value in [ 231 | ("maxWait", max_wait), 232 | ("offset", offset), 233 | ("pageCursor", page_cursor), 234 | ("pageNumber", page_number), 235 | ("pageSize", page_size), 236 | ("resultFormat", result_format), 237 | ]: 238 | if value is not None: 239 | params.update({name: value}) 240 | 241 | while True: 242 | r = self.get_job_results( 243 | job_id=job_id, params=params, enforce_json=enforce_json, **kwargs 244 | ) 245 | r_json = r.json() 246 | if r_json["state"] == "DONE": 247 | page_cursor = r_json["page"].get("pageCursor") 248 | if page_cursor is not None: 249 | params["pageCursor"] = page_cursor 250 | yield r 251 | else: 252 | yield r 253 | break 254 | elif r_json["state"] in ("RUNNING", "PENDING"): 255 | time.sleep(1) 256 | continue 257 | elif r_json["state"] == "FAILED": 258 | yield r 259 | break 260 | else: 261 | raise CortexError("Bad state: %s" % r_json["state"]) 262 | 263 | def list_jobs( 264 | self, 265 | max_jobs=None, 266 | created_after=None, 267 | state=None, 268 | job_type=None, 269 | tenant_id=None, 270 | **kwargs 271 | ): 272 | """Get all jobs matching criteria. 273 | 274 | Args: 275 | limit (int): Max number of jobs. 276 | created_after (int): List jobs created after this unix epoch UTC datetime. 277 | state (str): Job state, e.g. 'RUNNING', 'PENDING', 'FAILED', 'DONE'. 278 | job_type (str): Query type hint. 279 | tenant_id (str): Tenant ID. 280 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 281 | 282 | Returns: 283 | requests.Response: Requests [Response()](https://docs.python-requests.org/en/latest/api/#requests.Response) object. 284 | 285 | """ 286 | params = kwargs.pop("params", {}) 287 | for name, value in [ 288 | ("maxJobs", max_jobs), 289 | ("createdAfter", created_after), 290 | ("state", state), 291 | ("type", job_type), 292 | ("tenantId", tenant_id), 293 | ]: 294 | if value is not None: 295 | params.update({name: value}) 296 | endpoint = "/query/v2/jobs" 297 | r = self._httpclient.request( 298 | method="GET", url=self.url, params=params, endpoint=endpoint, **kwargs 299 | ) 300 | self.stats.list_jobs += 1 301 | return r 302 | -------------------------------------------------------------------------------- /pan_cortex_data_lake/httpclient.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import absolute_import 4 | 5 | import logging 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | import requests 10 | from requests.adapters import HTTPAdapter 11 | 12 | # support ujson in place of standard json library 13 | try: 14 | import ujson 15 | 16 | requests.models.complexjson = ujson 17 | logger.debug("Monkey patched requests with ujson") 18 | except ImportError: 19 | pass 20 | 21 | from .exceptions import ( 22 | UnexpectedKwargsError, 23 | RequiredKwargsError, 24 | HTTPError, 25 | CortexError, 26 | ) 27 | from . import __version__ 28 | from .utils import ApiStats 29 | 30 | 31 | class HTTPClient(object): 32 | """HTTP client for the Cortex™ REST API""" 33 | 34 | def __init__(self, **kwargs): 35 | """Persist `Session()` attributes and implement connection-pooling. 36 | 37 | :::info 38 | Built on top of the `Requests` library, `HTTPClient` is an 39 | abstraction layer for preparing and sending HTTP `requests` to the 40 | Application Framework REST APIs and handling `responses`. All 41 | `Requests` are prepared as `Session` objects, with the option 42 | to persist certain attributes such as `cert`, `headers`, 43 | `proxies`, etc. `HTTPAdapter` is implemented to enable more 44 | granular performance and reliability tuning. 45 | ::: 46 | 47 | Parameters: 48 | auto_refresh (bool): Perform token refresh prior to request if `access_token` is `None` or expired. Defaults to `True`. 49 | auto_retry (bool): Retry last failed HTTP request following a token refresh. Defaults to `True`. 50 | credentials (Credentials): [Credentials](credentials.md#credentials) object. Defaults to `None`. 51 | enforce_json (bool): Require properly-formatted JSON or raise [CortexError](exceptions.md#cortexerror). Defaults to `False`. 52 | force_trace (bool): If `True`, forces trace and forces `x-request-id` to be returned in the response headers. Defaults to `False`. 53 | port (int): TCP port to append to URL. Defaults to `443`. 54 | raise_for_status (bool): If `True`, raises [HTTPError](exceptions.md#httperror) if status_code not in 2XX. Defaults to `False`. 55 | url (str): URL to send API requests to - gets combined with `port` and `endpoint` parameter. Defaults to `None`. 56 | 57 | Args: 58 | **kwargs: Supported [Session](https://github.com/psf/requests/blob/main/requests/sessions.py#L337) and 59 | [HTTPAdapter](https://github.com/psf/requests/blob/main/requests/adapters.py#L85) parameters. 60 | 61 | """ 62 | self.kwargs = kwargs.copy() # used for __repr__ 63 | with requests.Session() as self.session: 64 | self._default_headers() # apply default headers 65 | self.session.auth = kwargs.pop("auth", self.session.auth) 66 | self.session.cert = kwargs.pop("cert", self.session.cert) 67 | self.session.cookies = kwargs.pop("cookies", self.session.cookies) 68 | self.session.headers.update(kwargs.pop("headers", {})) 69 | self.session.params = kwargs.pop("params", self.session.params) 70 | self.session.proxies = kwargs.pop("proxies", self.session.proxies) 71 | self.session.stream = kwargs.pop("stream", self.session.stream) 72 | self.session.trust_env = kwargs.pop("trust_env", self.session.trust_env) 73 | self.session.verify = kwargs.pop("verify", self.session.verify) 74 | 75 | # HTTPAdapter key-word arguments 76 | _kwargs = {} 77 | for x in ["pool_connections", "pool_maxsize", "pool_block", "max_retries"]: 78 | if x in kwargs: 79 | _kwargs[x] = kwargs.pop(x) 80 | self.adapter = HTTPAdapter(**_kwargs) 81 | self.session.mount("https://", self.adapter) 82 | self.session.mount("http://", self.adapter) 83 | 84 | # Non-Requests key-word arguments 85 | self.auto_refresh = kwargs.pop("auto_refresh", True) 86 | self.credentials = kwargs.pop("credentials", None) 87 | self.enforce_json = kwargs.pop("enforce_json", False) 88 | self.force_trace = kwargs.pop("force_trace", False) 89 | if self.force_trace is True: 90 | self.session.headers.update({"x-envoy-force-trace": ""}) 91 | self.port = kwargs.pop("port", 443) 92 | self.raise_for_status = kwargs.pop("raise_for_status", False) 93 | self.url = kwargs.pop("url", "https://api.us.cdl.paloaltonetworks.com") 94 | 95 | if len(kwargs) > 0: # Handle invalid kwargs 96 | raise UnexpectedKwargsError(kwargs) 97 | 98 | self.stats = ApiStats({"transactions": 0}) 99 | 100 | def __repr__(self): 101 | for k in self.kwargs.get("headers", {}): 102 | if k.lower() == "authorization": 103 | x = dict(self.kwargs["headers"].items()) 104 | x[k] = "*" * 6 # starrify token 105 | return "{}({}, {})".format( 106 | self.__class__.__name__, 107 | ", ".join( 108 | "%s=%r" % (x, _) 109 | for x, _ in self.kwargs.items() 110 | if x != "headers" 111 | ), 112 | "headers=%r" % x, 113 | ) 114 | return "{}({})".format( 115 | self.__class__.__name__, ", ".join("%s=%r" % x for x in self.kwargs.items()) 116 | ) 117 | 118 | @staticmethod 119 | def _apply_credentials(auto_refresh=True, credentials=None, headers=None): 120 | """Update Authorization header. 121 | 122 | Update request headers with latest `access_token`. Perform token 123 | `refresh` if token is `None`. 124 | 125 | Args: 126 | auto_refresh (bool): Perform token refresh if access_token is `None` or expired. Defaults to `True`. 127 | credentials (class): Read-only credentials. 128 | headers (class): Requests `CaseInsensitiveDict`. 129 | 130 | """ 131 | token = credentials.get_credentials().access_token 132 | if auto_refresh is True: 133 | if token is None: 134 | token = credentials.refresh(access_token=None, timeout=10) 135 | logger.debug("Token refreshed due to 'None' condition") 136 | elif credentials.jwt_is_expired(token): 137 | token = credentials.refresh(timeout=10) 138 | logger.debug("Token refreshed due to 'expired' condition") 139 | headers.update({"Authorization": "Bearer {}".format(token)}) 140 | logger.debug("Credentials applied to authorization header") 141 | 142 | def _default_headers(self): 143 | """Update default headers. 144 | 145 | The requests library default headers are set in the `utils.py` 146 | `default_headers()` function. 147 | 148 | """ 149 | self.session.headers.update( 150 | { 151 | "Accept": "application/json", 152 | "User-Agent": "%s/%s" % ("cortex-data-lake-python", __version__), 153 | } 154 | ) 155 | logger.debug("Default headers applied: %r" % self.session.headers) 156 | 157 | def _send_request(self, enforce_json, method, raise_for_status, url, **kwargs): 158 | """Send HTTP request. 159 | 160 | Args: 161 | enforce_json (bool): Require properly-formatted JSON or raise [CortexError](exceptions.md#cortexerror). Defaults to `False`. 162 | method (str): HTTP method. 163 | raise_for_status (bool): If `True`, raises [HTTPError](exceptions.md#httperror) if status_code not in 2XX. Defaults to `False`. 164 | url (str): Request URL. 165 | **kwargs (dict): Re-packed key-word arguments. 166 | 167 | Returns: 168 | requests.Response: [Response()](https://docs.python-requests.org/en/latest/api/#requests.Response) object 169 | 170 | """ 171 | r = self.session.request(method, url, **kwargs) 172 | if raise_for_status: 173 | r.raise_for_status() 174 | if enforce_json: 175 | if "application/json" in self.session.headers.get("Accept", ""): 176 | try: 177 | r.json() 178 | except ValueError as e: 179 | raise CortexError("Invalid JSON: {}".format(e)) 180 | self.stats.transactions += 1 181 | return r 182 | 183 | def request(self, **kwargs): 184 | """Generate HTTP request using given parameters. 185 | 186 | :::info 187 | The request method prepares HTTP requests using class or 188 | method-level attributes/variables. Class-level attributes may be 189 | overridden by method-level variables offering greater 190 | flexibility and efficiency. 191 | ::: 192 | 193 | Parameters: 194 | enforce_json (bool): Require properly-formatted JSON or raise [HTTPError](exceptions.md#httperror). Defaults to `False`. 195 | path (str): URI path to append to URL. Defaults to `empty`. 196 | raise_for_status (bool): If `True`, raises [HTTPError](exceptions.md#httperror) if status_code not in 2XX. Defaults to `False`. 197 | 198 | Args: 199 | **kwargs: Supported [Session](https://github.com/psf/requests/blob/main/requests/sessions.py#L337) and 200 | [HTTPAdapter](https://github.com/psf/requests/blob/main/requests/adapters.py#L85) parameters. 201 | 202 | Returns: 203 | requests.Response: Requests [Response()](https://docs.python-requests.org/en/latest/api/#requests.Response) object 204 | 205 | Raises: 206 | HTTPError: If `raise_for_status = True` and non-2XX HTTP status returned or `enforce_json = True` and failure to decode JSON 207 | response or `HTTPError` raised by requests. 208 | RequiredKwargsError: If `method` kwarg not included in `request()`. 209 | UnexpectedKwargsError: If unsupported kwarg is passed. 210 | 211 | """ 212 | url = kwargs.pop("url", self.url) 213 | 214 | # Session() overrides 215 | auth = kwargs.pop("auth", self.session.auth) 216 | cert = kwargs.pop("cert", self.session.cert) 217 | cookies = kwargs.pop("cookies", self.session.cookies) 218 | headers = kwargs.pop("headers", self.session.headers.copy()) 219 | params = kwargs.pop("params", self.session.params) 220 | proxies = kwargs.pop("proxies", self.session.proxies) 221 | stream = kwargs.pop("stream", self.session.stream) 222 | verify = kwargs.pop("verify", self.session.verify) 223 | 224 | # Non-Requests key-word arguments 225 | auto_refresh = kwargs.pop("auto_refresh", self.auto_refresh) 226 | credentials = kwargs.pop("credentials", self.credentials) 227 | endpoint = kwargs.pop("endpoint", "") # default to empty endpoint 228 | enforce_json = kwargs.pop("enforce_json", self.enforce_json) 229 | raise_for_status = kwargs.pop("raise_for_status", self.raise_for_status) 230 | url = "{}:{}{}".format(url, self.port, endpoint) 231 | 232 | if credentials: 233 | logger.debug("Applying method-level credentials") 234 | self._apply_credentials( 235 | auto_refresh=auto_refresh, credentials=credentials, headers=headers 236 | ) 237 | 238 | k = { # Re-pack kwargs to dictionary 239 | "params": params, 240 | "headers": headers, 241 | "cookies": cookies, 242 | "auth": auth, 243 | "proxies": proxies, 244 | "verify": verify, 245 | "stream": stream, 246 | "cert": cert, 247 | } 248 | 249 | # Request() overrides 250 | for x in ["allow_redirects", "data", "json", "method", "timeout"]: 251 | if x in kwargs: 252 | k[x] = kwargs.pop(x) 253 | 254 | # Handle invalid kwargs 255 | if len(kwargs) > 0: 256 | raise UnexpectedKwargsError(kwargs) 257 | 258 | try: 259 | method = k.pop("method") 260 | except KeyError: 261 | raise RequiredKwargsError("method") 262 | 263 | # Prepare and send the Request() and return Response() 264 | try: 265 | r = self._send_request(enforce_json, method, raise_for_status, url, **k) 266 | return r 267 | except requests.RequestException as e: 268 | raise HTTPError(e) 269 | -------------------------------------------------------------------------------- /pan_cortex_data_lake/credentials.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | :::info 5 | The Credentials object can be used to access, store and refresh credentials. 6 | ::: 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | import os 12 | import sys 13 | import uuid 14 | from collections import namedtuple 15 | from threading import Lock 16 | 17 | try: 18 | from urllib.parse import urlparse 19 | except ImportError: 20 | from urlparse import urlparse 21 | 22 | from requests import Request 23 | from time import time 24 | from base64 import b64decode 25 | from json import loads 26 | 27 | from .httpclient import HTTPClient 28 | from .exceptions import CortexError, PartialCredentialsError 29 | 30 | # Constants 31 | API_BASE_URL = "https://api.paloaltonetworks.com" 32 | AUTH_BASE_URL = "https://identity.paloaltonetworks.com/as/authorization.oauth2" 33 | DEVELOPER_TOKEN_PROVIDER = "https://app.apiexplorer.rocks/request_token" 34 | 35 | ReadOnlyCredentials = namedtuple( 36 | "ReadOnlyCredentials", 37 | ["access_token", "client_id", "client_secret", "refresh_token"], 38 | ) 39 | 40 | 41 | class Credentials(object): 42 | """An Application Framework credentials object.""" 43 | 44 | def __init__( 45 | self, 46 | access_token=None, 47 | auth_base_url=None, 48 | cache_token=True, 49 | client_id=None, 50 | client_secret=None, 51 | developer_token=None, 52 | developer_token_provider=None, 53 | instance_id=None, 54 | profile=None, 55 | redirect_uri=None, 56 | region=None, 57 | refresh_token=None, 58 | scope=None, 59 | storage_adapter=None, 60 | storage_params=None, 61 | token_url=None, 62 | **kwargs 63 | ): 64 | """Persist `Session()` and credentials attributes. 65 | 66 | :::info 67 | The `Credentials` class is an abstraction layer for accessing, 68 | storing and refreshing credentials needed for interacting with 69 | the Application Framework. 70 | 71 | `Credentials` resolves credentials from the following locations, 72 | in the following order: 73 | 74 | 1. Class instance variables 75 | 2. Environment variables 76 | 3. Credentials store 77 | ::: 78 | 79 | Args: 80 | access_token (str): OAuth2 access token. Defaults to `None`. 81 | auth_base_url (str): IdP base authorization URL. Default to `None`. 82 | cache_token (bool): If `True`, stores `access_token` in token store. Defaults to `True`. 83 | client_id (str): OAuth2 client ID. Defaults to `None`. 84 | client_secret (str): OAuth2 client secret. Defaults to `None`. 85 | developer_token (str): Developer Token. Defaults to `None`. 86 | developer_token_provider (str): Developer Token Provider URL. Defaults to `None`. 87 | instance_id (str): Instance ID. Defaults to `None`. 88 | profile (str): Credentials profile. Defaults to 'default'. 89 | redirect_uri (str): Redirect URI. Defaults to `None`. 90 | region (str): Region. Defaults to `None`. 91 | refresh_token (str): OAuth2 refresh token. Defaults to `None`. 92 | scope (str): OAuth2 scope. Defaults to `None`. 93 | storage_adapter (str): Namespace path to storage adapter module. Defaults to "pan_cortex_data_lake.adapters.tinydb_adapter.TinyDBStore". 94 | storage_params (dict): Storage adapter parameters. Defaults to `None`. 95 | token_url (str): Refresh URL. Defaults to `None`. 96 | token_revoke_url (str): Revoke URL. Defaults to `None`. 97 | **kwargs: Supported [Session](https://github.com/psf/requests/blob/main/requests/sessions.py#L337) parameters. 98 | 99 | Examples: 100 | 101 | ```python 102 | from pan_cortex_data_lake import Credentials 103 | 104 | 105 | # Load credentials from envars or ~/.config/pan_cortex_data_lake/credentials.json 106 | c = Credentials() 107 | 108 | # Load credentials with static access_token 109 | access_token = "eyJ..." 110 | c = Credentials(access_token=access_token) 111 | 112 | # Load full credentials 113 | client_id = "trash" 114 | client_secret = "panda" 115 | refresh_token = "eyJ..." 116 | 117 | c = Credentials( 118 | client_id=client_id, 119 | client_secret=client_secret, 120 | refresh_token=refresh_token 121 | ) 122 | ``` 123 | 124 | """ 125 | self.access_token_ = access_token 126 | self.auth_base_url = auth_base_url or AUTH_BASE_URL 127 | self.cache_token_ = cache_token 128 | self.client_id_ = client_id 129 | self.client_secret_ = client_secret 130 | self.developer_token_ = developer_token 131 | self.developer_token_provider_ = developer_token_provider 132 | self.instance_id = instance_id 133 | self.jwt_exp_ = None 134 | self.profile = profile or "default" 135 | self.redirect_uri = redirect_uri 136 | self.region = region 137 | self.refresh_token_ = refresh_token 138 | self.scope = scope 139 | self.session = kwargs.pop("session", None) 140 | self.state = None 141 | self.adapter = ( 142 | storage_adapter 143 | or "pan_cortex_data_lake.adapters.tinydb_adapter.TinyDBStore" 144 | ) 145 | self.storage = self._init_adapter(storage_params) 146 | self.token_lock = Lock() 147 | self.token_url = token_url or API_BASE_URL 148 | self._credentials_found_in_instance = any( 149 | [ 150 | self.access_token_, 151 | self.client_id_, 152 | self.client_secret_, 153 | self.refresh_token_, 154 | ] 155 | ) 156 | self._httpclient = self.session or HTTPClient(**kwargs) 157 | 158 | def __repr__(self): 159 | args = self.__dict__.copy() 160 | for k in [ 161 | "access_token_", 162 | "refresh_token_", 163 | "client_secret_", 164 | "developer_token_", 165 | ]: 166 | if args[k] is not None: 167 | args[k] = "*" * 6 168 | return "{}({})".format( 169 | self.__class__.__name__, 170 | ", ".join("%s=%r" % x for x in args.items()), 171 | ) 172 | 173 | @property 174 | def access_token(self): 175 | """Get access_token.""" 176 | if self.cache_token: 177 | return self.access_token_ or self._resolve_credential("access_token") 178 | return self.access_token_ 179 | 180 | @access_token.setter 181 | def access_token(self, access_token): 182 | """Set access_token.""" 183 | self.access_token_ = access_token 184 | 185 | @property 186 | def cache_token(self): 187 | """Get cache_token setting.""" 188 | return self.cache_token_ 189 | 190 | @property 191 | def client_id(self): 192 | """Get client_id.""" 193 | return self.client_id_ or self._resolve_credential("client_id") 194 | 195 | @client_id.setter 196 | def client_id(self, client_id): 197 | """Set client_id.""" 198 | self.client_id_ = client_id 199 | 200 | @property 201 | def client_secret(self): 202 | """Get client_secret.""" 203 | return self.client_secret_ or self._resolve_credential("client_secret") 204 | 205 | @client_secret.setter 206 | def client_secret(self, client_secret): 207 | """Set client_secret.""" 208 | self.client_secret_ = client_secret 209 | 210 | @property 211 | def developer_token(self): 212 | """Get developer token.""" 213 | return self.developer_token_ or os.getenv("PAN_DEVELOPER_TOKEN") 214 | 215 | @developer_token.setter 216 | def developer_token(self, developer_token): 217 | """Set developer token.""" 218 | self.developer_token_ = developer_token 219 | 220 | @property 221 | def developer_token_provider(self): 222 | """Get developer token provider.""" 223 | return ( 224 | self.developer_token_provider_ 225 | or os.getenv("PAN_DEVELOPER_TOKEN_PROVIDER") 226 | or DEVELOPER_TOKEN_PROVIDER 227 | ) 228 | 229 | @developer_token_provider.setter 230 | def developer_token_provider(self, developer_token_provider): 231 | """Set developer token provider.""" 232 | self.developer_token_provider_ = developer_token_provider 233 | 234 | @property 235 | def jwt_exp(self): 236 | """Get JWT exp.""" 237 | return self.jwt_exp_ or self._decode_exp() 238 | 239 | @jwt_exp.setter 240 | def jwt_exp(self, jwt_exp): 241 | """Set jwt_exp.""" 242 | self.jwt_exp_ = jwt_exp 243 | 244 | @property 245 | def refresh_token(self): 246 | """Get refresh_token.""" 247 | return self.refresh_token_ or self._resolve_credential("refresh_token") 248 | 249 | @refresh_token.setter 250 | def refresh_token(self, refresh_token): 251 | """Set refresh_token.""" 252 | self.refresh_token_ = refresh_token 253 | 254 | @staticmethod 255 | def _credentials_found_in_envars(): 256 | """Check for credentials in envars. 257 | 258 | Returns: 259 | bool: `True` if at least one is found, otherwise `False`. 260 | 261 | """ 262 | return any( 263 | [ 264 | os.getenv("PAN_ACCESS_TOKEN"), 265 | os.getenv("PAN_CLIENT_ID"), 266 | os.getenv("PAN_CLIENT_SECRET"), 267 | os.getenv("PAN_REFRESH_TOKEN"), 268 | ] 269 | ) 270 | 271 | def _init_adapter(self, storage_params=None): 272 | module_path = self.adapter.rsplit(".", 1)[0] 273 | adapter = self.adapter.split(".")[-1] 274 | try: 275 | __import__(module_path) 276 | except ImportError as e: 277 | raise CortexError("Module import error: %s: %s" % (module_path, e)) 278 | 279 | try: 280 | class_ = getattr(sys.modules[module_path], adapter) 281 | except AttributeError: 282 | raise CortexError("Class not found: %s" % adapter) 283 | 284 | return class_(storage_params=storage_params) 285 | 286 | def _resolve_credential(self, credential): 287 | """Resolve credential from envars or credentials store. 288 | 289 | Args: 290 | credential (str): Credential to resolve. 291 | 292 | Returns: 293 | str or None: Resolved credential or `None`. 294 | 295 | """ 296 | if self._credentials_found_in_instance: 297 | return 298 | elif self._credentials_found_in_envars(): 299 | return os.getenv("PAN_" + credential.upper()) 300 | else: 301 | return self.storage.fetch_credential( 302 | credential=credential, profile=self.profile 303 | ) 304 | 305 | def decode_jwt_payload(self, access_token=None): 306 | """Extract payload field from JWT. 307 | 308 | Args: 309 | access_token (str): Access token to decode. Defaults to `None`. 310 | 311 | Returns: 312 | dict: JSON object that contains the claims conveyed by the JWT. 313 | 314 | Raises: 315 | CortexError: If unable to decode JWT payload. 316 | 317 | """ 318 | c = self.get_credentials() 319 | jwt = access_token or c.access_token 320 | try: 321 | _, payload, _ = jwt.split(".") # header, payload, sig 322 | rem = len(payload) % 4 323 | if rem > 0: # add padding 324 | payload += "=" * (4 - rem) 325 | try: 326 | decoded_jwt = b64decode(payload).decode("utf-8") 327 | except TypeError as e: 328 | raise CortexError("Failed to base64 decode JWT: %s" % e) 329 | else: 330 | try: 331 | x = loads(decoded_jwt) 332 | except ValueError as e: 333 | raise CortexError("Invalid JSON: %s" % e) 334 | except (AttributeError, ValueError) as e: 335 | raise CortexError("Invalid JWT: %s" % e) 336 | 337 | return x 338 | 339 | def _decode_exp(self, access_token=None): 340 | """Extract exp field from access token. 341 | 342 | Args: 343 | access_token (str): Access token to decode. Defaults to `None`. 344 | 345 | Returns: 346 | int: JWT expiration in epoch seconds. 347 | 348 | Raises: 349 | CortexError: If `exp` not in JWT claims. 350 | 351 | """ 352 | c = self.get_credentials() 353 | jwt = access_token or c.access_token 354 | x = self.decode_jwt_payload(jwt) 355 | 356 | if "exp" in x: 357 | try: 358 | exp = int(x["exp"]) 359 | except ValueError: 360 | raise CortexError("Expiration time (exp) must be an integer") 361 | else: 362 | self.jwt_exp = exp 363 | return exp 364 | else: 365 | raise CortexError("No exp field found in payload") 366 | 367 | def fetch_tokens( 368 | self, client_id=None, client_secret=None, code=None, redirect_uri=None, **kwargs 369 | ): 370 | """Exchange authorization code for token. 371 | 372 | Args: 373 | client_id (str): OAuth2 client ID. Defaults to `None`. 374 | client_secret (str): OAuth2 client secret. Defaults to `None`. 375 | code (str): Authorization code. Defaults to `None`. 376 | redirect_uri (str): Redirect URI. Defaults to `None`. 377 | 378 | Returns: 379 | dict: Response from token URL. 380 | 381 | Raises: 382 | CortexError: If non-2XX response or 'error' received from API or invalid JSON. 383 | 384 | """ 385 | client_id = client_id or self.client_id 386 | client_secret = client_secret or self.client_secret 387 | redirect_uri = redirect_uri or self.redirect_uri 388 | data = { 389 | "grant_type": "authorization_code", 390 | "client_id": client_id, 391 | "client_secret": client_secret, 392 | "code": code, 393 | "redirect_uri": redirect_uri, 394 | } 395 | r = self._httpclient.request( 396 | method="POST", 397 | url=self.token_url, 398 | json=data, 399 | endpoint="/api/oauth2/RequestToken", 400 | auth=None, 401 | **kwargs 402 | ) 403 | if not r.ok: 404 | raise CortexError("%s %s: %s" % (r.status_code, r.reason, r.text)) 405 | try: 406 | r_json = r.json() 407 | except ValueError as e: 408 | raise CortexError("Invalid JSON: %s" % e) 409 | else: 410 | if r.json().get("error_description") or r.json().get("error"): 411 | raise CortexError(r.text) 412 | self.access_token = r_json.get("access_token") 413 | self.jwt_exp = self._decode_exp(self.access_token_) 414 | self.refresh_token = r_json.get("refresh_token") 415 | self.write_credentials() 416 | return r_json 417 | 418 | def get_authorization_url( 419 | self, 420 | client_id=None, 421 | instance_id=None, 422 | redirect_uri=None, 423 | region=None, 424 | scope=None, 425 | state=None, 426 | ): 427 | """Generate authorization URL. 428 | 429 | Args: 430 | client_id (str): OAuth2 client ID. Defaults to `None`. 431 | instance_id (str): App Instance ID. Defaults to `None`. 432 | redirect_uri (str): Redirect URI. Defaults to `None`. 433 | region (str): App Region. Defaults to `None`. 434 | scope (str): Permissions. Defaults to `None`. 435 | state (str): UUID to detect CSRF. Defaults to `None`. 436 | 437 | Returns: 438 | str, str: Auth URL, state 439 | 440 | """ 441 | client_id = client_id or self.client_id 442 | instance_id = instance_id or self.instance_id 443 | redirect_uri = redirect_uri or self.redirect_uri 444 | region = region or self.region 445 | scope = scope or self.scope 446 | state = state or str(uuid.uuid4()) 447 | self.state = state 448 | return ( 449 | Request( 450 | "GET", 451 | self.auth_base_url, 452 | params={ 453 | "client_id": client_id, 454 | "instance_id": instance_id, 455 | "redirect_uri": redirect_uri, 456 | "region": region, 457 | "response_type": "code", 458 | "scope": scope, 459 | "state": state, 460 | }, 461 | ) 462 | .prepare() 463 | .url, 464 | state, 465 | ) 466 | 467 | def get_credentials(self): 468 | """Get read-only credentials. 469 | 470 | Returns: 471 | class: Read-only credentials. 472 | 473 | """ 474 | return ReadOnlyCredentials( 475 | self.access_token, self.client_id, self.client_secret, self.refresh_token 476 | ) 477 | 478 | def jwt_is_expired(self, access_token=None, leeway=0): 479 | """Validate JWT access token expiration. 480 | 481 | Args: 482 | access_token (str): Access token to validate. Defaults to `None`. 483 | leeway (float): Time in seconds to adjust for local clock skew. Defaults to 0. 484 | 485 | Returns: 486 | bool: `True` if expired, otherwise `False`. 487 | 488 | """ 489 | if access_token is not None: 490 | exp = self._decode_exp(access_token) 491 | else: 492 | exp = self.jwt_exp 493 | now = time() 494 | if exp < (now - leeway): 495 | return True 496 | return False 497 | 498 | def remove_profile(self, profile): 499 | """Remove profile from credentials store. 500 | 501 | Args: 502 | profile (str): Credentials profile to remove. 503 | 504 | Returns: 505 | Return value of `self.storage.remove_profile()`. 506 | 507 | """ 508 | return self.storage.remove_profile(profile=profile) 509 | 510 | def refresh(self, access_token=None, **kwargs): 511 | """Refresh access and refresh tokens. 512 | 513 | Args: 514 | access_token (str): Access token to refresh. Defaults to `None`. 515 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 516 | 517 | Returns: 518 | str: Refreshed access token and refresh token (if available). 519 | 520 | Raises: 521 | CortexError: If non-2XX response or 'error' received from API or invalid JSON. 522 | PartialCredentialsError: If one or more required credentials are missing. 523 | 524 | """ 525 | if not self.token_lock.locked(): 526 | with self.token_lock: 527 | if access_token == self.access_token or access_token is None: 528 | if self.developer_token is not None and not any( 529 | [ 530 | os.getenv("PAN_ACCESS_TOKEN"), 531 | self._credentials_found_in_instance, 532 | ] 533 | ): 534 | parsed_provider = urlparse(self.developer_token_provider) 535 | url = "{}://{}".format( 536 | parsed_provider.scheme, parsed_provider.netloc 537 | ) 538 | endpoint = parsed_provider.path 539 | r = self._httpclient.request( 540 | method="POST", 541 | url=url, 542 | endpoint=endpoint, 543 | headers={ 544 | "Authorization": "Bearer {}".format( 545 | self.developer_token 546 | ) 547 | }, 548 | timeout=30, 549 | raise_for_status=True, 550 | ) 551 | 552 | elif all([self.client_id, self.client_secret, self.refresh_token]): 553 | data = { 554 | "client_id": self.client_id, 555 | "client_secret": self.client_secret, 556 | "refresh_token": self.refresh_token, 557 | "grant_type": "refresh_token", 558 | } 559 | r = self._httpclient.request( 560 | method="POST", 561 | url=self.token_url, 562 | json=data, 563 | endpoint="/api/oauth2/RequestToken", 564 | **kwargs 565 | ) 566 | else: 567 | raise PartialCredentialsError( 568 | "Missing one or more required credentials" 569 | ) 570 | 571 | if r: 572 | if not r.ok: 573 | raise CortexError( 574 | "%s %s: %s" % (r.status_code, r.reason, r.text) 575 | ) 576 | try: 577 | r_json = r.json() 578 | except ValueError as e: 579 | raise CortexError("Invalid JSON: %s" % e) 580 | else: 581 | if r.json().get("error_description") or r.json().get( 582 | "error" 583 | ): 584 | raise CortexError(r.text) 585 | self.access_token = r_json.get("access_token", None) 586 | self.jwt_exp = self._decode_exp(self.access_token_) 587 | if r_json.get("refresh_token", None): 588 | self.refresh_token = r_json.get("refresh_token") 589 | self.write_credentials() 590 | return self.access_token_ 591 | 592 | def revoke_access_token(self, **kwargs): 593 | """Revoke access token. 594 | 595 | Args: 596 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 597 | 598 | Returns: 599 | dict: JSON object that contains the response from API. 600 | 601 | Raises: 602 | CortexError: If non-2XX response or 'error' received from API or invalid JSON. 603 | 604 | """ 605 | c = self.get_credentials() 606 | data = { 607 | "client_id": c.client_id, 608 | "client_secret": c.client_secret, 609 | "token": c.access_token, 610 | "token_type_hint": "access_token", 611 | } 612 | r = self._httpclient.request( 613 | method="POST", 614 | url=self.token_url, 615 | json=data, 616 | endpoint="/api/oauth2/RevokeToken", 617 | **kwargs 618 | ) 619 | if not r.ok: 620 | raise CortexError("%s %s: %s" % (r.status_code, r.reason, r.text)) 621 | try: 622 | r_json = r.json() 623 | except ValueError as e: 624 | raise CortexError("Invalid JSON: %s" % e) 625 | else: 626 | if r.json().get("error_description") or r.json().get("error"): 627 | raise CortexError(r.text) 628 | return r_json 629 | 630 | def revoke_refresh_token(self, **kwargs): 631 | """Revoke refresh token. 632 | 633 | Args: 634 | **kwargs: Supported [HTTPClient.request()](httpclient.md#request) parameters. 635 | 636 | Returns: 637 | dict: JSON object that contains the response from API. 638 | 639 | Raises: 640 | CortexError: If non-2XX response or 'error' received from API or invalid JSON. 641 | 642 | """ 643 | c = self.get_credentials() 644 | data = { 645 | "client_id": c.client_id, 646 | "client_secret": c.client_secret, 647 | "token": c.refresh_token, 648 | "token_type_hint": "refresh_token", 649 | } 650 | r = self._httpclient.request( 651 | method="POST", 652 | url=self.token_url, 653 | json=data, 654 | endpoint="/api/oauth2/RevokeToken", 655 | **kwargs 656 | ) 657 | if not r.ok: 658 | raise CortexError("%s %s: %s" % (r.status_code, r.reason, r.text)) 659 | try: 660 | r_json = r.json() 661 | except ValueError as e: 662 | raise CortexError("Invalid JSON: %s" % e) 663 | else: 664 | if r.json().get("error_description") or r.json().get("error"): 665 | raise CortexError(r.text) 666 | return r_json 667 | 668 | def write_credentials(self): 669 | """Write credentials. 670 | 671 | :::info 672 | Write credentials to credentials store. 673 | ::: 674 | 675 | Returns: 676 | Return value of `self.storage.write_credentials()`. 677 | 678 | """ 679 | c = self.get_credentials() 680 | return self.storage.write_credentials( 681 | credentials=c, profile=self.profile, cache_token=self.cache_token 682 | ) 683 | --------------------------------------------------------------------------------