├── .github └── workflows │ └── main.yml ├── .gitignore ├── .gitmodules ├── .pylintrc ├── .pylintrc.examples ├── LICENSE ├── MANIFEST.in ├── README ├── README.md ├── dist.py ├── docs ├── conf.py ├── docutils.conf └── sdk │ ├── README.html │ ├── addcertbasedauth1.png │ ├── addcertbasedauth10.png │ ├── addcertbasedauth2.png │ ├── addcertbasedauth3.png │ ├── addcertbasedauth4.png │ ├── addcertbasedauth5.png │ ├── addcertbasedauth6.png │ ├── addcertbasedauth7.png │ ├── addcertbasedauth8.png │ ├── addcertbasedauth9.png │ ├── addtopicgroup1.png │ ├── addtopicgroup2.png │ ├── addtopicgroup3.png │ ├── addtopicgroup4.png │ ├── addtopicgroup5.png │ ├── addtopicgroup6.png │ ├── advancedcliprovisioning.rst │ ├── advancedeventsexample.rst │ ├── advancedserviceexample.rst │ ├── architecture.rst │ ├── basiccliprovisioning.rst │ ├── basiceventsexample.rst │ ├── basicserviceexample.rst │ ├── certcreation.rst │ ├── editdxlcerts-save.png │ ├── editdxlcerts-selectca.png │ ├── editdxlcerts.png │ ├── enablemarauth1.png │ ├── enablemarauth2.png │ ├── enablemarauth3.png │ ├── enablemarauth4.png │ ├── enablemarauth5.png │ ├── enablemarauth6.png │ ├── enablemarauth7.png │ ├── epobrokercertsexport.rst │ ├── epobrokerlistexport.rst │ ├── epocaimport.rst │ ├── epoexternalcertissuance.rst │ ├── features.rst │ ├── index.rst │ ├── installation.rst │ ├── integrationtypes.rst │ ├── marsearchexample.rst │ ├── marsendauth.rst │ ├── openconsoleprovisioning.rst │ ├── overview.rst │ ├── provisioningoverview.rst │ ├── sampleconfig.rst │ ├── serversettings-certs.png │ ├── serversettings.png │ ├── servicewrapperexample.rst │ ├── tiefilerepexample.rst │ ├── topicauthgroupcreation.rst │ ├── topicauthgrouprestrictions.rst │ ├── topicauthoverview.rst │ └── updatingconfigfromcli.rst ├── dxlclient ├── __init__.py ├── __main__.py ├── _callback_manager.py ├── _cli │ ├── __init__.py │ ├── _cli_subcommands.py │ ├── _crypto.py │ └── _management_service.py ├── _compat.py ├── _dxl_utils.py ├── _global_settings.py ├── _product_props.py ├── _request_manager.py ├── _thread_pool.py ├── _uuid_generator.py ├── broker.py ├── callbacks.py ├── client.py ├── client_config.py ├── exceptions.py ├── message.py ├── service.py └── test │ ├── __init__.py │ ├── async_callback_timeout_test.py │ ├── async_flood_test.py │ ├── async_request_test.py │ ├── base_test.py │ ├── broker_registry_query_test.py │ ├── broker_service_registry_test.py │ ├── client_config.cfg.template │ ├── dxlclient_test.py │ ├── event_request_test.py │ ├── event_throughput_runner_test.py │ ├── message_other_fields_test.py │ ├── message_payload_test.py │ ├── register_service_test.py │ ├── subs_count_test.py │ ├── sync_during_callback_test.py │ ├── sync_request_test.py │ ├── sync_request_troughput_test.py │ ├── test_broker.py │ ├── test_callback_manager.py │ ├── test_cli.py │ ├── test_dxlclient.py │ ├── test_message.py │ ├── test_request_manager.py │ ├── test_service.py │ ├── test_wildcard.py │ └── thread_executor.py ├── examples ├── __init__.py ├── advanced │ ├── event_publisher_sample.py │ ├── event_subscriber_sample.py │ ├── service_invoker_sample.py │ └── service_provider_sample.py ├── basic │ ├── event_example.py │ └── service_example.py ├── common.py ├── dxlclient.config ├── mar │ └── search_example.py ├── servicewrapper │ ├── openweather_common.py │ ├── openweather_service_invoker.py │ └── openweather_service_wrapper.py └── tie │ └── file_rep_sample.py ├── setup.cfg └── setup.py /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | 2 | name: Build 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | pull_request: 9 | schedule: 10 | - cron: '0 0 * * *' 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | python-version: [3.7, 3.8, 3.9] 19 | fail-fast: false 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v1 24 | with: 25 | submodules: recursive 26 | 27 | - name: Set up Python ${{ matrix.python-version }} 28 | uses: actions/setup-python@v1 29 | with: 30 | python-version: ${{ matrix.python-version }} 31 | 32 | - name: Install broker 33 | run: | 34 | docker pull opendxl/opendxl-broker 35 | docker run -d -p 8883:8883 -p 8443:8443 -p 443:443 opendxl/opendxl-broker 36 | docker ps -a 37 | 38 | - name: Install dependencies 39 | run: | 40 | pip install --upgrade pip 41 | pip install --upgrade pipenv --no-deps 42 | pip install wheel 43 | pip install .[test] 44 | python setup.py install 45 | 46 | - name: Execute CI 47 | run: | 48 | python -m dxlclient provisionconfig dxlclient/test 127.0.0.1 client -u admin -p password 49 | cp dxlclient/test/dxlclient.config dxlclient/test/client_config.cfg 50 | sed -i -e "s/127.0.0.1;127.0.0.1/127.0.0.1/g" -e "/local/d" -e "/docker/d" dxlclient/test/client_config.cfg 51 | cat dxlclient/test/client_config.cfg 52 | echo Running tests with MQTT 53 | python setup.py ci 54 | sed -i -e "s/= False/= True/g" -e "s/;8883/;443/g" -e "s/Brokers/BrokersWebSockets/g" dxlclient/test/client_config.cfg 55 | cat dxlclient/test/client_config.cfg 56 | echo Running tests with WebSockets 57 | python setup.py ci 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | dxlclient.egg-info 4 | .eggs 5 | MANIFEST 6 | dxlclient/test/client_config.cfg 7 | *.pyc 8 | *.crt 9 | *.csr 10 | *.key 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "paho_mqtt_dxl"] 2 | path = paho_mqtt_dxl 3 | url = https://github.com/opendxl-community/paho.mqtt.python.git 4 | branch = master 5 | [submodule "oscrypto"] 6 | path = oscrypto 7 | url = https://github.com/wbond/oscrypto.git 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | 3 | recursive-exclude dxlclient/test * 4 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | OpenDXL Python Client 2 | ===================== 3 | 4 | Overview 5 | -------- 6 | The OpenDXL Python Client enables the development of applications that connect to the `McAfee Data Exchange Layer `_ messaging fabric for the purposes of sending/receiving events and invoking/providing services. 7 | 8 | Documentation 9 | ------------- 10 | 11 | See the `GitHub Wiki `_ for an overview of the Data Exchange Layer (DXL), the OpenDXL Python client, and examples. 12 | 13 | See the `Python Client SDK Documentation `_ for installation instructions, API documentation, and examples. 14 | 15 | Bugs and Feedback 16 | ----------------- 17 | 18 | For bugs, questions and discussions please use the `GitHub Issues `_. 19 | 20 | LICENSE 21 | ------- 22 | 23 | Copyright 2018 McAfee, LLC 24 | 25 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 26 | 27 | ``_ 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenDXL Python Client 2 | 3 | [![Latest PyPI Version](https://img.shields.io/pypi/v/dxlclient.svg)](https://pypi.python.org/pypi/dxlclient) 4 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 5 | [![Actions Status](https://github.com/opendxl/opendxl-client-python/workflows/Build/badge.svg)](https://github.com/opendxl/opendxl-client-python/actions) 6 | 7 | ## Overview 8 | 9 | The OpenDXL Python Client enables the development of applications that connect to the [McAfee Data Exchange Layer](http://www.mcafee.com/us/solutions/data-exchange-layer.aspx) messaging fabric for the purposes of sending/receiving events and invoking/providing services. 10 | 11 | ## Documentation 12 | 13 | See the [Wiki](https://github.com/opendxl/opendxl-client-python/wiki) for an overview of the Data Exchange Layer (DXL), the OpenDXL Python client, and examples. 14 | 15 | See the [Python Client SDK Documentation](https://opendxl.github.io/opendxl-client-python/pydoc) for installation instructions, API documentation, and examples. 16 | 17 | ## Installation 18 | 19 | To start using the OpenDXL Python client: 20 | 21 | * Download the [Latest Release](https://github.com/opendxl/opendxl-client-python/releases/latest) 22 | * Extract the release .zip file 23 | * View the `README.html` file located at the root of the extracted files. 24 | * The `README` links to the SDK documentation which includes installation instructions, API details, and samples. 25 | * The SDK documentation is also available on-line [here](https://opendxl.github.io/opendxl-client-python/pydoc). 26 | 27 | ## Bugs and Feedback 28 | 29 | For bugs, questions and discussions please use the [Github Issues](https://github.com/opendxl/opendxl-client-python/issues). 30 | 31 | ## LICENSE 32 | 33 | Copyright 2024 Musarubra US LLC. 34 | 35 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 36 | 37 | http://www.apache.org/licenses/LICENSE-2.0 38 | 39 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 40 | -------------------------------------------------------------------------------- /dist.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ Builds documentation and packages for release """ 7 | 8 | from __future__ import absolute_import 9 | from __future__ import print_function 10 | import os 11 | import re 12 | import subprocess 13 | 14 | # pylint: disable=no-name-in-module, import-error, missing-docstring 15 | from distutils.dir_util import copy_tree, remove_tree 16 | from distutils.file_util import copy_file, move_file 17 | from distutils.core import run_setup 18 | from distutils.archive_util import make_archive 19 | 20 | from tempfile import mkstemp 21 | from shutil import move 22 | 23 | def replace(file_path, pattern, subst): 24 | # Create temp file 25 | handle, abs_path = mkstemp() 26 | with open(abs_path, "w") as new_file: 27 | with open(file_path) as old_file: 28 | for line in old_file: 29 | new_file.write(line.replace(pattern, subst)) 30 | os.close(handle) 31 | # Remove original file 32 | os.remove(file_path) 33 | # Move new file 34 | move(abs_path, file_path) 35 | 36 | VERSION = __import__('dxlclient').get_product_version() 37 | RELEASE_NAME = "dxlclient-python-sdk-" + str(VERSION) 38 | 39 | DIST_PY_FILE_LOCATION = os.path.dirname(os.path.realpath(__file__)) 40 | DIST_DIRECTORY = os.path.join(DIST_PY_FILE_LOCATION, "dist") 41 | DIST_DOCTMP_DIR = os.path.join(DIST_DIRECTORY, "doctmp") 42 | SETUP_PY = os.path.join(DIST_PY_FILE_LOCATION, "setup.py") 43 | DIST_LIB_DIRECTORY = os.path.join(DIST_DIRECTORY, "lib") 44 | DIST_RELEASE_DIR = os.path.join(DIST_DIRECTORY, RELEASE_NAME) 45 | 46 | # Remove the dist directory if it exists 47 | if os.path.exists(DIST_DIRECTORY): 48 | print("\nRemoving dist directory: " + DIST_DIRECTORY + "\n") 49 | remove_tree(DIST_DIRECTORY, verbose=1) 50 | 51 | # Make the dist directory 52 | print("\nMaking dist directory: " + DIST_DIRECTORY + "\n") 53 | os.makedirs(DIST_DIRECTORY) 54 | 55 | # Call Sphinx to create API doc 56 | print("\nCalling sphinx-apidoc\n") 57 | subprocess.check_call(["sphinx-apidoc", 58 | "--force", 59 | "--separate", 60 | "--no-toc", 61 | "--output-dir=" + DIST_DOCTMP_DIR, 62 | os.path.join(DIST_PY_FILE_LOCATION, "dxlclient")]) 63 | 64 | # Delete test files 65 | for f in os.listdir(DIST_DOCTMP_DIR): 66 | if re.search("dxlclient.test.*", f): 67 | os.remove(os.path.join(DIST_DOCTMP_DIR, f)) 68 | 69 | # Copy files to dist/doctmp 70 | print("\nCopying conf.py, docutils.conf, and sdk directory\n") 71 | copy_file(os.path.join(DIST_PY_FILE_LOCATION, "docs", "conf.py"), 72 | os.path.join(DIST_DOCTMP_DIR, "conf.py")) 73 | copy_file(os.path.join(DIST_PY_FILE_LOCATION, "docs", "docutils.conf"), 74 | os.path.join(DIST_DOCTMP_DIR, "docutils.conf")) 75 | copy_tree(os.path.join(DIST_PY_FILE_LOCATION, "docs", "sdk"), DIST_DOCTMP_DIR) 76 | 77 | # Call Sphinx build 78 | print("\nCalling sphinx-build\n") 79 | subprocess.check_call(["sphinx-build", "-b", "html", DIST_DOCTMP_DIR, os.path.join(DIST_DIRECTORY, "doc")]) 80 | 81 | replace(os.path.join(DIST_DIRECTORY, "doc", "_static", "classic.css"), "text-align: justify", "text-align: none") 82 | 83 | # Delete .doctrees 84 | remove_tree(os.path.join(os.path.join(DIST_DIRECTORY, "doc"), ".doctrees"), verbose=1) 85 | # Delete .buildinfo 86 | os.remove(os.path.join(os.path.join(DIST_DIRECTORY, "doc"), ".buildinfo")) 87 | 88 | # Move README.html to root of dist directory 89 | print("\nMoving README.html\n") 90 | move_file(os.path.join(DIST_DOCTMP_DIR, "README.html"), DIST_DIRECTORY) 91 | 92 | # Remove doctmp directory 93 | print("\nDeleting doctmp directory\n") 94 | remove_tree(DIST_DOCTMP_DIR) 95 | 96 | # Call setup.py 97 | print("\nRunning setup.py sdist\n") 98 | run_setup(SETUP_PY, 99 | ["sdist", 100 | "--format", 101 | "zip", 102 | "--dist-dir", 103 | DIST_LIB_DIRECTORY]) 104 | 105 | print("\nRunning setup.py bdist_wheel\n") 106 | run_setup(SETUP_PY, 107 | ["bdist_wheel", 108 | "--dist-dir", 109 | DIST_LIB_DIRECTORY, 110 | "--universal"]) 111 | 112 | # cp -rf sample dist 113 | print("\nCopying sample in to dist directory\n") 114 | copy_tree(os.path.join(DIST_PY_FILE_LOCATION, "examples"), os.path.join(DIST_DIRECTORY, "sample")) 115 | 116 | # Copy everything in to release dir 117 | print("\nCopying dist to " + DIST_RELEASE_DIR + "\n") 118 | copy_tree(DIST_DIRECTORY, DIST_RELEASE_DIR) 119 | 120 | # rm -rf build 121 | print("\nRemoving build directory\n") 122 | remove_tree(os.path.join(DIST_PY_FILE_LOCATION, "build")) 123 | 124 | # rm -rf dxlclient.egg-info 125 | print("\nRemoving dxlclient.egg-info\n") 126 | remove_tree(os.path.join(DIST_PY_FILE_LOCATION, "dxlclient.egg-info")) 127 | 128 | # Make dist zip 129 | print("\nMaking dist zip\n") 130 | make_archive(DIST_RELEASE_DIR, "zip", DIST_DIRECTORY, RELEASE_NAME) 131 | 132 | print("\nRemoving " + DIST_RELEASE_DIR + "\n") 133 | remove_tree(DIST_RELEASE_DIR) 134 | 135 | print("\nFinished") 136 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # DXL Python SDK documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Jan 13 11:34:32 2015. 5 | 6 | from __future__ import absolute_import 7 | import sys 8 | import os 9 | 10 | # If extensions (or modules to document with autodoc) are in another directory, 11 | # add these directories to sys.path here. If the directory is relative to the 12 | # documentation root, use os.path.abspath to make it absolute, like shown here. 13 | 14 | sys.path.insert(0, os.path.abspath('../../')) 15 | 16 | # -- General configuration ------------------------------------------------ 17 | 18 | # Add any Sphinx extension module names here, as strings. They can be 19 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 20 | # ones. 21 | extensions = [ 22 | 'sphinx.ext.autodoc' 23 | ] 24 | 25 | # Add any paths that contain templates here, relative to this directory. 26 | templates_path = ['_templates'] 27 | 28 | # The suffix of source filenames. 29 | source_suffix = '.rst' 30 | 31 | # The master toctree document. 32 | master_doc = 'index' 33 | 34 | # General information about the project. 35 | project = u'OpenDXL Python SDK' 36 | copyright = u'2024 Musarubra US LLC' 37 | 38 | # The version info for the project you're documenting, acts as replacement for 39 | # |version| and |release|, also used in various other places throughout the 40 | # built documents. 41 | # 42 | VERSION = __import__('dxlclient').get_product_version() 43 | # The short X.Y version. 44 | version = VERSION 45 | # The full version, including alpha/beta/rc tags. 46 | release = VERSION 47 | 48 | # List of patterns, relative to source directory, that match files and 49 | # directories to ignore when looking for source files. 50 | exclude_patterns = ['_build'] 51 | 52 | # The name of the Pygments (syntax highlighting) style to use. 53 | pygments_style = 'sphinx' 54 | 55 | # -- Options for HTML output ---------------------------------------------- 56 | 57 | # The theme to use for HTML and HTML Help pages. See the documentation for 58 | # a list of builtin themes. 59 | html_theme = 'default' 60 | 61 | # Output file base name for HTML help builder. 62 | htmlhelp_basename = 'sphinxdoc' 63 | 64 | autoclass_content = 'both' 65 | 66 | modindex_common_prefix = ['dxlclient.'] 67 | 68 | html_use_smartypants = False 69 | -------------------------------------------------------------------------------- /docs/docutils.conf: -------------------------------------------------------------------------------- 1 | [parsers] 2 | smart_quotes = False 3 | -------------------------------------------------------------------------------- /docs/sdk/README.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Data Exchange Layer (DXL) Python SDK Documentation 5 | 6 | 7 | 8 | 9 |
10 |
11 |
12 |
13 |

Data Exchange Layer (DXL) Python SDK Documentation

14 |

15 | The DXL Python SDK contains the following directories: 16 |

    17 |
  • doc
    • Documentation for the DXL Python SDK
    • 18 |
  • lib
    • DXL Python libraries (.zip, .whl)
    • 19 |
  • sample
    • Samples that demonstrate use of the DXL Python SDK
    • 20 |
21 |

22 |

23 | Click here to view the documentation which 24 | includes installation instructions, Python DXL API details, and samples. 25 |

26 |
27 |
28 |
29 |
30 |
31 | © Copyright 2018, McAfee LLC 32 |
33 |
34 | 35 | 36 | -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth1.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth10.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth2.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth3.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth4.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth5.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth6.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth7.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth8.png -------------------------------------------------------------------------------- /docs/sdk/addcertbasedauth9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addcertbasedauth9.png -------------------------------------------------------------------------------- /docs/sdk/addtopicgroup1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addtopicgroup1.png -------------------------------------------------------------------------------- /docs/sdk/addtopicgroup2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addtopicgroup2.png -------------------------------------------------------------------------------- /docs/sdk/addtopicgroup3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addtopicgroup3.png -------------------------------------------------------------------------------- /docs/sdk/addtopicgroup4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addtopicgroup4.png -------------------------------------------------------------------------------- /docs/sdk/addtopicgroup5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addtopicgroup5.png -------------------------------------------------------------------------------- /docs/sdk/addtopicgroup6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/addtopicgroup6.png -------------------------------------------------------------------------------- /docs/sdk/advancedcliprovisioning.rst: -------------------------------------------------------------------------------- 1 | Command Line Provisioning (Advanced) 2 | ==================================== 3 | 4 | This page contain details regarding the advanced usage of the 5 | ``provisionconfig`` operation. 6 | 7 | Refer to :doc:`basiccliprovisioning` for basic usage details. 8 | 9 | .. _subject-attributes-label: 10 | 11 | Additional Certificate Signing Request (CSR) Information 12 | ******************************************************** 13 | 14 | Attributes other than the Common Name (CN) may also optionally be provided for 15 | the CSR subject. 16 | 17 | For example:: 18 | 19 | dxlclient provisionconfig config myserver client1 --country US --state-or-province Oregon --locality Hillsboro --organization Engineering --organizational-unit "DXL Team" --email-address dxl@mcafee.com 20 | 21 | By default, the CSR does not include any Subject Alternative Names. To include 22 | one or more entries of type ``DNS Name``, provide the ``-s`` option. 23 | 24 | For example:: 25 | 26 | dxlclient provisionconfig config myserver client1 -s client1.myorg.com client1.myorg.net 27 | 28 | .. _encrypting-private-key-label: 29 | 30 | Encrypting the Client's Private Key 31 | *********************************** 32 | 33 | The private key file which the ``provisionconfig`` operation generates can 34 | optionally be encrypted with a passphrase. 35 | 36 | For example:: 37 | 38 | dxlclient provisionconfig config myserver client1 --passphrase 39 | 40 | If the passphrase is specified with no trailing option (as above), the 41 | provision operation prompts for the passphrase to be used:: 42 | 43 | Enter private key passphrase: 44 | 45 | The passphrase can alternatively be specified as an additional argument 46 | following the ``--passphrase`` argument, in which case no prompt is displayed. 47 | 48 | For example:: 49 | 50 | dxlclient provisionconfig config myserver client1 --passphrase itsasecret 51 | 52 | 53 | `NOTE:` If the private key is encrypted, the passphrase used to encrypt it 54 | must be specified when the client attempts to establish a connection to 55 | the DXL fabric. 56 | 57 | The only way to enter this passphrase is via a prompt:: 58 | 59 | Enter PEM pass phrase: 60 | 61 | Additional Options 62 | ****************** 63 | 64 | The provision operation assumes that the default web server port is 8443, 65 | the default port under which the ePO web interface and OpenDXL Broker Management 66 | Console is hosted. 67 | 68 | A custom port can be specified via the ``-t`` option. 69 | 70 | For example:: 71 | 72 | dxlclient provisionconfig config myserver client1 -t 443 73 | 74 | The provision operation stores each of the certificate artifacts (private key, CSR, 75 | certificate, etc.) with a base name of ``client`` by default. To use an 76 | alternative base name for the stored files, use the ``-f`` option. 77 | 78 | For example:: 79 | 80 | dxlclient provisionconfig config myserver client1 -f theclient 81 | 82 | The output of the command above should appear similar to the following:: 83 | 84 | INFO: Saving csr file to config/theclient.csr 85 | INFO: Saving private key file to config/theclient.key 86 | INFO: Saving DXL config file to config/dxlclient.config 87 | INFO: Saving ca bundle file to config/ca-bundle.crt 88 | INFO: Saving client certificate file to config/theclient.crt 89 | 90 | If the management server's CA certificate is stored in a local CA truststore 91 | file -- one or more PEM-formatted certificates concatenated together into a 92 | single file -- the provision operation can be configured to validate 93 | the management server's certificate against that truststore during TLS session 94 | negotiation by supplying the ``-e`` option. 95 | 96 | The name of the truststore file should be supplied along with the option:: 97 | 98 | dxlclient config myserver -e config/ca-bundle.crt 99 | 100 | Generating the CSR Separately from Signing the Certificate 101 | ********************************************************** 102 | 103 | By default, the ``provisionconfig`` command generates a CSR and immediately 104 | sends it to a management server for signing. Certificate generation and signing 105 | could alternatively be performed as separate steps -- for example, to enable a 106 | workflow where the CSR is signed by a certificate authority at a later time. 107 | 108 | The ``generatecsr`` operation can be used to generate the CSR and private 109 | key without sending the CSR to the server. 110 | 111 | For example:: 112 | 113 | dxlclient generatecsr config client1 114 | 115 | The output of the command above should appear similar to the following:: 116 | 117 | INFO: Saving csr file to config/client.csr 118 | INFO: Saving private key file to config/client.key 119 | 120 | Note that the ``generatecsr`` operation has options similar to those available 121 | in the ``provisionconfig`` operation for including additional subject attributes 122 | and/or subject alternative names in the generated CSR and for encrypting the 123 | private key. 124 | 125 | See the :ref:`subject-attributes-label` and :ref:`encrypting-private-key-label` 126 | sections for more information. 127 | 128 | If the ``provisionconfig`` operation includes a ``-r`` option, the 129 | ``COMMON_OR_CSRFILE_NAME`` argument is interpreted as the name of a 130 | CSR file to load from disk rather than the Common Name to insert into a new 131 | CSR file. 132 | 133 | For example:: 134 | 135 | dxlclient provisionconfig config myserver -r config/client.csr 136 | 137 | In this case, the command line output shows that the certificate and 138 | configuration-related files received from the server are stored but no 139 | new private key or CSR file is generated:: 140 | 141 | INFO: Saving DXL config file to config/dxlclient.config 142 | INFO: Saving ca bundle file to config/ca-bundle.crt 143 | INFO: Saving client certificate file to config/client.crt 144 | -------------------------------------------------------------------------------- /docs/sdk/advancedeventsexample.rst: -------------------------------------------------------------------------------- 1 | Advanced Events Sample 2 | ====================== 3 | 4 | Prior to running this sample make sure you have completed the samples configuration step (:doc:`sampleconfig`). 5 | 6 | This sample differs from the :doc:`basiceventsexample` by breaking the "event subscriber" and "event publisher" 7 | functionality into two distinct scripts. When executed, each of these scripts will contain a unique instance of 8 | a :class:`dxlclient.client.DxlClient`. 9 | 10 | Event Subscriber 11 | **************** 12 | 13 | The first step is to start the "event subscriber". This script will remain running and receive 14 | :class:`dxlclient.message.Event` messages from the "event publisher". 15 | 16 | To start the "event subscriber", execute the ``sample\advanced\event_subscriber_sample.py`` script as follows: 17 | 18 | .. parsed-literal:: 19 | 20 | c:\\dxlclient-python-sdk-\ |version|\>python sample\\advanced\\event_subscriber_sample.py 21 | 22 | The output should appear similar to the following: 23 | 24 | .. parsed-literal:: 25 | 26 | 2015-12-30 08:55:44,936 __main__ - INFO - Event Subscriber - Load DXL config from: c:\\dxlclient-python-sdk-\ |version|\\sample/dxlclient.config 27 | 2015-12-30 08:55:44,938 __main__ - INFO - Event Subscriber - Creating DXL Client 28 | 2015-12-30 08:55:44,956 __main__ - INFO - Event Subscriber - Connecting to Broker 29 | 2015-12-30 08:55:44,957 dxlclient.client - INFO - Waiting for broker list... 30 | 2015-12-30 08:55:44,957 dxlclient.client - INFO - Checking brokers... 31 | 2015-12-30 08:55:44,959 dxlclient.client - INFO - Trying to connect... 32 | 2015-12-30 08:55:44,959 dxlclient.client - INFO - Trying to connect to broker {Unique id: mybroker, Host name: mybroker.mcafee.com, IP address: 10.84.221.144, Port: 8883}... 33 | 2015-12-30 08:55:45,224 dxlclient.client - INFO - Connected to broker mybroker 34 | 2015-12-30 08:55:45,226 dxlclient.client - INFO - Launching event loop... 35 | 2015-12-30 08:55:45,226 dxlclient.client - INFO - Connected with result code 0 36 | 2015-12-30 08:55:45,226 dxlclient.client - INFO - Subscribing to /mcafee/client/{1d79d1a9-8efd-41f3-bc2a-bea5a34b9faa} 37 | 2015-12-30 08:55:45,229 __main__ - INFO - Event Subscriber - Subscribing to Topic: /isecg/sample/event 38 | 2015-12-30 08:55:45,229 __main__ - INFO - Adding Event callback function to Topic: /isecg/sample/event 39 | Enter 9 to quit 40 | Enter value: 41 | 42 | The subscriber will remain running until ``9`` is entered to quit. 43 | 44 | Any :class:`dxlclient.message.Event` messages received by the "event publisher" will be displayed in the output. 45 | 46 | The code for the subscriber is very similar to what is being used in the :doc:`basiceventsexample`: 47 | 48 | .. code-block:: python 49 | 50 | # Event callback class to handle incoming DXL Events 51 | class MyEventCallback(EventCallback): 52 | def on_event(self, event): 53 | # Extract information from Event payload, in this sample we 54 | # expect it is UTF-8 encoded 55 | logger.info("Event Subscriber - Event received:\n Topic: %s\n Payload: %s", 56 | event.destination_topic, event.payload.decode()) 57 | 58 | # Add Event callback to DXL client 59 | logger.info("Adding Event callback function to Topic: %s", EVENT_TOPIC) 60 | client.add_event_callback(EVENT_TOPIC, MyEventCallback()) 61 | 62 | A :class:`dxlclient.callbacks.EventCallback` is registered with the client for a specific topic. By default 63 | :func:`dxlclient.client.DxlClient.add_event_callback` will also subscribe 64 | (:func:`dxlclient.client.DxlClient.subscribe`) to the same topic on the fabric. 65 | 66 | Event Publisher 67 | *************** 68 | 69 | The next step is to start the "event publisher". This script must be executed in a separate command prompt (or shell), 70 | leaving the "event subscriber" running. 71 | 72 | To start the "event publisher", execute the ``sample\advanced\event_publisher_sample.py`` script as follows: 73 | 74 | .. parsed-literal:: 75 | 76 | c:\\dxlclient-python-sdk-\ |version|\>python sample\\advanced\\event_publisher_sample.py 77 | 78 | The output should appear similar to the following: 79 | 80 | .. parsed-literal:: 81 | 82 | 2015-12-30 09:00:38,076 __main__ - INFO - Event Publisher - Load DXL config from: C:\\dxlclient-python-sdk-\ |version|\\sample/dxlclient.config 83 | 2015-12-30 09:00:38,078 __main__ - INFO - Event Publisher - Creating DXL Client 84 | 2015-12-30 09:00:38,094 __main__ - INFO - Event Publisher - Connecting to Broker 85 | 2015-12-30 09:00:38,095 dxlclient.client - INFO - Waiting for broker list... 86 | 2015-12-30 09:00:38,095 dxlclient.client - INFO - Checking brokers... 87 | 2015-12-30 09:00:38,096 dxlclient.client - INFO - Trying to connect... 88 | 2015-12-30 09:00:38,096 dxlclient.client - INFO - Trying to connect to broker {Unique id: mybroker, Host name: mybroker.mcafee.com, IP address: 10.84.221.144, Port: 8883}... 89 | 2015-12-30 09:00:38,364 dxlclient.client - INFO - Connected to broker mybroker 90 | 2015-12-30 09:00:38,365 dxlclient.client - INFO - Launching event loop... 91 | 2015-12-30 09:00:38,365 dxlclient.client - INFO - Connected with result code 0 92 | 2015-12-30 09:00:38,365 dxlclient.client - INFO - Subscribing to /mcafee/client/{41eae910-2409-4e4b-9a0f-94b54290a2cf} 93 | Enter 1 to publish a DXL Event 94 | Enter 9 to quit 95 | Enter value: 96 | 97 | To publish a :class:`dxlclient.message.Event` message, enter ``1``. 98 | 99 | Information similar to the following should appear in the "event subscriber" output indicating that the 100 | :class:`dxlclient.message.Event` message was properly received: 101 | 102 | .. code-block:: python 103 | 104 | 2015-12-30 09:03:45,444 __main__ - INFO - Event Subscriber - Event received: 105 | Topic: /isecg/sample/event 106 | Payload: Sample Event Payload 107 | 108 | The publisher will remain running until ``9`` is entered to quit. 109 | 110 | The code for the publisher is very similar to what is being used in the :doc:`basiceventsexample`: 111 | 112 | .. code-block:: python 113 | 114 | # Create the Event 115 | logger.info("Event Publisher - Creating Event for Topic %s", EVENT_TOPIC) 116 | event = Event(EVENT_TOPIC) 117 | 118 | # Encode string payload as UTF-8 119 | event.payload = "Sample Event Payload".encode() 120 | 121 | # Publish the Event to the DXL Fabric on the Topic 122 | logger.info("Event Publisher - Publishing Event to %s", EVENT_TOPIC) 123 | client.send_event(event) 124 | 125 | An :class:`dxlclient.message.Event` event message is created and a payload is assigned. The event 126 | is delivered to the fabric via the :func:`dxlclient.client.DxlClient.send_event` method. 127 | 128 | -------------------------------------------------------------------------------- /docs/sdk/architecture.rst: -------------------------------------------------------------------------------- 1 | Architecture 2 | ============ 3 | 4 | The Data Exchange Layer (DXL) architecture is a publish/subscribe-based middleware that allows DXL clients 5 | to communicate with each over the message bus in near-real time. 6 | 7 | Broker 8 | ------ 9 | 10 | Brokers are responsible for routing messages between the clients that are connected to the message bus. 11 | Brokers can be connected to each other ("bridged") to allow for redundancy, scalability, and communication 12 | across different geographical locations. 13 | 14 | Client 15 | ------ 16 | 17 | Clients connect to brokers for the purposes of exchanging messages. Communication with brokers is over a 18 | TLS-based connection with bi-directional authentication (PKI). -------------------------------------------------------------------------------- /docs/sdk/basiccliprovisioning.rst: -------------------------------------------------------------------------------- 1 | Command Line Provisioning (Basic) 2 | ================================= 3 | 4 | .. _basiccliprovisioning: 5 | 6 | The OpenDXL Python Client's command line interface supports the 7 | ``provisionconfig`` operation which generates the information necessary for 8 | a client to connect to a DXL fabric (certificates, keys, and broker 9 | information). 10 | 11 | As part of the provisioning process, a remote call will be made to a 12 | provisioning server (ePO or OpenDXL Broker) which contains the 13 | Certificate Authority (CA) that will sign the client's certificate. 14 | 15 | `NOTE: ePO-managed environments must have 4.0 (or newer) versions of 16 | DXL ePO extensions installed.` 17 | 18 | Here is an example usage of ``provisionconfig`` operation:: 19 | 20 | dxlclient provisionconfig config myserver client1 21 | 22 | The parameters are as follows: 23 | 24 | * ``config`` is the directory to contain the results of the provisioning 25 | operation. 26 | * ``myserver`` is the host name or IP address of the server (ePO or OpenDXL 27 | Broker) that will be used to provision the client. 28 | * ``client1`` is the value for the Common Name (CN) attribute stored in the 29 | subject of the client's certificate. 30 | 31 | `NOTE:` If a non-standard port (not 8443) is being used for ePO or the 32 | management interface of the OpenDXL Broker, an additional "port" argument 33 | must be specified. For example ``-t 443`` could be specified as part of the 34 | provision operation to connect to the server on port 443. 35 | 36 | When prompted, provide credentials for the OpenDXL Broker Management Console 37 | or ePO (the ePO user must be an administrator):: 38 | 39 | Enter server username: 40 | Enter server password: 41 | 42 | On success, output similar to the following should be displayed:: 43 | 44 | INFO: Saving csr file to config/client.csr 45 | INFO: Saving private key file to config/client.key 46 | INFO: Saving DXL config file to config/dxlclient.config 47 | INFO: Saving ca bundle file to config/ca-bundle.crt 48 | INFO: Saving client certificate file to config/client.crt 49 | 50 | As an alternative to prompting, the username and password values can be 51 | specified via command line options:: 52 | 53 | dxlclient provisionconfig config myserver client1 -u myuser -p mypass 54 | 55 | See the :doc:`advancedcliprovisioning` section for advanced 56 | ``provisionconfig`` operation options. 57 | -------------------------------------------------------------------------------- /docs/sdk/basiceventsexample.rst: -------------------------------------------------------------------------------- 1 | Basic Events Sample 2 | =================== 3 | 4 | This sample demonstrates how to register a callback to receive :class:`dxlclient.message.Event` messages 5 | from the DXL fabric. Once the callback is registered, the sample sends a set number of 6 | :class:`dxlclient.message.Event` messages to the fabric and waits for them all to be received by 7 | the callback. 8 | 9 | Prior to running this sample make sure you have completed the samples configuration step (:doc:`sampleconfig`). 10 | 11 | To run this sample execute the ``sample\basic\event_example.py`` script as follows: 12 | 13 | .. parsed-literal:: 14 | 15 | c:\\dxlclient-python-sdk-\ |version|\>python sample\\basic\\event_example.py 16 | 17 | The output should appear similar to the following: 18 | 19 | .. code-block:: python 20 | 21 | Received event: 0 22 | Received event: 1 23 | Received event: 2 24 | Received event: 3 25 | Received event: 4 26 | Received event: 5 27 | 28 | ... 29 | 30 | Received event: 994 31 | Received event: 995 32 | Received event: 996 33 | Received event: 997 34 | Received event: 998 35 | Received event: 999 36 | Elapsed time (ms): 441.999912262 37 | 38 | The code for the sample is broken into two main sections. 39 | 40 | The first section is responsible for registering an :class:`dxlclient.callbacks.EventCallback` for a specific 41 | topic. The :func:`dxlclient.client.DxlClient.add_event_callback` by default will also 42 | subscribe (:func:`dxlclient.client.DxlClient.subscribe`) to the topic. 43 | 44 | .. code-block:: python 45 | 46 | # 47 | # Register callback and subscribe 48 | # 49 | 50 | # Create and add event listener 51 | class MyEventCallback(EventCallback): 52 | def on_event(self, event): 53 | with event_count_condition: 54 | # Print the payload for the received event 55 | print("Received event: " + event.payload.decode()) 56 | # Increment the count 57 | event_count[0] += 1 58 | # Notify that the count was increment 59 | event_count_condition.notify_all() 60 | 61 | # Register the callback with the client 62 | client.add_event_callback(EVENT_TOPIC, MyEventCallback()) 63 | 64 | The second section sends a set amount of :class:`dxlclient.message.Event` messages via the 65 | :func:`dxlclient.client.DxlClient.send_event` method of the :class:`dxlclient.client.DxlClient`. 66 | 67 | It then waits for all of the events to be received by the :class:`dxlclient.callbacks.EventCallback` that was 68 | previously registered. 69 | 70 | .. code-block:: python 71 | 72 | # 73 | # Send events 74 | # 75 | 76 | # Record the start time 77 | start = time.time() 78 | 79 | # Loop and send the events 80 | for event_id in range(TOTAL_EVENTS): 81 | # Create the event 82 | event = Event(EVENT_TOPIC) 83 | # Set the payload 84 | event.payload = str(event_id).encode() 85 | # Send the event 86 | client.send_event(event) 87 | 88 | # Wait until all events have been received 89 | with event_count_condition: 90 | while event_count[0] < TOTAL_EVENTS: 91 | event_count_condition.wait() 92 | 93 | # Print the elapsed time 94 | print("Elapsed time (ms): " + str((time.time() - start) * 1000)) 95 | -------------------------------------------------------------------------------- /docs/sdk/basicserviceexample.rst: -------------------------------------------------------------------------------- 1 | Basic Service Sample 2 | ===================== 3 | 4 | This sample demonstrates how to register a DXL service to receive :class:`dxlclient.message.Request` 5 | messages and send :class:`dxlclient.message.Response` messages back to an invoking 6 | :class:`dxlclient.client.DxlClient`. 7 | 8 | Prior to running this sample make sure you have completed the samples configuration step (:doc:`sampleconfig`). 9 | 10 | To run this sample execute the ``sample\basic\service_example.py`` script as follows: 11 | 12 | .. parsed-literal:: 13 | 14 | c:\\dxlclient-python-sdk-\ |version|\>python sample\\basic\\service_example.py 15 | 16 | The output should appear similar to the following: 17 | 18 | .. code-block:: python 19 | 20 | Service received request payload: ping 21 | Client received response payload: pong 22 | 23 | The code for the sample is broken into two main sections. 24 | 25 | The first section is responsible for creating a :class:`dxlclient.callbacks.RequestCallback` that will be 26 | invoked for a specific topic associated with the service. The callback will send back a 27 | :class:`dxlclient.message.Response` message with a payload of ``pong`` for any 28 | :class:`dxlclient.message.Request` messages that are received. 29 | 30 | It then creates a :class:`dxlclient.service.ServiceRegistrationInfo` instance and registers the request 31 | callback with it via the :func:`dxlclient.service.ServiceRegistrationInfo.add_topic` method. 32 | 33 | Finally it registers the service with the fabric via the :func:`dxlclient.client.DxlClient.register_service_sync` 34 | method of the :class:`dxlclient.client.DxlClient`. 35 | 36 | .. code-block:: python 37 | 38 | # 39 | # Register the service 40 | # 41 | 42 | # Create incoming request callback 43 | class MyRequestCallback(RequestCallback): 44 | def on_request(self, request): 45 | # Extract information from request 46 | print("Service received request payload: " + request.payload.decode()) 47 | # Create the response message 48 | res = Response(request) 49 | # Populate the response payload 50 | res.payload = "pong".encode() 51 | # Send the response 52 | client.send_response(res) 53 | 54 | # Create service registration object 55 | info = ServiceRegistrationInfo(client, "myService") 56 | 57 | # Add a topic for the service to respond to 58 | info.add_topic(SERVICE_TOPIC, MyRequestCallback()) 59 | 60 | # Register the service with the fabric (wait up to 10 seconds for registration to complete) 61 | client.register_service_sync(info, 10) 62 | 63 | The second section sends a :class:`dxlclient.message.Request` message to the service 64 | that contains a payload of ``ping`` via the :func:`dxlclient.client.DxlClient.sync_request` method of 65 | the :class:`dxlclient.client.DxlClient`. 66 | 67 | The payloads of the :class:`dxlclient.message.Request` and :class:`dxlclient.message.Response` messages 68 | are printed. 69 | 70 | .. code-block:: python 71 | 72 | # 73 | # Invoke the service (send a request) 74 | # 75 | 76 | # Create the request message 77 | req = Request(SERVICE_TOPIC) 78 | 79 | # Populate the request payload 80 | req.payload = "ping".encode() 81 | 82 | # Send the request and wait for a response (synchronous) 83 | res = client.sync_request(req) 84 | 85 | # Extract information from the response (if an error did not occur) 86 | if res.message_type != Message.MESSAGE_TYPE_ERROR: 87 | print("Client received response payload: " + res.payload.decode()) 88 | -------------------------------------------------------------------------------- /docs/sdk/editdxlcerts-save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/editdxlcerts-save.png -------------------------------------------------------------------------------- /docs/sdk/editdxlcerts-selectca.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/editdxlcerts-selectca.png -------------------------------------------------------------------------------- /docs/sdk/editdxlcerts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/editdxlcerts.png -------------------------------------------------------------------------------- /docs/sdk/enablemarauth1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/enablemarauth1.png -------------------------------------------------------------------------------- /docs/sdk/enablemarauth2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/enablemarauth2.png -------------------------------------------------------------------------------- /docs/sdk/enablemarauth3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/enablemarauth3.png -------------------------------------------------------------------------------- /docs/sdk/enablemarauth4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/enablemarauth4.png -------------------------------------------------------------------------------- /docs/sdk/enablemarauth5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/enablemarauth5.png -------------------------------------------------------------------------------- /docs/sdk/enablemarauth6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/enablemarauth6.png -------------------------------------------------------------------------------- /docs/sdk/enablemarauth7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/enablemarauth7.png -------------------------------------------------------------------------------- /docs/sdk/epobrokercertsexport.rst: -------------------------------------------------------------------------------- 1 | ePO Broker Certificates Export 2 | ============================== 3 | 4 | The certificate information for DXL Brokers must be available to DXL clients attempting to connect 5 | to the fabric. This certificate information allows clients to ensure the Brokers being connected to are valid 6 | (via mutual authentication). 7 | 8 | The following steps walk through the process to export the DXL Broker certificate information: 9 | 10 | 1. Navigate to **Server Settings** and select the **DXL Certificates (Third Party)** setting on the left navigation bar. 11 | 12 | .. note:: 13 | 14 | If using an older version of the DXL ePO Extensions, the **Server Settings** in the ePO console may have only a 15 | single option for all DXL certificates (titled "**DXL Certificates**"). Click that setting, and proceed with the 16 | rest of the steps in this guide as normal. 17 | 18 | .. _here: _images/serversettings_old.png 19 | 20 | .. image:: serversettings-certs.png 21 | 22 | 2. Click the **Edit** button in the lower right corner (as shown in the image above) 23 | 24 | .. image:: editdxlcerts-save.png 25 | 26 | 3. Click the **Export All** button in the **Broker Certificates** section (as shown in the image above) 27 | 28 | 4. The exported file, ``brokercerts.crt``, will be saved locally. 29 | 30 | This file is specified as the ``broker_ca_bundle`` parameter when constructing a 31 | :class:`dxlclient.client_config.DxlClientConfig` instance. 32 | 33 | This file can also be specified via a configuration file used to instantiate a 34 | :class:`dxlclient.client_config.DxlClientConfig` instance. 35 | 36 | See the :func:`dxlclient.client_config.DxlClientConfig.create_dxl_config_from_file` method for more information. 37 | 38 | 39 | -------------------------------------------------------------------------------- /docs/sdk/epobrokerlistexport.rst: -------------------------------------------------------------------------------- 1 | ePO Broker List Export 2 | ====================== 3 | 4 | DXL clients need to have a list of Brokers on the fabric to connect to. 5 | 6 | The following steps walk through the process to export the list of all DXL brokers on the fabric for use by DXL clients: 7 | 8 | 1. Navigate to **Server Settings** and select the **DXL Certificates (Third Party)** setting on the left navigation bar. 9 | 10 | .. note:: 11 | 12 | If using an older version of the DXL ePO Extensions, the **Server Settings** in the ePO console may have only a 13 | single option for all DXL certificates (titled "**DXL Certificates**"). Click that setting, and proceed with the 14 | rest of the steps in this guide as normal. 15 | 16 | .. _here: _images/serversettings_old.png 17 | 18 | .. image:: serversettings-certs.png 19 | 20 | 2. Click the **Edit** button in the lower right corner (as shown in the image above) 21 | 22 | .. image:: editdxlcerts-save.png 23 | 24 | 3. Click the **Export All** button in the **Broker List** section (as shown in the image above) 25 | 26 | 4. The exported file, ``brokerlist.properties``, will be saved locally. 27 | 28 | The contents of this file can be copied into the ``[Brokers]`` section of a configuration file used to 29 | instantiate a :class:`dxlclient.client_config.DxlClientConfig` instance. 30 | 31 | See the :func:`dxlclient.client_config.DxlClientConfig.create_dxl_config_from_file` method for more information. 32 | 33 | 34 | -------------------------------------------------------------------------------- /docs/sdk/epocaimport.rst: -------------------------------------------------------------------------------- 1 | ePO Certificate Authority (CA) Import 2 | ===================================== 3 | 4 | Prior to connecting DXL clients to the fabric, a Certificate Authority (CA) that is being used to sign 5 | DXL client certificates must be imported into ePO. 6 | 7 | If you have not created the Certificate Authority (CA), please follow the steps outlined in 8 | the :doc:`certcreation` section. 9 | 10 | The steps to import the CA certificate into ePO are listed below: 11 | 12 | 1. Navigate to **Server Settings** and select the **DXL Certificates (Third Party)** setting on the left navigation bar. 13 | 14 | .. note:: 15 | 16 | If using an older version of the DXL ePO Extensions, the **Server Settings** in the ePO console may have only a 17 | single option for all DXL certificates (titled "**DXL Certificates**"). Click that setting, and proceed with the 18 | rest of the steps in this guide as normal. 19 | 20 | .. _here: _images/serversettings_old.png 21 | 22 | .. image:: serversettings.png 23 | 24 | 2. Click the **Edit** button in the lower right corner (as shown in the image above) 25 | 26 | .. image:: editdxlcerts.png 27 | 28 | 3. Click the **Import** button in the **Client Certificates** section (as shown in the image above) 29 | 30 | .. image:: editdxlcerts-selectca.png 31 | 32 | 4. Select the Certificate (For example, ``ca.crt``) for the Certificate Authority (CA) that was created previously. 33 | 34 | See the :doc:`certcreation` section for information on creating a Certificate Authority (CA) 35 | 36 | 5. Click the **OK** button in the lower right corner (as shown in the image above) 37 | 38 | .. image:: editdxlcerts-save.png 39 | 40 | 6. Click the **Save** button in the lower right corner (as shown in the image above) 41 | 42 | The imported Certificate Authority (CA) information will propagate to the DXL brokers. This process can take 43 | several minutes to complete. 44 | 45 | -------------------------------------------------------------------------------- /docs/sdk/epoexternalcertissuance.rst: -------------------------------------------------------------------------------- 1 | External Certificate Authority (CA) Provisioning 2 | ================================================ 3 | 4 | .. _epoexternalcertissuance: 5 | 6 | This page describes how to provision a client that uses certificates 7 | signed by an externally managed Certificate Authority (CA) with an ePO-based 8 | DXL fabric. 9 | 10 | This is in contrast to certificates that are signed by the internal ePO 11 | Certificate Authority (CA). 12 | 13 | `NOTE: While using an external Certificate Authority (CA) is technically 14 | possible with an OpenDXL Broker, the actual steps are outside the scope 15 | of this documentation.` 16 | 17 | The following steps walk through the creation of an external Certificate Authority (CA), 18 | generation of a client key-pair, and export of broker-related information from ePO: 19 | 20 | * Create a Certificate Authority (CA) and Client Certificate Files (:doc:`certcreation`) 21 | * Import Certificate Authority (CA) into ePO (:doc:`epocaimport`) 22 | * Export the Broker Certificates (:doc:`epobrokercertsexport`) 23 | * Export the list of DXL Brokers (:doc:`epobrokerlistexport`) 24 | 25 | The final step is to populate the contents of a ``dxlclient.config`` file. In this particular case, the 26 | ``dxlclient.config`` file that is located in the ``sample`` sub-directory of the Python DXL SDK 27 | will be populated. 28 | 29 | The following steps walk through the process of populating this file: 30 | 31 | 1. Open the ``dxlclient.config`` file located in the ``sample`` sub-directory of the Python DXL SDK. 32 | 33 | The contents should appear as follows: 34 | 35 | .. code-block:: python 36 | 37 | [Certs] 38 | BrokerCertChain= 39 | CertFile= 40 | PrivateKey= 41 | 42 | [Brokers] 43 | unique_broker_id_1=broker_id_1;broker_port_1;broker_hostname_1;broker_ip_1 44 | unique_broker_id_2=broker_id_2;broker_port_2;broker_hostname_2;broker_ip_2 45 | 46 | [BrokersWebSockets] 47 | unique_broker_id_1=broker_id_1;broker_websocket_port_1;broker_hostname_1;broker_ip_1 48 | unique_broker_id_2=broker_id_2;broker_websocket_port_2;broker_hostname_2;broker_ip_2 49 | 50 | 2. Update the ``CertFile`` and ``PrivateKey`` values to point to the certificate file (``client.crt``) and 51 | private key file (``client.key``) that were created during the certificate provisioning steps. 52 | 53 | See the :doc:`certcreation` section for more information on the creation of client key-pairs. 54 | 55 | After completing this step the contents of the configuration file should look similar to: 56 | 57 | .. code-block:: python 58 | 59 | [Certs] 60 | BrokerCertChain= 61 | CertFile=c:\\certificates\\client.crt 62 | PrivateKey=c:\\certificates\\client.key 63 | 64 | [Brokers] 65 | unique_broker_id_1=broker_id_1;broker_port_1;broker_hostname_1;broker_ip_1 66 | unique_broker_id_2=broker_id_2;broker_port_2;broker_hostname_2;broker_ip_2 67 | 68 | [BrokersWebSockets] 69 | unique_broker_id_1=broker_id_1;broker_websocket_port_1;broker_hostname_1;broker_ip_1 70 | unique_broker_id_2=broker_id_2;broker_websocket_port_2;broker_hostname_2;broker_ip_2 71 | 72 | 3. Update the ``BrokerCertChain`` value to point to the Broker Certificates file (``brokercerts.crt``) 73 | that was created when exporting the Broker Certificates. 74 | 75 | See the :doc:`epobrokercertsexport` section for more information on exporting Broker Certificates. 76 | 77 | After completing this step the contents of the configuration file should look similar to: 78 | 79 | .. code-block:: python 80 | 81 | [Certs] 82 | BrokerCertChain=c:\\certificates\\brokercerts.crt 83 | CertFile=c:\\certificates\\client.crt 84 | PrivateKey=c:\\certificates\\client.key 85 | 86 | [Brokers] 87 | unique_broker_id_1=broker_id_1;broker_port_1;broker_hostname_1;broker_ip_1 88 | unique_broker_id_2=broker_id_2;broker_port_2;broker_hostname_2;broker_ip_2 89 | 90 | [BrokersWebSockets] 91 | unique_broker_id_1=broker_id_1;broker_websocket_port_1;broker_hostname_1;broker_ip_1 92 | unique_broker_id_2=broker_id_2;broker_websocket_port_2;broker_hostname_2;broker_ip_2 93 | 94 | 3. Update the ``[Brokers]`` and ``[BrokersWebSockets]`` sections to include the contents of the broker 95 | list file (``brokerlist.properties``) that was created when exporting the Broker List. 96 | 97 | See the :doc:`epobrokerlistexport` section for more information on exporting the Broker List. 98 | 99 | After completing this step the contents of the configuration file should look similar to: 100 | 101 | .. code-block:: python 102 | 103 | [Certs] 104 | BrokerCertChain=c:\\certificates\\brokercerts.crt 105 | CertFile=c:\\certificates\\client.crt 106 | PrivateKey=c:\\certificates\\client.key 107 | 108 | [Brokers] 109 | {5d73b77f-8c4b-4ae0-b437-febd12facfd4}={5d73b77f-8c4b-4ae0-b437-febd12facfd4};8883;mybroker.mcafee.com;192.168.1.12 110 | {24397e4d-645f-4f2f-974f-f98c55bdddf7}={24397e4d-645f-4f2f-974f-f98c55bdddf7};8883;mybroker2.mcafee.com;192.168.1.13 111 | 112 | [BrokersWebSockets] 113 | {5d73b77f-8c4b-4ae0-b437-febd12facfd4}={5d73b77f-8c4b-4ae0-b437-febd12facfd4};443;mybroker.mcafee.com;192.168.1.12 114 | {24397e4d-645f-4f2f-974f-f98c55bdddf7}={24397e4d-645f-4f2f-974f-f98c55bdddf7};443;mybroker2.mcafee.com;192.168.1.13 115 | 116 | 4. At this point you can run the samples included with the Python SDK. 117 | -------------------------------------------------------------------------------- /docs/sdk/features.rst: -------------------------------------------------------------------------------- 1 | Features 2 | ======== 3 | 4 | Location Unaware 5 | ---------------- 6 | 7 | Communication between clients on the DXL fabric is based on sending "messages" to a "topic". Clients are unaware 8 | of the location of the other DXL clients they are communicating with (its hostname, IP address, etc.). 9 | 10 | For example, if a client wanted to determine the reputation for a file, it could send a request message to the topic 11 | ``/mcafee/service/tie/file/reputation``. A service would receive the request and send a response with the appropriate 12 | reputation information. All of this communication occurs without either party knowing the location of the other 13 | (they could be in the same building or on the other side of the world). 14 | 15 | Persistent Connection 16 | --------------------- 17 | 18 | Connections are established from a DXL client to a DXL broker. These connections are persistent and allow for 19 | bi-directional communication. The benefits to this style of connection include: 20 | 21 | * Firewall Friendly 22 | 23 | Clients are responsible for establishing the connection to brokers (never from a 24 | broker to a client). Therefore, we can communicate with clients that were previously unreachable. 25 | For example, a mobile client can connect to a broker exposed in a demilitarized zone (DMZ). Since 26 | the communication is bi-directional, we can now communicate with the client from products 27 | also connected to the fabric (sending an agent wakeup for cloud ePO, etc.). 28 | 29 | * Near Real-Time Communication 30 | 31 | Communication on the DXL fabric is extremely efficient because the 32 | expense of continually establishing connections is eliminated. 33 | 34 | Multiple communication models 35 | ----------------------------- 36 | 37 | DXL Supports two different models of communication. A service-based model with point-to-point (request/response) 38 | communication and a publish/subscribe event-based model. 39 | 40 | * Service-based 41 | 42 | The DXL fabric allows for "services" to be registered and exposed that respond to requests 43 | sent by invoking clients. This communication is point-to-point (one-to-one), meaning the communication is 44 | solely between an invoking client and the service that is being invoked. It is important to note that in 45 | this model the client actively invokes the service by sending it requests. 46 | 47 | For example, the Threat Intelligence Exchange service is exposed via DXL allowing for DXL clients to request 48 | reputations for files and certificates. 49 | 50 | * Event-based 51 | 52 | The DXL fabric also allows for event-based communication. This model is typically referred to as 53 | "publish/subscribe" wherein clients register interest by subscribing to a particular topic and publishers 54 | periodically send events to that topic. The event is delivered by the DXL fabric to all of the currently 55 | subscribed clients for the topic. Therefore, a single event sent can reach multiple clients (one-to-many). 56 | It is important to note that in this model the client passively receives events when they are sent by a publisher. 57 | 58 | For example, Advanced Threat Defense (ATD) servers send events to the topic ``/mcafee/event/atd/file/report`` when 59 | they have successfully determined the reputation for a file. Any clients currently subscribed to this topic will 60 | receive the report (Threat Intelligence Exchange Server and the Enterprise Security Manager currently subscribe 61 | to this topic). 62 | 63 | Secure Communication 64 | -------------------- 65 | 66 | Communication over the DXL fabric is secured via TLS version 1.2 and PKI mutual authentication. 67 | 68 | The fabric also supports topic-level authorization wherein individual topics can be restricted in terms of 69 | which clients can publish messages to and which clients can receive messages on a particular topic. 70 | 71 | For example, the DXL clients embedded in the Threat Intelligence Exchange servers are the only ones 72 | authorized to publish reputation change events to the topic ``/mcafee/event/tie/file/repchange``. -------------------------------------------------------------------------------- /docs/sdk/index.rst: -------------------------------------------------------------------------------- 1 | 2 | Data Exchange Layer (DXL) Python SDK Documentation 3 | ================================================== 4 | 5 | Introduction 6 | ------------ 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | 11 | overview 12 | architecture 13 | features 14 | integrationtypes 15 | 16 | Installation 17 | ------------ 18 | 19 | .. toctree:: 20 | :maxdepth: 1 21 | 22 | installation 23 | 24 | Provisioning 25 | ------------ 26 | 27 | .. toctree:: 28 | :maxdepth: 1 29 | 30 | provisioningoverview 31 | 32 | Command Line Interface (CLI) - `(Requires 4.0 or newer client)` 33 | 34 | .. toctree:: 35 | :maxdepth: 1 36 | 37 | basiccliprovisioning 38 | advancedcliprovisioning 39 | updatingconfigfromcli 40 | 41 | OpenDXL Broker Management Console 42 | 43 | .. toctree:: 44 | :maxdepth: 1 45 | 46 | openconsoleprovisioning 47 | 48 | External Certificate Authority (CA) 49 | 50 | .. toctree:: 51 | :maxdepth: 1 52 | 53 | epoexternalcertissuance 54 | 55 | 56 | Python API 57 | ---------- 58 | 59 | .. toctree:: 60 | :titlesonly: 61 | 62 | dxlclient 63 | 64 | Samples 65 | ------- 66 | 67 | Configuration 68 | 69 | .. toctree:: 70 | :maxdepth: 1 71 | 72 | sampleconfig 73 | 74 | Basic 75 | 76 | .. toctree:: 77 | :maxdepth: 1 78 | 79 | basiceventsexample 80 | basicserviceexample 81 | 82 | Advanced 83 | 84 | .. toctree:: 85 | :maxdepth: 1 86 | 87 | advancedeventsexample 88 | advancedserviceexample 89 | 90 | Service Wrapper 91 | 92 | .. toctree:: 93 | :maxdepth: 1 94 | 95 | servicewrapperexample 96 | 97 | Threat Intelligence Exchange (TIE) 98 | 99 | .. toctree:: 100 | :maxdepth: 1 101 | 102 | tiefilerepexample 103 | 104 | McAfee Active Response (MAR) 105 | 106 | .. toctree:: 107 | :maxdepth: 1 108 | 109 | marsearchexample 110 | marsendauth 111 | 112 | Authorization 113 | ------------- 114 | 115 | .. toctree:: 116 | :maxdepth: 1 117 | 118 | topicauthoverview 119 | topicauthgroupcreation 120 | topicauthgrouprestrictions 121 | 122 | Certificates 123 | ------------ 124 | 125 | .. toctree:: 126 | :maxdepth: 1 127 | 128 | certcreation 129 | epobrokercertsexport 130 | epobrokerlistexport 131 | epocaimport -------------------------------------------------------------------------------- /docs/sdk/installation.rst: -------------------------------------------------------------------------------- 1 | Python SDK Installation 2 | ======================= 3 | 4 | Prerequisites 5 | ************* 6 | 7 | * DXL Brokers (3.0.1 or later) deployed within an ePO managed environment 8 | * DXL Extensions (3.0.1 or later) 9 | * Python 3.7 or higher installed within a Windows or Linux environment. 10 | * PIP (Included with Python 2.7.9 and later) - PIP is the preferred way to install the Python DXL Client SDK, 11 | but is not required (``setup.py install`` can be used as an alternative). 12 | * An OpenSSL version used by Python that supports TLSv1.2 (Version 1.0.1 or greater) 13 | 14 | * To check the version of OpenSSL used by Python, open a Python shell:: 15 | 16 | python 17 | 18 | * Type the following statements:: 19 | 20 | >>> import ssl 21 | >>> ssl.OPENSSL_VERSION 22 | 23 | * The output should appear similar to the following:: 24 | 25 | 'OpenSSL 1.0.2a 19 Mar 2015' 26 | 27 | * The version must be 1.0.1 or greater. Unfortunately, even the latest versions of OSX (Mac) still have version 28 | 0.9.8 installed. If you wish to use the Python SDK with OSX, one possible workaround is to use a third 29 | party package manager (such as `Brew `_) to install a compatible Python and OpenSSL version. 30 | 31 | Python SDK Installation 32 | *********************** 33 | 34 | Use ``pip`` to automatically install the module: 35 | 36 | .. parsed-literal:: 37 | 38 | pip install dxlclient-\ |version|\-py2.py3-none-any.whl 39 | 40 | Or with: 41 | 42 | .. parsed-literal:: 43 | 44 | pip install dxlclient-\ |version|\.zip 45 | 46 | As an alternative (without PIP), unpack the dxlclient-\ |version|\.zip (located in the lib folder) and run the setup 47 | script: 48 | 49 | .. code-block:: python 50 | 51 | python setup.py install 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /docs/sdk/integrationtypes.rst: -------------------------------------------------------------------------------- 1 | Integration Types 2 | ================= 3 | Interaction with the DXL fabric is comprised of two primary roles; Information consumers and information providers. 4 | 5 | Consumers 6 | ---------- 7 | 8 | Consumers receive information over the DXL fabric via its multiple communication models. 9 | 10 | **Event subscriber** 11 | 12 | An event-based consumer subscribes to topics and passively receives events from publishers (one-to-many communication mode). 13 | 14 | Common use cases that leverage event subscriber integrations include: 15 | 16 | * Orchestration 17 | 18 | A client listens for a particular set of events. Once an event is received an orchestration work flow is initiated. 19 | 20 | For example, a security product detects that a particular endpoint is sending data to a known "command and control" target and as a reaction sends an event to the DXL fabric. A client listening for the particular event initiates an orchestration work flow that triggers a remediation process utilizing other products available on the fabric. 21 | 22 | **Service invoker** 23 | 24 | A client that invokes methods on services registered on the DXL fabric. The client actively requests information from the service (one-to-one communication mode). 25 | 26 | Common use cases that leverage service invocation integrations include: 27 | 28 | * Data Collection 29 | 30 | Security services are invoked over the DXL fabric to query information in real-time (running-processes, etc.) or for forensics-based analysis. 31 | 32 | * Orchestration 33 | 34 | As part of an orchestration work flow various security services are invoked for the purpose of data collection, analytics, and remediation. 35 | 36 | Providers 37 | --------- 38 | 39 | Providers distribute information to the DXL fabric via its multiple communication models. 40 | 41 | **Event publisher** 42 | 43 | A client that periodically publishes events to specific topics on the DXL fabric (one-to-many communication mode). These events will be delivered to consumers that are currently subscribed to those topics. 44 | 45 | Common use cases that leverage event publisher integrations include: 46 | 47 | * Threat Events 48 | 49 | Events are sent to the fabric to indicate the presence of a threat. For example, malware is detected on a particular endpoint. 50 | 51 | * Informational Events 52 | 53 | Events are sent to the fabric to announce a particular piece of information. For example, a new vulnerability has been published, a user logged into a system, etc. 54 | 55 | **Service providers** 56 | 57 | A client that registers a service with the DXL fabric. A service is comprised of one or more methods that are exposed via corresponding topics. These service methods will be invoked by clients by sending a request message to a topic associated with a service-method. Once received, the service replies by sending a response message over the fabric back to the invoking client (one-to-one communication mode). 58 | 59 | Common service integration models include: 60 | 61 | * Native service 62 | 63 | A service is developed with a native DXL fabric integration. For example McAfee Threat Intelligence Exchange (TIE) natively supports communication with DXL fabrics. 64 | 65 | * Wrapped service 66 | 67 | A DXL service wrapper is created that delegates invocations to an existing service's API/SDK. For example, a security service that exposes a REST-based API can be easily wrapped by a DXL service wrapper to provides its functionality on the DXL fabric. -------------------------------------------------------------------------------- /docs/sdk/marsendauth.rst: -------------------------------------------------------------------------------- 1 | Authorize Client to Perform MAR Search 2 | ====================================== 3 | 4 | By default, McAfee Active Response (MAR) does not allow systems other than the MAR Server to utilize its API over DXL. 5 | In order to allow the McAfee Active Response (MAR) Search Sample to work the "Active Response Server API" authorization 6 | group's send restrictions must be modified to include the Certificate Authority (CA) and/or certificate used by 7 | the client executing the McAfee Active Response (MAR) Search Sample. 8 | 9 | Please see :doc:`topicauthoverview` for more information on DXL Topic Authorization. 10 | 11 | The following steps will walk through the process of allowing a DXL client to send messages on the 12 | DXL Topic ``/mcafee/mar/service/api/search`` which is associated with the 13 | DXL Topic Authorization Group ``Active Response Server API``: 14 | 15 | 1. Navigate to **Server Settings** and select the **DXL Topic Authorization** setting on the left navigation bar. 16 | 17 | .. image:: enablemarauth1.png 18 | 19 | 2. Click the **Edit** button in the lower right corner (as shown in the image above) 20 | 21 | .. image:: enablemarauth2.png 22 | 23 | 3. Select the check box next to the DXL Topic Authorization Group ``Active Response Server API`` (as shown in the image above) 24 | 25 | .. image:: enablemarauth3.png 26 | 27 | 4. Click the **Actions** button and select **Restrict Send Certificates** to select certificates allowed to send messages to the topics associated with the ``Active Response Server API`` authorization group (as shown in the image above) 28 | 29 | .. image:: enablemarauth4.png 30 | 31 | 5. Select the check box next to any certificate to indicate that only DXL Clients with the selected certs or child certs (or tags separately specified) will be allowed to send DXL messages on topics associated with the ``Active Response Server API`` authorization group 32 | 33 | 34 | .. image:: enablemarauth5.png 35 | 36 | 6. Click the **OK** button in the lower right corner (as shown in the image above) 37 | 38 | 39 | .. image:: enablemarauth6.png 40 | 41 | 7. Click the **Save** button in the lower right corner (as shown in the image above) 42 | 43 | .. image:: enablemarauth7.png 44 | 45 | The DXL Topic Authorization information will propagate to the DXL brokers. This process can take several minutes 46 | to complete. 47 | 48 | -------------------------------------------------------------------------------- /docs/sdk/openconsoleprovisioning.rst: -------------------------------------------------------------------------------- 1 | OpenDXL Broker Management Console Provisioning 2 | ============================================== 3 | 4 | .. _openconsoleprovisioning: 5 | 6 | The OpenDXL Broker Management Console includes a "Generate Client Configuration" page that 7 | is used to generate a client configuration bundle that contains the files necessary for 8 | an OpenDXL Client to connect to the OpenDXL Broker. 9 | 10 | See the `OpenDXL Broker Generate Client Configuration `_ 11 | page for more information. 12 | -------------------------------------------------------------------------------- /docs/sdk/overview.rst: -------------------------------------------------------------------------------- 1 | Data Exchange Layer (DXL) Overview 2 | ================================== 3 | 4 | The "Security Connected" Vision 5 | ------------------------------- 6 | 7 | An adaptive (security) system of interconnected services that communicate and share information to make real-time, 8 | accurate security decisions by individual security products and/or as a collective. Network, endpoint, mobile 9 | and other security solutions are no longer to operate as separate "silos", but as one synchronized, real-time, 10 | adaptive, security system. 11 | 12 | What is the Data Exchange Layer? 13 | -------------------------------- 14 | 15 | The Data Exchange Layer (DXL) is the foundation for making the Security Connected vision a reality. 16 | It is a messaging framework for enabling real-time security context sharing between products, 17 | to enable adaptive security, and for treating appropriately designed products as services. 18 | In short, it allows for scalable communication between anything on the DXL network "fabric" through an 19 | easy-to-use, easy-to-manage product. 20 | 21 | -------------------------------------------------------------------------------- /docs/sdk/provisioningoverview.rst: -------------------------------------------------------------------------------- 1 | Provisioning Overview 2 | ===================== 3 | 4 | In order for a client to connect to the DXL fabric it must be provisioned. 5 | 6 | A provisioned client includes certificate information required to establish 7 | an authenticated connection to the fabric as well as information regarding 8 | the brokers to connect to. 9 | 10 | Provisioning Options 11 | ******************** 12 | 13 | The following options are available for a provisioning a client: 14 | 15 | * :ref:`Command Line Interface (CLI) ` 16 | 17 | The provisioning process is performed via the OpenDXL Python Client's 18 | command line interface (CLI). A remote call will be made to a 19 | provisioning server (ePO or OpenDXL Broker) which contains the 20 | Certificate Authority (CA) that will sign the client's certificate. 21 | 22 | `NOTE: ePO-managed environments must have 4.0 (or newer) versions of 23 | DXL ePO extensions installed.` 24 | 25 | * :ref:`OpenDXL Broker Management Console ` 26 | 27 | The OpenDXL Broker Management Console includes a page that will generate 28 | and download client configuration packages. The OpenDXL Broker's 29 | Certificate Authority (CA) will be used to sign the certificates. 30 | 31 | `NOTE: This option is not compatible with ePO-managed environments.` 32 | 33 | * :ref:`External Certificate Authority (CA) ` 34 | 35 | This option allows for the signing of client certificates using an 36 | external Certificate Authority (CA). 37 | 38 | Updating Client Configuration 39 | ***************************** 40 | 41 | After the initial provisioning of a client, its configuration may 42 | need to be periodically updated. For example, if new brokers are added or 43 | removed from the fabric. 44 | 45 | The :doc:`updatingconfigfromcli` section describes how a client's 46 | configuration can be updated via the use of the OpenDXL Python Client's 47 | command line. 48 | -------------------------------------------------------------------------------- /docs/sdk/sampleconfig.rst: -------------------------------------------------------------------------------- 1 | Samples Configuration 2 | ===================== 3 | 4 | Prior to running any of the examples, you must complete the following: 5 | 6 | * Install the Python SDK (:doc:`installation`) 7 | * Provision the client to use with the samples (:doc:`provisioningoverview`) 8 | 9 | The ``dxlclient.config`` file located in the ``sample`` sub-directory of the Python DXL SDK 10 | must be populated. 11 | 12 | `NOTE:` If :ref:`Command Line Interface (CLI) ` provisioning is being performed, 13 | specify the ``sample`` directory as the output location for the provisioning operation. 14 | 15 | -------------------------------------------------------------------------------- /docs/sdk/serversettings-certs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/serversettings-certs.png -------------------------------------------------------------------------------- /docs/sdk/serversettings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/docs/sdk/serversettings.png -------------------------------------------------------------------------------- /docs/sdk/tiefilerepexample.rst: -------------------------------------------------------------------------------- 1 | Threat Intelligence Exchange (TIE) File Reputation Sample 2 | ========================================================= 3 | 4 | This sample demonstrates invoking the McAfee Threat Intelligence Exchange 5 | (TIE) DXL service to retrieve the reputation of files (as identified 6 | by their hashes). 7 | 8 | Prerequisites 9 | ************* 10 | * The samples configuration step has been completed (:doc:`sampleconfig`) 11 | * A TIE Service is available on the DXL fabric 12 | 13 | To run this sample execute the ``sample\tie\file_rep_sample.py`` script as follows: 14 | 15 | .. parsed-literal:: 16 | 17 | c:\\dxlclient-python-sdk-\ |version|\>python sample\\tie\\file_rep_sample.py 18 | 19 | The output should appear similar to the following: 20 | 21 | .. code-block:: python 22 | 23 | Notepad.exe reputation: 24 | { 25 | "props": { 26 | "serverTime": 1451505556, 27 | "submitMetaData": 1 28 | }, 29 | "reputations": [ 30 | { 31 | "attributes": { 32 | "2120340": "2139160704" 33 | }, 34 | "createDate": 1451502875, 35 | "providerId": 1, 36 | "trustLevel": 99 37 | }, 38 | { 39 | "attributes": { 40 | "2101652": "17", 41 | "2102165": "1451502875", 42 | "2111893": "21", 43 | "2114965": "0", 44 | "2139285": "72339069014638857" 45 | }, 46 | "createDate": 1451502875, 47 | "providerId": 3, 48 | "trustLevel": 0 49 | } 50 | ] 51 | } 52 | 53 | EICAR reputation: 54 | { 55 | "props": { 56 | "serverTime": 1451505556, 57 | "submitMetaData": 1 58 | }, 59 | "reputations": [ 60 | { 61 | "attributes": { 62 | "2120340": "2139162632" 63 | }, 64 | "createDate": 1451504331, 65 | "providerId": 1, 66 | "trustLevel": 1 67 | }, 68 | { 69 | "attributes": { 70 | "2101652": "11", 71 | "2102165": "1451504331", 72 | "2111893": "22", 73 | "2114965": "0", 74 | "2139285": "72339069014638857" 75 | }, 76 | "createDate": 1451504331, 77 | "providerId": 3, 78 | "trustLevel": 0 79 | } 80 | ] 81 | } 82 | 83 | The sample outputs the file reputation for two files. 84 | 85 | The first file queried in the TIE service is "notepad.exe". The McAfee Global Threat Intelligence (GTI) service 86 | is identified in the results as ``"providerId" : 1``. The trust level associated with the GTI response 87 | (``"trustLevel": 99``) indicates that the file is known good. 88 | 89 | The second file queried in the TIE service is the "EICAR Standard Anti-Virus Test File". The trust level associated 90 | with the GTI response (``"trustLevel": 1``) indicates that the file is known bad. 91 | 92 | The major functionality provided by the sample resides in the ``get_tie_file_reputation()`` method as shown 93 | below: 94 | 95 | .. code-block:: python 96 | 97 | def get_tie_file_reputation(client, md5_hex, sha1_hex): 98 | """ 99 | Returns a dictionary containing the results of a TIE file reputation request 100 | 101 | :param client: The DXL client 102 | :param md5_hex: The MD5 Hex string for the file 103 | :param sha1_hex: The SHA-1 Hex string for the file 104 | :return: A dictionary containing the results of a TIE file reputation request 105 | """ 106 | # Create the request message 107 | req = Request(FILE_REP_TOPIC) 108 | 109 | # Create a dictionary for the payload 110 | payload_dict = { 111 | "agentGuid" : "myagent", 112 | "hashes" : [ 113 | { "type" : "md5", "value" : base64_from_hex(md5_hex) }, 114 | { "type" : "sha1", "value" : base64_from_hex(sha1_hex) } 115 | ] 116 | } 117 | 118 | # Set the payload 119 | req.payload = json.dumps(payload_dict).encode() 120 | 121 | # Send the request and wait for a response (synchronous) 122 | res = client.sync_request(req) 123 | 124 | # Return a dictionary corresponding to the response payload 125 | if res.message_type != Message.MESSAGE_TYPE_ERROR: 126 | return json.loads(res.payload.decode(encoding="UTF-8")) 127 | else: 128 | raise Exception("Error: " + res.error_message + " (" + str(res.error_code) + ")") 129 | 130 | This method creates a :class:`dxlclient.message.Request` message that will be delivered to the 131 | file reputation request topic (``/mcafee/service/tie/file/reputation``) of a TIE service on the fabric. 132 | 133 | The required payload for a "TIE File Reputation" request is set on the message. 134 | 135 | The request message is delivered to the fabric via the :func:`dxlclient.client.DxlClient.sync_request` method on 136 | the DXL client. 137 | 138 | The payload of the :class:`dxlclient.message.Response` message received is converted to a Python ``dictionary`` 139 | object and returned to the caller of the method. 140 | -------------------------------------------------------------------------------- /docs/sdk/topicauthgroupcreation.rst: -------------------------------------------------------------------------------- 1 | Authorization Group Creation 2 | ================================== 3 | 4 | To simplify the management of topic restrictions, related topics are assembled into "topic groups" . For example, the 5 | topics exposed by the McAfee Threat Intelligence (TIE) service that can be invoked by any client (unrestricted) are 6 | grouped into the "TIE Server Service Operations" group (see image below). 7 | 8 | The following section walks through the steps to create a new DXL topic authorization group. In this particular 9 | example we are creating a group for the topics that will be exposed by a new DXL service. 10 | 11 | #. Navigate to **Server Settings** and select the **DXL Topic Authorization** setting on the left navigation bar. 12 | 13 | .. image:: addtopicgroup1.png 14 | 15 | #. Click the **Edit** button in the lower right corner (as shown in the image above) 16 | 17 | .. image:: addtopicgroup2.png 18 | 19 | #. Click the **Actions** button and select **Add Topic Group** (as shown in the image above) 20 | 21 | .. image:: addtopicgroup3.png 22 | 23 | #. Enter in a **Group Name** and a topic to associate with that **Group Name** 24 | 25 | .. image:: addtopicgroup4.png 26 | 27 | #. If more than one topic needs to be associated with the **Group Name** click the **+** button to add more rows for **Topics** 28 | 29 | .. image:: addtopicgroup5.png 30 | 31 | #. Click the **OK** button in the lower right corner (as shown in the image above) 32 | 33 | .. image:: addtopicgroup6.png 34 | 35 | #. Click the **Save** button in the lower right corner (as shown in the image above) -------------------------------------------------------------------------------- /docs/sdk/topicauthgrouprestrictions.rst: -------------------------------------------------------------------------------- 1 | Managing Authorization Group Restrictions 2 | =============================================== 3 | 4 | Prior to managing certificate-based topic authorization group restrictions, a Certificate Authority (CA) that is being 5 | used to sign client certificates or the certificate for a client must be imported into ePO. 6 | 7 | If you have not imported a Certificate Authority (CA) or certificate, please follow the steps outlined in 8 | the :doc:`epocaimport` section. Also if you have not created a topic authorization group, please follow the steps in 9 | the :doc:`topicauthgroupcreation` section. 10 | 11 | The following section walks through the steps of limiting which Certificate Authorities (CAs) and/or certificates are 12 | required to send and receive messages for a topic authorization group: 13 | 14 | 1. Navigate to **Server Settings** and select the **Topic Authorization** setting on the left navigation bar. 15 | 16 | .. image:: addcertbasedauth1.png 17 | 18 | #. Click the **Edit** button in the lower right corner (as shown in the image above) 19 | 20 | .. image:: addcertbasedauth2.png 21 | 22 | #. Select the check box next to a Topic Authorization Group (as shown in the image above) 23 | 24 | .. image:: addcertbasedauth3.png 25 | 26 | #. Click the **Actions** button and select **Restrict Receive Certificates** to select certificates allowed to receive 27 | messages on the topics associated with the selected Topic Authorization Group (as shown in the image above) 28 | 29 | .. image:: addcertbasedauth4.png 30 | 31 | #. Select the check box next to any certificate to indicate that only clients with the selected certs or child certs 32 | will be allowed to receive messages on the topics associated with the selected Topic Authorization Group 33 | 34 | .. image:: addcertbasedauth5.png 35 | 36 | #. Click the **OK** button in the lower right corner (as shown in the image above) 37 | 38 | .. image:: addcertbasedauth6.png 39 | 40 | #. Select the check box next to a Topic Authorization Group (as shown in the image above) 41 | 42 | #. Click the **Actions** button and select **Restrict Send Certificates** to select certificates allowed to send 43 | messages on the topics associated with the selected Topic Authorization Group 44 | 45 | .. image:: addcertbasedauth7.png 46 | 47 | #. Select the check box next to any certificate to indicate that only clients with the selected certs or child certs 48 | will be allowed to receive messages on the topics associated with the selected Topic Authorization Group 49 | 50 | .. image:: addcertbasedauth8.png 51 | 52 | #. Click the **OK** button in the lower right corner (as shown in the image above) 53 | 54 | .. image:: addcertbasedauth9.png 55 | 56 | #. Click the **Save** button in the lower right corner (as shown in the image above) 57 | 58 | .. image:: addcertbasedauth10.png 59 | 60 | The Topic Authorization information will propagate to the brokers. This process can take several minutes 61 | to complete. -------------------------------------------------------------------------------- /docs/sdk/topicauthoverview.rst: -------------------------------------------------------------------------------- 1 | Authorization Overview 2 | ================================ 3 | 4 | DXL topic authorization is used to restrict which clients can "send" and "receive" DXL messages on particular topics. 5 | 6 | Examples of using topic authorization include: 7 | 8 | * Restricting which clients can provide DXL services. 9 | 10 | When providing a service (McAfee Threat Intelligence Exchange (TIE), etc.) a restriction should be added to ensure that only clients that are providing the service are able to "receive" messages on the service-related topics. Without this protection other clients could masquerade as the service. 11 | 12 | * Restricting which clients can invoke DXL services. 13 | 14 | This is accomplished by limiting the clients that can "send" messages on the service-related topics. For example, the clients that can perform McAfee Active Response (MAR) queries are limited using topic authorization (see section :doc:`marsendauth`) 15 | 16 | * Restricting which clients can "send" event messages. 17 | 18 | For example, only authorized clients should be able to inform that fabric that a McAfee Threat Intelligence (TIE) reputation has changed by sending a DXL event. 19 | 20 | Python-based DXL clients are identified by their certificates. Client-specific certificates and/or Certificate 21 | Authorities (CAs) can be used to limit which clients can send and receive messages on particular topics. A client 22 | certificate can be used to establish a restriction for a single client whereas a certificate authority can be used 23 | to establish a restriction for all clients that were signed by that particular authority. 24 | 25 | Please see the :doc:`topicauthgroupcreation` and :doc:`topicauthgrouprestrictions` sections for information on how to 26 | utilize DXL topic authorization. -------------------------------------------------------------------------------- /docs/sdk/updatingconfigfromcli.rst: -------------------------------------------------------------------------------- 1 | Client Configuration Update via Command Line 2 | ============================================ 3 | 4 | The ``updateconfig`` command line operation can be used to update a previously 5 | provisioned client with the latest information from a management server 6 | (ePO or OpenDXL Broker). 7 | 8 | `NOTE: ePO-managed environments must have 4.0 (or newer) versions of 9 | DXL ePO extensions installed.` 10 | 11 | The ``updateconfig`` operation performs the following: 12 | 13 | * Retrieves the latest CA certificate bundle from the server and stores it 14 | at the file referenced by the ``BrokerCertChain`` setting in the ``[Certs]`` 15 | section of the ``dxlclient.config`` file. 16 | 17 | * Retrieves the latest broker information and updates the ``[Brokers]`` and 18 | ``[BrokersWebSockets]`` sections of the ``dxlclient.config`` file with 19 | that information. 20 | 21 | Basic Example 22 | ************* 23 | 24 | For example:: 25 | 26 | dxlclient updateconfig config myserver 27 | 28 | For this example, ``config`` is the name of the directory in which the 29 | ``dxlclient.config`` file resides and ``myserver`` is the hostname or 30 | IP address of ePO or an OpenDXL Broker. 31 | 32 | When prompted, provide credentials for the OpenDXL Broker Management Console 33 | or ePO (the ePO user must be an administrator):: 34 | 35 | Enter server username: 36 | Enter server password: 37 | 38 | If the operation is successful, output similar to the following 39 | should be displayed:: 40 | 41 | INFO: Updating certs in config/ca-bundle.crt 42 | INFO: Updating DXL config file at config/dxlclient.config 43 | 44 | To avoid the username and password prompts, supply the appropriate 45 | command line options (``-u`` and ``-p``):: 46 | 47 | dxlclient updateconfig config myserver -u myuser -p mypass 48 | 49 | Additional Options 50 | ****************** 51 | 52 | The update operation assumes that the default web server port is 8443, 53 | the default port under which the ePO web interface and OpenDXL Broker Management 54 | Console is hosted. 55 | 56 | A custom port can be specified via the ``-t`` option. 57 | 58 | For example:: 59 | 60 | dxlclient updateconfig config myserver -t 443 61 | 62 | If the management server's CA certificate is stored in a local CA truststore 63 | file -- one or more PEM-formatted certificates concatenated together into a 64 | single file -- the update operation can be configured to validate 65 | the management server's certificate against that truststore during TLS session 66 | negotiation by supplying the ``-e`` option. 67 | 68 | The name of the truststore file should be supplied along with the option:: 69 | 70 | dxlclient updateconfig config myserver -e config/ca-bundle.crt 71 | -------------------------------------------------------------------------------- /dxlclient/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ dxlclient APIs """ 7 | 8 | from __future__ import absolute_import 9 | import logging 10 | from threading import RLock 11 | 12 | # pylint: disable=wildcard-import 13 | from dxlclient._global_settings import * 14 | from dxlclient._product_props import * 15 | 16 | __version__ = get_product_version() 17 | 18 | class _NullHandler(logging.Handler): 19 | def emit(self, record): 20 | pass 21 | 22 | logging.getLogger(__name__).addHandler(_NullHandler()) 23 | 24 | class _ObjectTracker(object): 25 | """ 26 | Utility class used to track DXL Client-specific object instances 27 | """ 28 | 29 | # The object tracker instance (singleton) 30 | _instance = None 31 | 32 | def __init__(self): 33 | """Constructor""" 34 | self._obj_count = 0 35 | self._enabled = False 36 | self._lock = RLock() 37 | self._logger = logging.getLogger(__name__) 38 | 39 | @property 40 | def enabled(self): 41 | """ 42 | Whether the object tracker is enabled 43 | """ 44 | return self._enabled 45 | 46 | @enabled.setter 47 | def enabled(self, val): 48 | self._enabled = val 49 | 50 | def obj_constructed(self, obj): 51 | """ 52 | Tracks that the specified object was constructed 53 | 54 | :param obj: The object that was constructed 55 | """ 56 | if self._enabled: 57 | with self._lock: 58 | self._obj_count += 1 59 | self._logger.debug( 60 | "Constructed: %s.%s objCount=%d", 61 | obj.__module__, obj.__class__.__name__, self._obj_count) 62 | 63 | def obj_destructed(self, obj): 64 | """ 65 | Tracks that the specified object was destructed 66 | 67 | :param obj: The object that was destructed 68 | """ 69 | if self._enabled: 70 | with self._lock: 71 | self._obj_count -= 1 72 | self._logger.debug( 73 | "Destructed: %s.%s objCount=%d", 74 | obj.__module__, obj.__class__.__name__, self._obj_count) 75 | 76 | @property 77 | def obj_count(self): 78 | """ 79 | The current count of object instances 80 | """ 81 | with self._lock: 82 | return self._obj_count 83 | 84 | @staticmethod 85 | def get_instance(): 86 | """ 87 | Returns the object tracker instance 88 | 89 | :return: The object tracker instance 90 | """ 91 | # Instance creation should be synchronized 92 | if not _ObjectTracker._instance: 93 | _ObjectTracker._instance = _ObjectTracker() 94 | 95 | return _ObjectTracker._instance 96 | 97 | 98 | class _BaseObject(object): 99 | """ 100 | Base class for the DXL client-related classes 101 | """ 102 | 103 | def __init__(self): 104 | """Constructor""" 105 | _ObjectTracker.get_instance().obj_constructed(self) 106 | 107 | def __del__(self): 108 | """Destructor""" 109 | _ObjectTracker.get_instance().obj_destructed(self) 110 | 111 | # make all classes accessible from dxlclient package 112 | # pylint: disable=wrong-import-position 113 | from dxlclient._uuid_generator import * 114 | from dxlclient.broker import * 115 | from dxlclient._request_manager import * 116 | from dxlclient.message import * 117 | 118 | from dxlclient.exceptions import * 119 | from dxlclient.callbacks import * 120 | from dxlclient._callback_manager import * 121 | 122 | from dxlclient.client_config import * 123 | from dxlclient.client import * 124 | from dxlclient._dxl_utils import * 125 | from dxlclient.service import * 126 | -------------------------------------------------------------------------------- /dxlclient/__main__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ dxlclient bootstrap for CLI subcommands. """ 7 | 8 | from __future__ import absolute_import 9 | from dxlclient._cli import cli_run 10 | 11 | cli_run() 12 | -------------------------------------------------------------------------------- /dxlclient/_cli/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ############################################################################### 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ############################################################################### 5 | 6 | """Helpers for the client cli""" 7 | 8 | from __future__ import absolute_import 9 | import argparse 10 | import logging 11 | import sys 12 | 13 | from dxlclient._cli._cli_subcommands import \ 14 | GenerateCsrAndPrivateKeySubcommand,\ 15 | ProvisionDxlClientSubcommand,\ 16 | UpdateConfigSubcommand 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | _SUBCOMMAND_CLASSES = [GenerateCsrAndPrivateKeySubcommand, 21 | ProvisionDxlClientSubcommand, 22 | UpdateConfigSubcommand] 23 | 24 | 25 | def _create_argparser(): 26 | """ 27 | Create the top-level argparser for the cli 28 | :return: the argparser 29 | :rtype: argparse.ArgumentParser 30 | """ 31 | parser = argparse.ArgumentParser(prog="dxlclient") 32 | parser.add_argument("-s", "--silent", action="store_true", 33 | required=False, 34 | help="""show only errors (no info/debug messages) 35 | while a command is running""") 36 | parser.add_argument("-v", "--verbose", action="count", 37 | default=0, 38 | required=False, 39 | help="""Verbose mode. Additional v characters increase 40 | the verbosity level, e.g., -vv, -vvv.""") 41 | return parser 42 | 43 | 44 | def _add_subcommand_argparsers(parser): 45 | """ 46 | Append subparsers for each of the cli subcommands to the supplied argparser 47 | :param argparse.ArgumentParser parser: the base argparser 48 | """ 49 | subparsers = parser.add_subparsers(title="subcommands") 50 | 51 | # Adding these lines to force argparser to validate the presence of a 52 | # subcommand in Python 3. See https://bugs.python.org/issue9253#msg186387. 53 | subparsers.required = True 54 | subparsers.dest = 'subcommand' 55 | 56 | for subcommand_class in _SUBCOMMAND_CLASSES: 57 | subcommand = subcommand_class() 58 | subcommand_parser = subparsers.add_parser(subcommand.name, 59 | help=subcommand.help, 60 | parents=subcommand.parents) 61 | subcommand_parser.set_defaults(func=subcommand.execute) 62 | subcommand.add_parser_args(subcommand_parser) 63 | 64 | 65 | def _get_log_level(verbosity_level): 66 | """ 67 | Translate the supplied numeric verbosity level into a :mod:`logging` level 68 | :param int verbosity_level: the verbosity level 69 | :return: An appropriate level from :mod:`logging`, one of 70 | :const:`logging.ERROR`, :const:`logging.DEBUG`, or 71 | :const:`logging.INFO`. 72 | """ 73 | log_level = logging.ERROR 74 | if verbosity_level >= 2: 75 | log_level = logging.DEBUG 76 | elif verbosity_level >= 1: 77 | log_level = logging.INFO 78 | return log_level 79 | 80 | 81 | def _get_log_formatter(verbosity_level): 82 | """ 83 | Get a log formatter string based on the supplied numeric verbosity level. 84 | :param int verbosity_level: the verbosity level 85 | :return: the log formatter string 86 | :rtype: str 87 | """ 88 | formatter = "%(levelname)s: %(message)s" 89 | if verbosity_level >= 3: 90 | formatter = "%(levelname)s: %(name)s: %(message)s" 91 | return formatter 92 | 93 | 94 | def cli_run(): 95 | """ 96 | Main routine for running CLI commands 97 | """ 98 | parser = _create_argparser() 99 | _add_subcommand_argparsers(parser) 100 | 101 | input_args = sys.argv[1:] 102 | if not input_args: 103 | input_args = ["-h"] 104 | parsed_args = parser.parse_args(input_args) 105 | 106 | verbosity_level = 0 if parsed_args.silent else parsed_args.verbose+1 107 | logging.basicConfig(level=_get_log_level(verbosity_level), 108 | format=_get_log_formatter(verbosity_level)) 109 | try: 110 | parsed_args.func(parsed_args) 111 | except Exception as ex: # pylint: disable=broad-except 112 | logger.error("Command failed. Message: %s", ex) 113 | if verbosity_level >= 2: 114 | raise 115 | sys.exit(1) 116 | -------------------------------------------------------------------------------- /dxlclient/_cli/_management_service.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ############################################################################### 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ############################################################################### 5 | 6 | """Helpers for making requests to a Management Service for CLI subcommands""" 7 | 8 | from __future__ import absolute_import 9 | import json 10 | import logging 11 | import warnings 12 | import requests 13 | from requests.auth import HTTPBasicAuth 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class ManagementService(object): 19 | """ 20 | Handles REST invocation of Management Service 21 | """ 22 | # pylint: disable=too-many-arguments 23 | def __init__(self, host, port, username, password, verify): 24 | """ 25 | Constructor parameters: 26 | 27 | :param str host: the hostname of the Management Service to run remote 28 | commands on 29 | :param str port: the port of the desired Management Service 30 | :param str username: the username to run the remote commands as 31 | :param str password: the password for the Management Service user 32 | :param verify: If the value is a `bool`, the value specifies whether or 33 | not to verify the Management Service"s certificate. If the value 34 | is a `str`, the value specifies the location of a file of CA 35 | certificates to use when verifying the Management Service"s 36 | certificate. 37 | :type verify: bool or str 38 | """ 39 | self._host = host 40 | self._port = port 41 | self._base_url = "https://{}:{}/remote".format(host, port) 42 | self._port = port 43 | self._auth = HTTPBasicAuth(username, password) 44 | self._session = requests.Session() 45 | self._verify = verify 46 | 47 | def invoke_command(self, command_name, params=None): 48 | """ 49 | Invokes the given remote command by name with the supplied parameters 50 | 51 | :param str command_name: The name of the Management Service remote 52 | command to invoke 53 | :param dict params: Parameters to pass to the remote command 54 | :return: the response for the Management Service remote command 55 | :rtype: str or unicode 56 | """ 57 | params = params if params is not None else {} 58 | params[":output"] = "json" 59 | request_target = "{}:{}/{}".format(self._host, self._port, 60 | command_name) 61 | return self._parse_response(self._send_request(command_name, params), 62 | request_target) 63 | 64 | def _send_request(self, command_name, params=None): 65 | """ 66 | Sends a request to the Management Service with the supplied command 67 | name and parameters 68 | 69 | :param str command_name: The command name to invoke 70 | :param dict params: The parameters to provide for the command 71 | :return: the response object from Management Service 72 | :rtype: requests.Response 73 | """ 74 | _request_url = "{}/{}".format(self._base_url, command_name) 75 | logger.debug("Invoking request %s with the following parameters:", 76 | _request_url) 77 | logger.debug(params) 78 | with warnings.catch_warnings(): 79 | warnings.filterwarnings("ignore", ".*subjectAltName.*") 80 | if not self._verify: 81 | warnings.filterwarnings("ignore", "Unverified HTTPS request") 82 | return self._session.get(_request_url, 83 | auth=self._auth, 84 | params=params, 85 | verify=self._verify) 86 | 87 | @staticmethod 88 | def _parse_response(response, request_target): 89 | """ 90 | Parses the response object from Management Service. Removes the 91 | return status and code from the response body and returns just the 92 | JSON-decoded remote command response. Raises an exception if an error 93 | response is returned. 94 | 95 | :param requests.Response response: the Management Service remote 96 | command response object to parse 97 | :return: the Management Service remote command results 98 | :rtype: str 99 | :raise Exception: if the service returns a non-200 status code or the 100 | first line in the response has text other than "OK:" 101 | """ 102 | response_body = response.text 103 | status_code = response.status_code 104 | 105 | logger.debug("Response: %s", response_body) 106 | if status_code != 200: 107 | raise Exception( 108 | "Request to {} failed with HTTP error code: {}. Reason: {}". 109 | format(request_target, status_code, response.reason)) 110 | 111 | if ":" not in response_body: 112 | raise Exception( 113 | "Did not find ':' status delimiter in response body") 114 | 115 | response_status_delimiter = response_body.index(":") 116 | status = response_body[:response_status_delimiter] 117 | response_detail = response_body[response_status_delimiter+1:].strip() 118 | 119 | if status != "OK": 120 | raise Exception( 121 | "Request to {} failed with status: {}. Message: {}".format( 122 | request_target, status.strip(), response_detail)) 123 | 124 | return json.loads(response_detail) 125 | -------------------------------------------------------------------------------- /dxlclient/_compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ Abstraction layer for Python 2 / 3 compatibility. """ 7 | 8 | import sys 9 | 10 | # pylint: disable=invalid-name, unused-import, undefined-variable 11 | 12 | try: 13 | from queue import Queue 14 | except ImportError: 15 | from Queue import Queue 16 | 17 | if sys.version_info[0] > 2: 18 | def iter_dict_items(d): 19 | """ 20 | Python 3 wrapper for getting a dictionary iterator. 21 | 22 | :param d: The dictionary 23 | :return: The iterator. 24 | """ 25 | return d.items() 26 | def is_string(obj): 27 | """ 28 | Python 3 wrapper for determining if an object is a "string" (unicode). 29 | 30 | :param obj: The object 31 | :return: True if the object is a unicode string, False if not. 32 | :rtype: bool 33 | """ 34 | return isinstance(obj, str) 35 | else: 36 | def iter_dict_items(d): 37 | """ 38 | Python 2 wrapper for getting a dictionary iterator. 39 | 40 | :param d: The dictionary 41 | :return: The iterator. 42 | """ 43 | return d.iteritems() 44 | def is_string(obj): 45 | """ 46 | Python 2 wrapper for determining if an object is a "string" (unicode 47 | or byte-string). 48 | 49 | :param obj: The object 50 | :return: True if the object is a "string", False if not. 51 | :rtype: bool 52 | """ 53 | return isinstance(obj, basestring) 54 | -------------------------------------------------------------------------------- /dxlclient/_dxl_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ############################################################################### 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ############################################################################### 5 | 6 | """ DXL utility classes. """ 7 | 8 | from __future__ import absolute_import 9 | import errno 10 | import os 11 | 12 | from dxlclient import _BaseObject 13 | 14 | 15 | class DxlUtils(object): 16 | """ 17 | Utility methods for use by the DXL-related classes 18 | """ 19 | 20 | @staticmethod 21 | def func_name(): 22 | """ 23 | Returns the current function name 24 | :return: 25 | """ 26 | from inspect import currentframe 27 | 28 | return currentframe().f_back.f_code.co_name 29 | 30 | @staticmethod 31 | def _wildcard_generator(topic): 32 | """ 33 | Helper method that splits a topic to obtain it's wildcard 34 | i.e. /foo/bar -> /foo/# 35 | /foo/bar/ -> /foo/bar/# 36 | /foo/bar/# -> /foo/# 37 | :param topic: channel topic. 38 | :return: wildcarded topic 39 | """ 40 | if not topic: 41 | return "#" 42 | splitted = topic.split("/") 43 | if topic[-1] != "#": 44 | return "/".join(splitted[:-1]) + "/#" 45 | if len(topic) == 2: 46 | return "#" 47 | return "/".join(splitted[:-2]) + "/#" 48 | 49 | @staticmethod 50 | def _get_wildcards(topic): 51 | """ 52 | Helper method that gets a list containing all it's possible wildcards 53 | :param topic: channel topic 54 | :return: list containing all channel wildcards 55 | """ 56 | wildcards = [] 57 | while topic != "#": 58 | topic = DxlUtils._wildcard_generator(topic) 59 | wildcards.append(topic) 60 | return wildcards 61 | 62 | @staticmethod 63 | def _validate_callback(callback): 64 | """ 65 | Validates if 'callback' is a valid WildcardCallback 66 | 67 | :param callback: Callback to validate. 68 | """ 69 | if not issubclass(callback.__class__, WildcardCallback): 70 | raise ValueError("Type mismatch on callback argument") 71 | 72 | @staticmethod 73 | def iterate_wildcards(wildcard_callback, topic): 74 | """ 75 | Iterates the wildcards for the specified topic. 76 | NOTE: This only supports "#" wildcards (not "+"). 77 | 78 | :param wildcard_callback: The callback to invoke for each wildcard 79 | :param topic: The topic 80 | :return: 81 | """ 82 | 83 | DxlUtils._validate_callback(wildcard_callback) 84 | 85 | if topic is None: 86 | return 87 | 88 | topic_wildcards = DxlUtils._get_wildcards(topic) 89 | 90 | for wildcard in topic_wildcards: 91 | wildcard_callback.on_next_wildcard(wildcard) 92 | 93 | @staticmethod 94 | def makedirs(dir_path, mode=0o755): 95 | """ 96 | Create a directory (or directory tree) per the `dir_path` argument. 97 | This is basically the same as :func:`os.makedirs` except that if the 98 | directory already exists when this is called, no exception is raised. 99 | Also, the default permissions mode is 0o755 instead of what 100 | :func:`os.makedirs` uses as a default, 0o777. 101 | 102 | :param str dir_path: directory path to create 103 | :param int mode: permissions mode to use for each directory which is 104 | created 105 | """ 106 | if dir_path: 107 | try: 108 | os.makedirs(dir_path, mode) 109 | except OSError as ex: 110 | if ex.errno != errno.EEXIST: 111 | raise 112 | 113 | @staticmethod 114 | def save_to_file(filename, data, mode=0o644): 115 | """ 116 | Save a data string to a file. If any directories in the file path do 117 | not exist, the directories are created (using a permission mode of 118 | 0o755. If the file already exists, its contents are replaced with the 119 | contents of `data`. 120 | 121 | :param str filename: name of the file to save 122 | :param data: data to be saved 123 | :type data: str or bytes 124 | :param int mode: permissions mode to use for the file 125 | """ 126 | DxlUtils.makedirs(os.path.dirname(filename)) 127 | with os.fdopen(os.open(filename, os.O_WRONLY | os.O_CREAT, mode), 128 | 'wb' if isinstance(data, bytes) else 'w') as handle: 129 | handle.write(data) 130 | 131 | 132 | class WildcardCallback(_BaseObject): 133 | """ 134 | Callback that is invoked for each wildcard pattern found 135 | """ 136 | 137 | def on_next_wildcard(self, wildcard): 138 | """ 139 | Invoked for the next wildcard pattern found 140 | 141 | :param wildcard: The wildcard pattern 142 | """ 143 | raise NotImplementedError("Must be implemented in a child class.") 144 | -------------------------------------------------------------------------------- /dxlclient/_global_settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ Global configuration functions. """ 7 | 8 | from __future__ import absolute_import 9 | import os 10 | 11 | if os.name.lower() == "posix": 12 | PATH_CACHE = "/var/McAfee/{0}".format("dxlclient") 13 | else: 14 | PATH_CACHE = "./.{0}".format("dxlclient") 15 | 16 | PATH_CONFIG = PATH_CACHE + '/conf' 17 | PATH_KEYSTORE = PATH_CACHE + '/keystore' 18 | PATH_LOGS = PATH_CACHE + '/logs' 19 | 20 | FILE_CA_BUNDLE = "cabundle.pem" 21 | 22 | FILE_CERT_PFX = "dxlcert.p12" 23 | FILE_CERT_PEM = "dxlcert.pem" 24 | FILE_DXL_PRIVATE_KEY = "dxlprivatekey.pem" 25 | 26 | FILE_CONFIG = PATH_CONFIG + '/config' 27 | 28 | 29 | def get_cache_dir(): 30 | """ 31 | Returns current cache folder. 32 | 33 | :returns: {@code string}: Cache folder. 34 | """ 35 | return PATH_CACHE 36 | 37 | 38 | def get_config_dir(): 39 | """ 40 | Returns current configuration folder. 41 | 42 | :returns: {@code string}: Configuration folder. 43 | """ 44 | return PATH_CONFIG 45 | 46 | 47 | def get_keystore_dir(): 48 | """ 49 | Returns current keystore folder. 50 | 51 | :returns: {@code string}: Keystore folder. 52 | """ 53 | return PATH_KEYSTORE 54 | 55 | 56 | def get_logs_dir(): 57 | """ 58 | Returns current logs folder. 59 | 60 | :returns: {@code string}: Logs folder. 61 | """ 62 | return PATH_LOGS 63 | 64 | 65 | def get_ca_bundle_pem(): 66 | """ 67 | Returns current CA Bundle filename. 68 | 69 | :returns: {@code string}: CA Bundle filename. 70 | """ 71 | return os.path.join(PATH_KEYSTORE, FILE_CA_BUNDLE) 72 | 73 | 74 | def get_cert_file_pfx(): 75 | """ 76 | Returns current PFX certificate filename. 77 | 78 | :returns: {@code string}: PFX certificate filename. 79 | """ 80 | return os.path.join(PATH_KEYSTORE, FILE_CERT_PFX) 81 | 82 | 83 | def get_cert_file_pem(): # pylint: disable=invalid-name 84 | """ 85 | Returns current PEM certificate filename. 86 | 87 | :returns: {@code string}: PEM certificate filename. 88 | """ 89 | return os.path.join(PATH_KEYSTORE, FILE_CERT_PEM) 90 | 91 | def get_dxl_private_key(): # pylint: disable=invalid-name 92 | """ 93 | Returns current PEM certificate filename. 94 | 95 | :returns: {@code string}: PEM certificate filename. 96 | """ 97 | return os.path.join(PATH_KEYSTORE, FILE_DXL_PRIVATE_KEY) 98 | 99 | def get_dxl_config_file(): 100 | """ 101 | Returns default dxl config file 102 | 103 | :return: 104 | """ 105 | return FILE_CONFIG 106 | -------------------------------------------------------------------------------- /dxlclient/_product_props.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2024 Musarubra US LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ Product properties, used for packaging. """ 7 | 8 | __version__ = "5.7.0.1" 9 | 10 | __product_id__ = "DXL_____1000" 11 | 12 | __product_name__ = "McAfee Data Exchange Layer" 13 | 14 | __product_props__ = { 15 | "General": 16 | { 17 | "Version": __version__, 18 | "ProductName": __product_name__, 19 | "Language": "0000" 20 | } 21 | } 22 | 23 | def get_product_id(): 24 | """ 25 | Returns DXL Client product ID. 26 | 27 | :returns: {@code string}: Product ID. 28 | """ 29 | return __product_id__ 30 | 31 | 32 | def get_product_version(): 33 | """ 34 | Returns DXL Client version. 35 | 36 | :returns: {@code string}: version. 37 | """ 38 | return __version__ 39 | 40 | 41 | def get_product_name(): 42 | """ 43 | Returns DXL Client product name. 44 | 45 | :returns: {@code string}: product name. 46 | """ 47 | return __product_name__ 48 | 49 | 50 | def get_product_props(): 51 | """ 52 | Returns DXL Client properties. 53 | 54 | :returns: {@code dict}: Properties of the client.. 55 | """ 56 | return __product_props__ 57 | -------------------------------------------------------------------------------- /dxlclient/_thread_pool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ 7 | Classes which provide a thread pool implementation, e.g., for use in 8 | concurrent processing of incoming DXL messages by registered callbacks. 9 | """ 10 | 11 | from __future__ import absolute_import 12 | from threading import Thread 13 | import logging 14 | 15 | from dxlclient import _BaseObject, _ObjectTracker 16 | from dxlclient._uuid_generator import UuidGenerator 17 | 18 | from ._compat import Queue 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class ThreadPoolWorker(Thread): 24 | """ 25 | Thread executing tasks from a given tasks queue. 26 | """ 27 | 28 | def __init__(self, tasks, thread_prefix): 29 | """ 30 | Constructs a ThreadPoolWorker. 31 | """ 32 | Thread.__init__(self) 33 | 34 | _ObjectTracker.get_instance().obj_constructed(self) 35 | 36 | self.tasks = tasks 37 | self.daemon = True 38 | self.name = thread_prefix + "-" + UuidGenerator.generate_id_as_string() 39 | self.start() 40 | 41 | def __del__(self): 42 | _ObjectTracker.get_instance().obj_destructed(self) 43 | 44 | def run(self): 45 | """ 46 | Runs the worker. 47 | """ 48 | while True: 49 | func, args, kargs = self.tasks.get() 50 | try: 51 | if func is None: 52 | # Exit the thread 53 | return 54 | func(*args, **kargs) 55 | except Exception: # pylint: disable=broad-except 56 | logger.exception("Error in worker thread") 57 | del func 58 | self.tasks.task_done() 59 | 60 | 61 | class ThreadPool(_BaseObject): 62 | """ 63 | Pool of threads consuming tasks from a queue. 64 | """ 65 | 66 | def __init__(self, queue_size, num_threads, thread_prefix): 67 | """ 68 | Creates a ThreadPool. 69 | """ 70 | super(ThreadPool, self).__init__() 71 | self._tasks = Queue(queue_size) 72 | self._threads = [] 73 | for _ in range(num_threads): 74 | t = ThreadPoolWorker(self._tasks, thread_prefix) 75 | self._threads.append(t) 76 | 77 | def add_task(self, func, *args, **kargs): 78 | """Add a task to the queue""" 79 | self._tasks.put((func, args, kargs)) 80 | 81 | def wait_completion(self): 82 | """Wait for completion of all the tasks in the queue""" 83 | self._tasks.join() 84 | 85 | def shutdown(self, wait_complete=True): 86 | """Shuts down the thread pool""" 87 | logger.debug("Shutting down thread pool...") 88 | if wait_complete: 89 | self.wait_completion() 90 | 91 | # Add task to stop the thread 92 | for _ in self._threads: 93 | self.add_task(None) 94 | 95 | # Wait for threads to exit 96 | if wait_complete: 97 | for t in self._threads: 98 | t.join() 99 | -------------------------------------------------------------------------------- /dxlclient/_uuid_generator.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ 7 | Contains the :class:`UuidGenerator` class, which generates formatted UUIDs, 8 | e.g., for applying unique identifiers to new DXL messages. 9 | """ 10 | 11 | from __future__ import absolute_import 12 | import uuid 13 | 14 | 15 | class UuidGenerator(object): 16 | """ 17 | Generator used to generate a universally unique identifier (UUID) string that 18 | is all lowercase and has enclosing brackets (following McAfee Agent format). 19 | """ 20 | 21 | @staticmethod 22 | def generate_id(): 23 | """ 24 | Generates and returns a UUID 25 | 26 | :return: The generated UUID 27 | """ 28 | return uuid.uuid4() 29 | 30 | @staticmethod 31 | def generate_id_as_string(): 32 | """ 33 | Generates and returns a random UUID that is all lowercase and has enclosing brackets 34 | 35 | :return: A UUID string that is all lowercase and has enclosing brackets 36 | """ 37 | return "{" + str(UuidGenerator.generate_id()).lower() + "}" 38 | 39 | @staticmethod 40 | def from_string(string): 41 | """ 42 | Converts the specified UUID string into a UUID instance 43 | 44 | :param string: The UUID string 45 | :return: The corresponding UUID instance 46 | """ 47 | return uuid.UUID(string) 48 | 49 | @staticmethod 50 | def to_string(uid): 51 | """ 52 | Converts the specified UUID into string that is all lowercase and has enclosing brackets 53 | 54 | :param uid: The UUID 55 | :return: A UUID string that is all lowercase and has enclosing brackets 56 | """ 57 | return "{" + str(uid).lower() + "}" 58 | 59 | @staticmethod 60 | def normalize(string): 61 | """ 62 | Normalizes the specified UUID string 63 | 64 | :param string: The UUID string 65 | :return: A UUID string that is all lowercase and has enclosing brackets 66 | """ 67 | return UuidGenerator.to_string(UuidGenerator.from_string(string)) 68 | -------------------------------------------------------------------------------- /dxlclient/callbacks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ Classes for the different DXL message callbacks. """ 7 | 8 | from __future__ import absolute_import 9 | from dxlclient import _BaseObject 10 | 11 | 12 | class MessageCallback(_BaseObject): 13 | """ 14 | Base class for the different callbacks 15 | """ 16 | pass 17 | 18 | class EventCallback(MessageCallback): 19 | """ 20 | Concrete instances of this interface are used to receive :class:`dxlclient.message.Event` messages. 21 | 22 | To receive events, a concrete instance of this callback must be created and registered with a 23 | :class:`dxlclient.client.DxlClient` instance via the :func:`dxlclient.client.DxlClient.add_event_callback` 24 | method. 25 | 26 | The following is a simple example of using an event callback: 27 | 28 | .. code-block:: python 29 | 30 | from dxlclient.callbacks import EventCallback 31 | 32 | class MyEventCallback(EventCallback): 33 | def on_event(self, event): 34 | print("Received event! " + event.source_client_id) 35 | 36 | dxl_client.add_event_callback("/testeventtopic", MyEventCallback()) 37 | 38 | **NOTE:** By default when registering an event callback the client will automatically subscribe 39 | (:func:`dxlclient.client.DxlClient.subscribe`) to the topic. 40 | 41 | The following demonstrates a client that is sending an event message that would be received by the 42 | callback above. 43 | 44 | .. code-block:: python 45 | 46 | from dxlclient.message import Event 47 | 48 | # Create the event message 49 | evt = Event("/testeventtopic") 50 | 51 | # Populate the event payload 52 | evt.payload = "testing".encode() 53 | 54 | # Send the event 55 | dxl_client.send_event(evt) 56 | """ 57 | 58 | def on_event(self, event): 59 | """ 60 | Invoked when an :class:`dxlclient.message.Event` has been received. 61 | 62 | :param event: The :class:`dxlclient.message.Event` message that was received 63 | """ 64 | raise NotImplementedError("Must be implemented in a child class.") 65 | 66 | 67 | class RequestCallback(MessageCallback): 68 | """ 69 | Concrete instances of this interface are used to receive :class:`dxlclient.message.Request` messages. 70 | 71 | Request callbacks are typically used when implementing a "service". 72 | 73 | See :class:`dxlclient.service.ServiceRegistrationInfo` for more information on how to register a 74 | service. 75 | """ 76 | def on_request(self, request): 77 | """ 78 | Invoked when an :class:`dxlclient.message.Request` has been received. 79 | 80 | :param request: The :class:`dxlclient.message.Request` message that was received 81 | """ 82 | raise NotImplementedError("Must be implemented in a child class.") 83 | 84 | 85 | class ResponseCallback(MessageCallback): 86 | """ 87 | Concrete instances of this interface are used to receive :class:`dxlclient.message.Response` messages. 88 | 89 | Response callbacks are typically used when invoking a service asynchronously. 90 | 91 | The following is a simple example of using a response callback with an asynchronous service invocation: 92 | 93 | .. code-block:: python 94 | 95 | from dxlclient.message import Request 96 | from dxlclient.callbacks import ResponseCallback 97 | 98 | class MyResponseCallback(ResponseCallback): 99 | def on_response(self, response): 100 | print("Received response! " + response.service_id) 101 | 102 | request = Request("/testservice/testrequesttopic") 103 | dxl_client.async_request(request, MyResponseCallback()) 104 | 105 | """ 106 | 107 | def on_response(self, response): 108 | """ 109 | Invoked when an :class:`dxlclient.message.Response` has been received. 110 | 111 | :param response: The :class:`dxlclient.message.Response` message that was received 112 | """ 113 | raise NotImplementedError("Must be implemented in a child class.") 114 | -------------------------------------------------------------------------------- /dxlclient/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ Classes for the different exceptions that the dxlclient APIs can raise. """ 7 | 8 | class DxlException(Exception): 9 | """ 10 | A general Data Exchange Layer (DXL) exception 11 | """ 12 | 13 | 14 | class MalformedBrokerUriException(DxlException): 15 | """ 16 | An exception that is raised when a URL related to a DXL broker is malformed 17 | """ 18 | 19 | 20 | class WaitTimeoutException(DxlException): 21 | """ 22 | Exception that is raised when a wait timeout is exceeded 23 | """ 24 | 25 | 26 | class BrokerListError(Exception): 27 | """ 28 | Exception raised when a specified broker list is invalid 29 | """ 30 | pass 31 | 32 | 33 | class InvalidProxyConfigurationError(Exception): 34 | """ 35 | Exception raised when specified HTTP proxy address or port is invalid 36 | """ 37 | 38 | 39 | class NoBrokerSpecifiedError(Exception): 40 | """ 41 | Exception raised when no brokers are specified 42 | """ 43 | -------------------------------------------------------------------------------- /dxlclient/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/dxlclient/test/__init__.py -------------------------------------------------------------------------------- /dxlclient/test/async_callback_timeout_test.py: -------------------------------------------------------------------------------- 1 | """ Test that ensures that async callbacks are being cleaned up via timeout """ 2 | 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | import time 6 | from nose.plugins.attrib import attr 7 | from dxlclient import ResponseCallback, UuidGenerator, ServiceRegistrationInfo, Request 8 | from dxlclient.test.base_test import BaseClientTest 9 | from dxlclient.test.test_service import TestService 10 | 11 | # pylint: disable=missing-docstring 12 | 13 | 14 | class AsyncCallbackTimeoutTest(BaseClientTest): 15 | 16 | @attr('manual') 17 | def test_execute_async_callback_timeout(self): 18 | 19 | # TODO: Set SYSPROP_ASYNC_CALLBACK_CHECK_INTERVAL = 10000 when it is available 20 | def resp_callback(): 21 | pass 22 | 23 | callback = ResponseCallback() 24 | callback.on_response = resp_callback 25 | 26 | with self.create_client() as client: 27 | client.connect() 28 | 29 | req_topic = UuidGenerator.generate_id_as_string() 30 | missing_topic = UuidGenerator.generate_id_as_string() 31 | 32 | test_service = TestService(client, 1) 33 | 34 | def empty_on_request(_): 35 | pass 36 | 37 | test_service.on_request = empty_on_request 38 | 39 | reg_info = ServiceRegistrationInfo(client, "async_callback_test_service") 40 | reg_info.add_topic(req_topic, test_service) 41 | # Register the service 42 | client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) 43 | 44 | async_req = Request(destination_topic=req_topic) 45 | client.async_request(async_req, callback) # TODO: Use the method with timeout when is will available 46 | 47 | for _ in range(0, 10): 48 | req = Request(destination_topic=req_topic) 49 | client.async_request(req, callback) # TODO: Use the updated method with timeout when it is available 50 | 51 | req_for_error = Request(destination_topic=missing_topic) 52 | client.async_request(req_for_error) 53 | async_callback_count = client._get_async_callback_count() 54 | self.assertEqual(11, async_callback_count) 55 | 56 | for _ in range(0, 20): 57 | print("asyncCallbackCount = " + str(client._get_async_callback_count())) 58 | time.sleep(1) 59 | req = Request(destination_topic=req_topic) 60 | client.async_request(req, callback) 61 | 62 | self.assertEqual(1, async_callback_count) 63 | 64 | # TODO: Restore the value of SYSPROP_ASYNC_CALLBACK_CHECK_INTERVAL 65 | -------------------------------------------------------------------------------- /dxlclient/test/async_flood_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Slams a service with a flood of asynchronous tests. The PAHO library by default will 3 | deadlock when it is waiting to complete a publish and at the same time receives an 4 | incoming message. 5 | 6 | This test ensures that the changes we made to the PAHO library now work in this particular 7 | scenario. 8 | """ 9 | 10 | from __future__ import absolute_import 11 | from __future__ import print_function 12 | from threading import Condition 13 | import time 14 | from nose.plugins.attrib import attr 15 | from dxlclient import ServiceRegistrationInfo, UuidGenerator 16 | from dxlclient import RequestCallback, Response, Message, ResponseCallback, Request 17 | from dxlclient.test.base_test import BaseClientTest 18 | 19 | # pylint: disable=missing-docstring 20 | 21 | 22 | @attr('system') 23 | class AsyncFloodTest(BaseClientTest): 24 | # The count of requests to send 25 | REQUEST_COUNT = 1000 26 | 27 | # Amount of time to wait for the test to succeed 28 | WAIT_TIME = 90 29 | 30 | # The service registration information 31 | m_info = None 32 | 33 | resp_condition = Condition() 34 | response_count = 0 35 | error_count = 0 36 | 37 | @attr('system') 38 | def test_async_flood(self): 39 | 40 | channel = UuidGenerator.generate_id_as_string() 41 | 42 | with self.create_client() as client: 43 | 44 | self.m_info = ServiceRegistrationInfo(client, channel) 45 | client.connect() 46 | client.subscribe(channel) 47 | 48 | def my_request_callback(request): 49 | try: 50 | time.sleep(0.05) 51 | resp = Response(request) 52 | resp.payload = request.payload 53 | client.send_response(resp) 54 | except Exception as ex: # pylint: disable=broad-except 55 | print(ex) 56 | 57 | req_callback = RequestCallback() 58 | req_callback.on_request = my_request_callback 59 | 60 | self.m_info.add_topic(channel, req_callback) 61 | client.register_service_sync(self.m_info, 10) 62 | 63 | with self.create_client() as client2: 64 | client2.connect() 65 | 66 | def my_response_callback(response): 67 | if response.message_type == Message.MESSAGE_TYPE_ERROR: 68 | print("Received error response: " + response._error_response) 69 | with self.resp_condition: 70 | self.error_count += 1 71 | self.resp_condition.notify_all() 72 | else: 73 | with self.resp_condition: 74 | if self.response_count % 10 == 0: 75 | print("Received request " + str(self.response_count)) 76 | self.response_count += 1 77 | self.resp_condition.notify_all() 78 | 79 | callback = ResponseCallback() 80 | callback.on_response = my_response_callback 81 | 82 | client2.add_response_callback("", callback) 83 | 84 | for i in range(0, self.REQUEST_COUNT): 85 | if i % 100 == 0: 86 | print("Sent: " + str(i)) 87 | 88 | request = Request(channel) 89 | request.payload = str(i) 90 | client2.async_request(request) 91 | 92 | if self.error_count > 0: 93 | break 94 | 95 | # Wait for all responses, an error to occur, or we timeout 96 | start_time = time.time() 97 | with self.resp_condition: 98 | while (self.response_count != self.REQUEST_COUNT) and (self.error_count == 0) and (time.time() - start_time < self.WAIT_TIME): 99 | self.resp_condition.wait(5) 100 | 101 | if self.error_count != 0: 102 | raise Exception("Received an error response!") 103 | 104 | self.assertEqual(self.REQUEST_COUNT, self.response_count, "Did not receive all messages!") 105 | -------------------------------------------------------------------------------- /dxlclient/test/async_request_test.py: -------------------------------------------------------------------------------- 1 | """ Tests the asynchronous request methods of the DxlClient """ 2 | 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | from threading import Condition 6 | from nose.plugins.attrib import attr 7 | from dxlclient.test.test_service import TestService 8 | from dxlclient import UuidGenerator, Request, ResponseCallback, ServiceRegistrationInfo 9 | from dxlclient.test.base_test import BaseClientTest, atomize 10 | 11 | # pylint: disable=missing-docstring, no-self-use 12 | 13 | 14 | class AsyncRequestTests(BaseClientTest): 15 | 16 | # The number of requests to send 17 | REQ_COUNT = 100 18 | MAX_RESPONSE_WAIT = 1 * 60 19 | 20 | response_count = 0 21 | outstanding_requests = [] 22 | # Conditions 23 | response_condition = Condition() 24 | outstanding_requests_condition = Condition() 25 | 26 | @atomize(outstanding_requests_condition) 27 | def append_outstanding_request(self, message_id): 28 | self.outstanding_requests.append(message_id) 29 | 30 | @atomize(outstanding_requests_condition) 31 | def remove_outstanding_request(self, message_id): 32 | self.outstanding_requests.remove(message_id) 33 | 34 | # 35 | # Tests the asynchronous request methods of the DxlClient. 36 | # 37 | @attr('system') 38 | def test_execute_async_request(self): 39 | 40 | # Create and register a response callback. Once the request has been processed 41 | # by the service, the requesting client will receive a notification via this 42 | # callback. At that point, we note that the request has been responded to (remove 43 | # it from the set of outstanding requests), and increment the count of responses 44 | # we have received via the callbacks. 45 | def my_response(response): 46 | with self.response_condition: 47 | # Increment count of responses received 48 | self.response_count += 1 49 | # Remove from outstanding requests 50 | self.remove_outstanding_request(response.request_message_id) 51 | # Notify that a response has been received (are we done yet?) 52 | self.response_condition.notify_all() 53 | callback = ResponseCallback() 54 | callback.on_response = my_response 55 | 56 | with self.create_client(max_retries=0) as client: 57 | try: 58 | client.connect() 59 | # Create a test service that responds to requests on a particular topic. 60 | test_service = TestService(client, 1) 61 | topic = UuidGenerator.generate_id_as_string() 62 | reg_info = ServiceRegistrationInfo(client, "async_request_runner_service") 63 | reg_info.add_topic(topic, test_service) 64 | # Register the service 65 | client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) 66 | # Add a response callback (not channel-specific) 67 | client.add_response_callback("", callback) 68 | 69 | for _ in range(0, self.REQ_COUNT): 70 | # Send a request without specifying a callback for the current request 71 | req = Request(topic) 72 | self.append_outstanding_request(req.message_id) 73 | client.async_request(req) 74 | 75 | # Send a request with a specific callback that is to receive the response. 76 | # The response will be received by two callbacks. 77 | req = Request(topic) 78 | self.append_outstanding_request(req.message_id) 79 | client.async_request(req, response_callback=callback) 80 | 81 | # Wait until all the responses are received via the response callbacks. 82 | # The "times 3" is due to the fact that 20000 requests were sent in total. 83 | # 20000 were handled via the global callback, an additional 10000 were handled 84 | # via the callback explicitly passed in the second set of requests. 85 | with self.response_condition: 86 | while self.response_count != self.REQ_COUNT * 3: 87 | current_count = self.response_count 88 | self.response_condition.wait(self.MAX_RESPONSE_WAIT) 89 | if current_count == self.response_count: 90 | self.fail("Response wait timeout") 91 | 92 | # Make sure there are no outstanding requests 93 | self.assertEqual(0, len(self.outstanding_requests)) 94 | print("Async request test: PASSED") 95 | 96 | except Exception as ex: 97 | print(ex) 98 | raise ex 99 | -------------------------------------------------------------------------------- /dxlclient/test/base_test.py: -------------------------------------------------------------------------------- 1 | """ Base class and functions for the various client tests. """ 2 | 3 | from __future__ import absolute_import 4 | from functools import wraps 5 | import os 6 | import sys 7 | from unittest import TestCase 8 | from dxlclient import DxlClientConfig, DxlClient 9 | 10 | # pylint: disable=missing-docstring, no-self-use 11 | 12 | 13 | def atomize(lock): 14 | def decorator(wrapped): 15 | @wraps(wrapped) 16 | def _wrapper(*args, **kwargs): 17 | with lock: 18 | return wrapped(*args, **kwargs) 19 | return _wrapper 20 | return decorator 21 | 22 | 23 | class TestDxlClient(DxlClient): 24 | def _destroy(self, wait_complete=True): 25 | client = self._client 26 | super(TestDxlClient, self)._destroy(wait_complete) 27 | if client: 28 | # Close out sockets that the MQTT client is holding in order to 29 | # avoid socket ResourceWarning messages appearing when tests are 30 | # run on Python 3. This should be removed when this issue is 31 | # addressed: 32 | # https://github.com/eclipse/paho.mqtt.python/issues/170 33 | if hasattr(client, "_sockpairR") and client._sockpairR: 34 | client._sockpairR.close() 35 | client._sockpairR = None 36 | if hasattr(client, "_sockpairW") and client._sockpairW: 37 | client._sockpairW.close() 38 | client._sockpairW = None 39 | 40 | 41 | class BaseClientTest(TestCase): 42 | DEFAULT_TIMEOUT = 5 * 60 43 | DEFAULT_RETRIES = 3 44 | POST_OP_DELAY = 8 45 | REG_DELAY = 60 46 | 47 | def create_client(self, max_retries=DEFAULT_RETRIES, incoming_message_thread_pool_size=1): 48 | config = DxlClientConfig.create_dxl_config_from_file(os.path.dirname(os.path.abspath(__file__)) + 49 | "/client_config.cfg") 50 | config.incoming_message_thread_pool_size = incoming_message_thread_pool_size 51 | 52 | config.connect_retries = max_retries 53 | 54 | return TestDxlClient(config) 55 | 56 | if sys.version_info[0] > 2: 57 | import builtins # pylint: disable=import-error, unused-import 58 | else: 59 | import __builtin__ # pylint: disable=import-error 60 | builtins = __builtin__ # pylint: disable=invalid-name 61 | -------------------------------------------------------------------------------- /dxlclient/test/broker_registry_query_test.py: -------------------------------------------------------------------------------- 1 | """ Tests the broker registry query. """ 2 | 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | from nose.plugins.attrib import attr 6 | from dxlclient.test.base_test import BaseClientTest 7 | from dxlclient import Request, ErrorResponse 8 | 9 | # pylint: disable=missing-docstring 10 | 11 | @attr('system') 12 | class BrokerRegistryQueryTest(BaseClientTest): 13 | 14 | @attr('system') 15 | def test_execute_registry_query(self): 16 | with self.create_client() as client: 17 | client.connect() 18 | topic = "/mcafee/service/dxl/brokerregistry/query" 19 | 20 | req = Request(topic) 21 | req.payload = "{}" 22 | response = client.sync_request(req) 23 | 24 | self.assertNotIsInstance(response, ErrorResponse) 25 | print("## sourceBrokerGuid: " + str(response.source_broker_id)) 26 | print("## sourceClientGuid: " + str(response.source_client_id)) 27 | print(str(response.payload)) 28 | -------------------------------------------------------------------------------- /dxlclient/test/client_config.cfg.template: -------------------------------------------------------------------------------- 1 | [Certs] 2 | BrokerCertChain=../../certificates/client/ca-broker.crt 3 | CertFile=../../certificates/client/client.crt 4 | PrivateKey=../../certificates/client/client.pem 5 | 6 | [Brokers] 7 | unique_broker_id_1=unique_broker_id_1;@BROKER_PORT@;@BROKER_HOSTNAME@;@BROKER_IP@ 8 | -------------------------------------------------------------------------------- /dxlclient/test/dxlclient_test.py: -------------------------------------------------------------------------------- 1 | """ Tests various methods of the DxlClient """ 2 | 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | import threading 6 | import time 7 | 8 | from nose.plugins.attrib import attr 9 | from dxlclient import UuidGenerator, ServiceRegistrationInfo, Request, ErrorResponse, EventCallback, Event 10 | from dxlclient.test.base_test import BaseClientTest 11 | from dxlclient.test.test_service import TestService 12 | 13 | # pylint: disable=missing-docstring 14 | 15 | 16 | @attr('system') 17 | class DxlClientTest(BaseClientTest): 18 | 19 | # 20 | # Tests the connect and disconnect methods of the DxlClient 21 | # 22 | @attr('system') 23 | def test_connect_and_disconnect(self): 24 | with self.create_client(max_retries=0) as client: 25 | client.connect() 26 | self.assertTrue(client.connected) 27 | client.disconnect() 28 | self.assertFalse(client.connected) 29 | 30 | # 31 | # Tests the subscribe and unsubscribe methods of the DxlClient 32 | # 33 | @attr('system') 34 | def test_subscribe_and_unsubscribe(self): 35 | with self.create_client(max_retries=0) as client: 36 | client.connect() 37 | topic = UuidGenerator.generate_id_as_string() 38 | client.subscribe(topic) 39 | self.assertIn(topic, client.subscriptions) 40 | client.unsubscribe(topic) 41 | self.assertNotIn(topic, client.subscriptions) 42 | 43 | # 44 | # Tests that an unsubscribe call made after a previous unsubscribe for the 45 | # same topic does not raise an error. 46 | # 47 | @attr('system') 48 | def test_unsubscribe_for_unknown_topic_does_not_raise_error(self): 49 | with self.create_client(max_retries=0) as client: 50 | client.connect() 51 | topic = UuidGenerator.generate_id_as_string() 52 | client.subscribe(topic) 53 | self.assertIn(topic, client.subscriptions) 54 | client.unsubscribe(topic) 55 | client.unsubscribe(topic) 56 | self.assertNotIn(topic, client.subscriptions) 57 | 58 | # 59 | # Test to ensure that ErrorResponse messages can be successfully delivered 60 | # from a service to a client. 61 | # 62 | @attr('system') 63 | def test_error_message(self): 64 | with self.create_client() as client: 65 | test_service = TestService(client, 1) 66 | client.connect() 67 | 68 | error_code = 9090 69 | error_message = "My error message" 70 | 71 | topic = UuidGenerator.generate_id_as_string() 72 | 73 | # 74 | # Create a test service that returns error messages 75 | # 76 | 77 | reg_info = ServiceRegistrationInfo(client, "testErrorMessageService") 78 | reg_info.add_topic(topic, test_service) 79 | client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) 80 | 81 | test_service.return_error = True 82 | test_service.error_code = error_code 83 | test_service.error_message = error_message 84 | client.add_request_callback(topic, test_service) 85 | 86 | # Send a request and ensure the response is an error message 87 | response = client.sync_request(Request(topic)) 88 | self.assertIsInstance(response, ErrorResponse, msg="Response is not an ErrorResponse") 89 | self.assertEqual(error_code, response.error_code) 90 | self.assertEqual(error_message, response.error_message) 91 | 92 | # 93 | # Tests threading of incoming requests 94 | # 95 | @attr('system') 96 | def test_incoming_message_threading(self): 97 | max_wait = 30 98 | thread_count = 10 99 | thread_name_condition = threading.Condition() 100 | thread_name = set() 101 | 102 | event_topic = UuidGenerator.generate_id_as_string() 103 | with self.create_client(incoming_message_thread_pool_size= 104 | thread_count) as client: 105 | client.connect() 106 | event_callback = EventCallback() 107 | 108 | def on_event(_): 109 | with thread_name_condition: 110 | thread_name.add(threading.current_thread()) 111 | if len(thread_name) == thread_count: 112 | thread_name_condition.notify_all() 113 | 114 | event_callback.on_event = on_event 115 | client.add_event_callback(event_topic, event_callback) 116 | 117 | for _ in range(0, 1000): 118 | evt = Event(event_topic) 119 | client.send_event(evt) 120 | 121 | start = time.time() 122 | with thread_name_condition: 123 | while (time.time() - start < max_wait) and \ 124 | len(thread_name) < thread_count: 125 | thread_name_condition.wait(max_wait) 126 | 127 | self.assertEqual(thread_count, len(thread_name)) 128 | -------------------------------------------------------------------------------- /dxlclient/test/event_request_test.py: -------------------------------------------------------------------------------- 1 | """ Tests the event-related methods of the DxlClient. """ 2 | 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | from threading import Condition 6 | from nose.plugins.attrib import attr 7 | from dxlclient import UuidGenerator, EventCallback, Event 8 | from dxlclient.test.base_test import BaseClientTest, atomize 9 | 10 | # pylint: disable=missing-docstring 11 | 12 | 13 | @attr('system') 14 | class EventTests(BaseClientTest): 15 | 16 | # The number of events to send 17 | EVENT_COUNT = 10000 18 | MAX_EVENT_WAIT = 2 * 60 19 | 20 | event_count = 0 21 | outstanding_events = [] 22 | # Conditions 23 | event_condition = Condition() 24 | outstanding_events_condition = Condition() 25 | 26 | @atomize(outstanding_events_condition) 27 | def append_outstanding_event(self, message_id): 28 | self.outstanding_events.append(message_id) 29 | 30 | @atomize(outstanding_events_condition) 31 | def remove_outstanding_event(self, message_id): 32 | self.outstanding_events.remove(message_id) 33 | 34 | # 35 | # Tests the events-related methods of the DxlClient. 36 | # 37 | @attr('system') 38 | def test_execute_events(self): 39 | with self.create_client(max_retries=0) as client: 40 | try: 41 | client.connect() 42 | 43 | topic = UuidGenerator.generate_id_as_string() 44 | 45 | # Create and register an event callback. Ensure that all sent events are received 46 | # (via the outstanding events set). Also, track the number of total events received. 47 | def event_callback(event): 48 | with self.event_condition: 49 | # Increment count of responses received 50 | self.event_count += 1 51 | # Remove from outstanding events 52 | self.remove_outstanding_event(event.message_id) 53 | # Notify that a response has been received (are we done yet?) 54 | self.event_condition.notify_all() 55 | 56 | callback = EventCallback() 57 | callback.on_event = event_callback 58 | 59 | client.add_event_callback(topic, callback) 60 | 61 | for _ in range(0, self.EVENT_COUNT): 62 | event = Event(topic) 63 | self.append_outstanding_event(event.message_id) 64 | client.send_event(event) 65 | 66 | with self.event_condition: 67 | while self.event_count != self.EVENT_COUNT: 68 | current_count = self.event_count 69 | self.event_condition.wait(self.MAX_EVENT_WAIT) 70 | if current_count == self.event_count: 71 | self.fail("Event wait timeout.") 72 | 73 | self.assertEqual(0, len(self.outstanding_events)) 74 | print("Events test: PASSED") 75 | 76 | except Exception as ex: 77 | print(ex) 78 | raise ex 79 | -------------------------------------------------------------------------------- /dxlclient/test/message_other_fields_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests whether the 'other fields' dictionary in a message can be successfully 3 | delivered from a client to the server. 4 | """ 5 | 6 | from __future__ import absolute_import 7 | from __future__ import print_function 8 | import time 9 | from threading import Condition 10 | 11 | from nose.plugins.attrib import attr 12 | 13 | from dxlclient import UuidGenerator, Event, EventCallback 14 | from dxlclient.test.base_test import BaseClientTest 15 | 16 | # pylint: disable=missing-docstring 17 | 18 | 19 | @attr('system') 20 | class MessageOtherFieldsTest(BaseClientTest): 21 | 22 | MAX_WAIT = 1 * 60 23 | OTHER_FIELDS_COUNT = 1000 24 | 25 | event_received = None 26 | event_received_condition = Condition() 27 | 28 | @attr('system') 29 | def test_execute_message_other_fields(self): 30 | with self.create_client(max_retries=0) as client: 31 | client.connect() 32 | topic = UuidGenerator.generate_id_as_string() 33 | def on_event(event): 34 | with self.event_received_condition: 35 | try: 36 | self.event_received = event 37 | except Exception as ex: # pylint: disable=broad-except 38 | print(ex) 39 | self.event_received_condition.notify_all() 40 | 41 | event_callback = EventCallback() 42 | event_callback.on_event = on_event 43 | client.add_event_callback(topic, event_callback) 44 | 45 | event = Event(destination_topic=topic) 46 | event.other_fields = {"key" + str(i): "value" + str(i) 47 | for i in range(self.OTHER_FIELDS_COUNT)} 48 | event.other_fields[b"key_as_bytes"] = b"val_as_bytes" 49 | client.send_event(event) 50 | # Bytes values for other field keys/values are expected to be 51 | # converted to unicode strings as received from the DXL fabric. 52 | del event.other_fields[b"key_as_bytes"] 53 | event.other_fields[u"key_as_bytes"] = u"val_as_bytes" 54 | start = time.time() 55 | with self.event_received_condition: 56 | while (time.time() - start < self.MAX_WAIT) and \ 57 | not self.event_received: 58 | self.event_received_condition.wait(self.MAX_WAIT) 59 | 60 | self.assertIsNotNone(self.event_received) 61 | self.assertIsNotNone(self.event_received.other_fields) 62 | for i in range(self.OTHER_FIELDS_COUNT): 63 | self.assertEqual( 64 | event.other_fields["key" + str(i)], 65 | self.event_received.other_fields.get("key" + str(i), "")) 66 | self.assertEqual(event.other_fields["key_as_bytes"], 67 | self.event_received.other_fields.get( 68 | "key_as_bytes", "")) 69 | -------------------------------------------------------------------------------- /dxlclient/test/message_payload_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests whether payloads can be successfully delivered from a client to the server. 3 | Payloads are simply bytes of data that are used to provide application-specific 4 | information. 5 | """ 6 | 7 | from __future__ import absolute_import 8 | from __future__ import print_function 9 | from io import BytesIO 10 | import time 11 | from threading import Condition 12 | 13 | import os 14 | os.environ['MSGPACK_PUREPYTHON'] = "1" 15 | # pylint: disable=wrong-import-position 16 | import msgpack 17 | 18 | from nose.plugins.attrib import attr 19 | 20 | from dxlclient import UuidGenerator, ServiceRegistrationInfo, RequestCallback, Request 21 | from dxlclient.test.base_test import BaseClientTest 22 | 23 | # pylint: disable=missing-docstring 24 | 25 | 26 | @attr('system') 27 | class MessagePayloadTest(BaseClientTest): 28 | 29 | MAX_WAIT = 1 * 60 30 | # A test string to send 31 | TEST_STRING = "SslUtils" 32 | # A test byte to send 33 | TEST_BYTE = 1 34 | # A test integer 35 | TEST_INT = 123456 36 | 37 | request_received = None 38 | request_complete_condition = Condition() 39 | 40 | # Tests whether payloads can be successfully delivered from a client to the server. 41 | # Payloads are simply bytes of data that are used to provide application-specific 42 | # information. 43 | @attr('system') 44 | def test_execute_message_payload(self): 45 | 46 | # Create a server that handles a request, unpacks the payload, and 47 | # asserts that the information in the payload was delivered successfully. 48 | with self.create_client(max_retries=0) as service_client: 49 | service_client.connect() 50 | topic = UuidGenerator.generate_id_as_string() 51 | reg_info = ServiceRegistrationInfo(service_client, 52 | "message_payload_runner_service") 53 | 54 | # callback definition 55 | def on_request(request): 56 | with self.request_complete_condition: 57 | try: 58 | self.request_received = request 59 | except Exception as ex: # pylint: disable=broad-except 60 | print(ex) 61 | self.request_complete_condition.notify_all() 62 | 63 | request_callback = RequestCallback() 64 | request_callback.on_request = on_request 65 | reg_info.add_topic(topic, request_callback) 66 | # Register the service 67 | service_client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) 68 | 69 | with self.create_client() as request_client: 70 | request_client.connect() 71 | packer = msgpack.Packer() 72 | 73 | # Send a request to the server with information contained 74 | # in the payload 75 | request = Request(destination_topic=topic) 76 | request.payload = packer.pack(self.TEST_STRING) 77 | request.payload += packer.pack(self.TEST_BYTE) 78 | request.payload += packer.pack(self.TEST_INT) 79 | request_client.async_request(request, request_callback) 80 | 81 | start = time.time() 82 | # Wait until the request has been processed 83 | with self.request_complete_condition: 84 | while (time.time() - start < self.MAX_WAIT) and \ 85 | not self.request_received: 86 | self.request_complete_condition.wait(self.MAX_WAIT) 87 | 88 | self.assertIsNotNone(self.request_received) 89 | unpacker = msgpack.Unpacker(file_like=BytesIO(request.payload)) 90 | self.assertEqual(next(unpacker).decode('utf8'), 91 | self.TEST_STRING) 92 | self.assertEqual(next(unpacker), self.TEST_BYTE) 93 | self.assertEqual(next(unpacker), self.TEST_INT) 94 | -------------------------------------------------------------------------------- /dxlclient/test/subs_count_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests the broker/subs topic on the broker in conjunction with service 3 | registrations made from the client. 4 | """ 5 | 6 | from __future__ import absolute_import 7 | from __future__ import print_function 8 | import json 9 | import time 10 | from nose.plugins.attrib import attr 11 | from dxlclient import Request, UuidGenerator 12 | from dxlclient.test.base_test import BaseClientTest 13 | 14 | # pylint: disable=missing-docstring 15 | 16 | 17 | @attr('system') 18 | class SubsCountTest(BaseClientTest): 19 | MAX_TIME = 60 20 | 21 | @attr('system') 22 | def test_subs_count(self): 23 | clients = [] 24 | 25 | with self.create_client() as client: 26 | client.connect() 27 | 28 | for _ in range(6): 29 | _client = self.create_client() 30 | _client.connect() 31 | clients.append(_client) 32 | 33 | random1 = UuidGenerator.generate_id_as_string() 34 | random2 = UuidGenerator.generate_id_as_string() 35 | topic1 = "/foo/bar/" + random1 + "/" + random2 36 | clients[0].subscribe(topic1) 37 | clients[1].subscribe(topic1) 38 | clients[2].subscribe(topic1) 39 | clients[3].subscribe("/foo/bar/" + random1 + "/#") 40 | clients[4].subscribe("/foo/+/" + random1 + "/#") 41 | 42 | topic2 = "/foo/baz/" + random2 43 | clients[1].subscribe(topic2) 44 | clients[2].subscribe(topic2) 45 | clients[5].subscribe("#") 46 | 47 | def get_subs_count(topic): 48 | req = Request("/mcafee/service/dxl/broker/subs") 49 | req.payload = "{\"topic\":\"" + topic + "\"}" 50 | resp = client.sync_request(req, 5) 51 | return json.loads(resp.payload.decode( 52 | "utf8").rstrip("\0"))["count"] 53 | 54 | def loop_until_expected_subs_count(expected_topic_count, topic): 55 | topic_count = get_subs_count(topic) 56 | start = time.time() 57 | while topic_count != expected_topic_count and \ 58 | time.time() - start < self.MAX_TIME: 59 | time.sleep(0.1) 60 | topic_count = get_subs_count(topic) 61 | return topic_count 62 | 63 | # Topic 1 64 | expected_topic1_count = 6 65 | self.assertEqual( 66 | expected_topic1_count, 67 | loop_until_expected_subs_count(expected_topic1_count, topic1)) 68 | 69 | # Topic 2 70 | expected_topic2_count = 3 71 | self.assertEqual( 72 | expected_topic2_count, 73 | loop_until_expected_subs_count(expected_topic2_count, topic2)) 74 | 75 | for _client in clients: 76 | _client.destroy() 77 | -------------------------------------------------------------------------------- /dxlclient/test/sync_during_callback_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test to ensure that synchronous requests can't be made on 3 | the incoming message thread 4 | """ 5 | 6 | from __future__ import absolute_import 7 | from threading import Condition 8 | import time 9 | from nose.plugins.attrib import attr 10 | from dxlclient import UuidGenerator, Request, EventCallback, Event 11 | from dxlclient.test.base_test import BaseClientTest 12 | 13 | # pylint: disable=missing-docstring 14 | 15 | 16 | @attr('system') 17 | class TestSyncDuringCallback(BaseClientTest): 18 | 19 | MAX_WAIT = 60 20 | request_exception_message = None 21 | event_received = False 22 | event_received_condition = Condition() 23 | 24 | def test_execute_sync_during_callback(self): 25 | 26 | event_topic = UuidGenerator.generate_id_as_string() 27 | req_topic = UuidGenerator.generate_id_as_string() 28 | 29 | with self.create_client() as client: 30 | client.connect() 31 | 32 | # callback 33 | def event_callback(_): 34 | with self.event_received_condition: 35 | self.event_received = True 36 | try: 37 | req = Request(destination_topic=req_topic) 38 | client.sync_request(req) 39 | except Exception as ex: # pylint: disable=broad-except 40 | self.request_exception_message = str(ex) 41 | self.event_received_condition.notify_all() 42 | 43 | callback = EventCallback() 44 | callback.on_event = event_callback 45 | 46 | client.add_event_callback(event_topic, callback) 47 | 48 | event = Event(destination_topic=event_topic) 49 | client.send_event(event) 50 | 51 | start = time.time() 52 | with self.event_received_condition: 53 | while (time.time() - start < self.MAX_WAIT) and \ 54 | not self.event_received: 55 | self.event_received_condition.wait(self.MAX_WAIT) 56 | self.assertIsNotNone(self.request_exception_message) 57 | self.assertIn("different thread", self.request_exception_message) 58 | -------------------------------------------------------------------------------- /dxlclient/test/sync_request_test.py: -------------------------------------------------------------------------------- 1 | """ Tests the synchronous request methods of the DxlClient. """ 2 | 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | from threading import Condition 6 | from nose.plugins.attrib import attr 7 | from dxlclient.test.base_test import BaseClientTest 8 | from dxlclient.test.test_service import TestService 9 | from dxlclient import ServiceRegistrationInfo, Request, ErrorResponse 10 | from .thread_executor import ThreadRunExecutor 11 | 12 | # pylint: disable=missing-docstring 13 | 14 | 15 | @attr('system') 16 | class SyncRequestTests(BaseClientTest): 17 | 18 | # The number of requests to send 19 | REQUEST_COUNT = 500 20 | # Maximum time to wait for the test to complete 21 | MAX_WAIT = 5 * 60 22 | # Maximum time to wait for a response 23 | RESPONSE_WAIT = 60 24 | 25 | response_count = 0 26 | response_count_condition = Condition() 27 | 28 | event_condition = Condition() 29 | 30 | # 31 | # Tests the synchronous request methods of the DxlClient. 32 | # 33 | @attr('system') 34 | def test_execute_sync_request(self): 35 | with self.create_client(max_retries=0) as client: 36 | # Create a test service that responds to requests on a particular topic. 37 | test_service = TestService(client, 1) 38 | client.connect() 39 | topic = "event_testing" # UuidGenerator.generate_id_as_string() 40 | reg_info = ServiceRegistrationInfo(client, "sync_request_runner_service") 41 | reg_info.add_topic(topic, test_service) 42 | # Register the service 43 | client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) 44 | 45 | executor = ThreadRunExecutor(self.REQUEST_COUNT) 46 | 47 | # Sends synchronous requests with a unique thread for each request. Ensure that the 48 | # response that is received corresponds with the request that was sent. Also, keep 49 | # track of the total number of responses received. 50 | def run(): 51 | try: 52 | request = Request(topic) 53 | response = client.sync_request(request, timeout=self.RESPONSE_WAIT) 54 | self.assertNotIsInstance(response, ErrorResponse) 55 | self.assertEqual(request.message_id, response.request_message_id) 56 | 57 | with self.response_count_condition: 58 | self.response_count += 1 59 | if self.response_count % 100 == 0: 60 | print(self.response_count) 61 | self.response_count_condition.notify_all() 62 | except Exception as ex: 63 | print(ex) 64 | raise ex 65 | 66 | executor.execute(run) 67 | 68 | # Wait for all of the requests to complete 69 | with self.response_count_condition: 70 | while self.response_count != self.REQUEST_COUNT: 71 | current_count = self.response_count 72 | self.response_count_condition.wait(self.MAX_WAIT) 73 | if current_count == self.response_count: 74 | self.fail("Request wait timeout.") 75 | 76 | self.assertEqual(self.REQUEST_COUNT, self.response_count) 77 | -------------------------------------------------------------------------------- /dxlclient/test/test_request_manager.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ################################################################################ 3 | # Copyright (c) 2018 McAfee LLC - All Rights Reserved. 4 | ################################################################################ 5 | 6 | """ 7 | Test cases for the RequestManager class 8 | """ 9 | 10 | # Run with python -m unittest dxlclient.test.test_request_manager 11 | 12 | from __future__ import absolute_import 13 | import unittest 14 | 15 | from dxlclient import exceptions 16 | from dxlclient.callbacks import ResponseCallback 17 | from dxlclient import RequestManager 18 | from dxlclient import Request 19 | from dxlclient import Response 20 | from dxlclient import UuidGenerator 21 | 22 | # pylint: disable=missing-docstring 23 | 24 | 25 | class MockDxlClient(object): 26 | def __init__(self): 27 | # The unique identifier of the client 28 | self.unique_id = UuidGenerator.generate_id_as_string() 29 | 30 | def add_response_callback(self, channel, response_callback): 31 | pass 32 | 33 | def _send_request(self, request): 34 | pass 35 | 36 | 37 | class MockResponseCallback(ResponseCallback): 38 | def on_response(self, response): 39 | pass 40 | 41 | 42 | class RequestManagerTest(unittest.TestCase): 43 | def setUp(self): 44 | self.client = MockDxlClient() 45 | self.request_manager = RequestManager(self.client) 46 | 47 | def test_current_request(self): 48 | uid = UuidGenerator.generate_id_as_string() 49 | self.request_manager.add_current_request(uid) 50 | self.assertEqual(1, self.request_manager.get_current_request_queue_size()) 51 | self.assertTrue(uid in self.request_manager.current_request_message_ids) 52 | self.request_manager.remove_current_request(uid) 53 | self.assertEqual(0, self.request_manager.get_current_request_queue_size()) 54 | self.assertFalse(uid in self.request_manager.current_request_message_ids) 55 | 56 | def test_register_wait_for_response(self): 57 | request = Request(destination_topic="/test") 58 | self.request_manager.register_wait_for_response(request) 59 | self.assertEqual(1, len(self.request_manager.sync_wait_message_ids)) 60 | self.assertTrue(request.message_id in self.request_manager.sync_wait_message_ids) 61 | self.request_manager.unregister_wait_for_response(request) 62 | self.assertEqual(0, len(self.request_manager.sync_wait_message_ids)) 63 | self.assertFalse(request.message_id in self.request_manager.sync_wait_message_ids) 64 | 65 | def test_register_async_callback(self): 66 | request = Request(destination_topic="/test") 67 | callback = MockResponseCallback() 68 | 69 | self.request_manager.register_async_callback(request, callback) 70 | self.assertTrue(request.message_id in self.request_manager.callback_map) 71 | self.request_manager.unregister_async_callback(request.message_id) 72 | self.assertFalse(request.message_id in self.request_manager.callback_map) 73 | 74 | def test_wait_for_response(self): 75 | request = Request(destination_topic="/test") 76 | 77 | with self.assertRaises(exceptions.WaitTimeoutException): 78 | self.request_manager.wait_for_response(request, 2) 79 | 80 | def test_on_response_for_sync_request(self): 81 | request = Request(destination_topic="/test") 82 | self.request_manager.register_wait_for_response(request) 83 | 84 | self.assertEqual(1, len(self.request_manager.sync_wait_message_ids)) 85 | self.assertTrue(request.message_id in self.request_manager.sync_wait_message_ids) 86 | 87 | response = Response(request=request) 88 | self.request_manager.on_response(response) 89 | 90 | self.assertEqual(0, len(self.request_manager.sync_wait_message_ids)) 91 | self.assertEqual(1, len(self.request_manager.sync_wait_message_responses)) 92 | self.assertTrue(request.message_id in self.request_manager.sync_wait_message_responses) 93 | 94 | result = self.request_manager.wait_for_response(request, 2) 95 | 96 | self.assertEqual(request.message_id, result.request_message_id) 97 | 98 | def test_sync_request(self): 99 | request = Request(destination_topic="/test") 100 | 101 | with self.assertRaises(exceptions.WaitTimeoutException): 102 | self.request_manager.sync_request(request, 2) 103 | 104 | self.assertEqual(0, len(self.request_manager.sync_wait_message_ids)) 105 | self.assertEqual(0, len(self.request_manager.sync_wait_message_responses)) 106 | 107 | def test_async_request(self): 108 | request = Request(destination_topic="/test") 109 | 110 | class TestResponseCallback(ResponseCallback): 111 | def __init__(self): 112 | super(TestResponseCallback, self).__init__() 113 | self.response = None 114 | 115 | def on_response(self, response): 116 | self.response = response 117 | 118 | callback = TestResponseCallback() 119 | self.assertIsNone(callback.response) 120 | 121 | self.request_manager.async_request(request, callback) 122 | 123 | self.assertEqual(0, len(self.request_manager.sync_wait_message_ids)) 124 | self.assertEqual(0, len(self.request_manager.sync_wait_message_responses)) 125 | self.assertTrue(request.message_id in self.request_manager.callback_map) 126 | 127 | response = Response(request=request) 128 | self.request_manager.on_response(response) 129 | 130 | self.assertIsNotNone(callback.response) 131 | self.assertEqual(request.message_id, callback.response.request_message_id) 132 | 133 | self.assertEqual(0, len(self.request_manager.sync_wait_message_ids)) 134 | self.assertEqual(0, len(self.request_manager.sync_wait_message_responses)) 135 | self.assertFalse(request.message_id in self.request_manager.callback_map) 136 | -------------------------------------------------------------------------------- /dxlclient/test/test_service.py: -------------------------------------------------------------------------------- 1 | """ Simple test service that sends back a generic Response or ErrorResponse. """ 2 | 3 | from __future__ import absolute_import 4 | from concurrent.futures import ThreadPoolExecutor 5 | import logging 6 | 7 | from dxlclient import ErrorResponse, Response 8 | from dxlclient.callbacks import RequestCallback 9 | 10 | # pylint: disable=missing-docstring 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class TestService(RequestCallback): 16 | 17 | # The Client 18 | _client = None 19 | # Whether to return standard responses or errors 20 | _return_error = False 21 | # Thread pool 22 | _executor = None 23 | # The error code to return 24 | _error_code = 99 25 | # The error message to return 26 | _error_message = "Error" 27 | 28 | def __init__(self, client, thread_count): 29 | super(TestService, self).__init__() 30 | self.m_client = client 31 | self.m_executor = ThreadPoolExecutor(max_workers=thread_count) 32 | 33 | @property 34 | def return_error(self): 35 | return self._return_error 36 | 37 | @return_error.setter 38 | def return_error(self, return_error): 39 | self._return_error = return_error 40 | 41 | @property 42 | def error_code(self): 43 | return self._error_code 44 | 45 | @error_code.setter 46 | def error_code(self, error_code): 47 | self._error_code = error_code 48 | 49 | @property 50 | def error_message(self): 51 | return self._error_message 52 | 53 | @error_message.setter 54 | def error_message(self, error_message): 55 | self._error_message = error_message 56 | 57 | def on_request(self, request): 58 | 59 | if self._return_error: 60 | response = ErrorResponse(request, error_code=self._error_code, error_message=self._error_message) 61 | else: 62 | response = Response(request) 63 | 64 | def run_task(): 65 | try: 66 | self.m_client.send_response(response) 67 | except Exception as ex: 68 | logging.info(ex) 69 | raise ex 70 | 71 | self.m_executor.submit(run_task) 72 | 73 | def close(self): 74 | self.m_executor.shutdown() 75 | -------------------------------------------------------------------------------- /dxlclient/test/thread_executor.py: -------------------------------------------------------------------------------- 1 | """ 2 | Class which executes the specified command a specified number of times, using a 3 | new thread for each execution. 4 | """ 5 | 6 | from __future__ import absolute_import 7 | from concurrent.futures import ThreadPoolExecutor 8 | 9 | # pylint: disable=missing-docstring 10 | 11 | 12 | class ThreadRunExecutor(object): 13 | 14 | futures = [] 15 | 16 | def __init__(self, run_count): 17 | self.executor = ThreadPoolExecutor(max_workers=run_count) 18 | self.run_count = run_count 19 | 20 | def execute(self, command): 21 | with self.executor as executor: 22 | for _ in range(0, self.run_count): 23 | self.futures.append(executor.submit(command)) 24 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendxl/opendxl-client-python/e6c1570900586bb8fe8f3ba334a32e67b72dd3d4/examples/__init__.py -------------------------------------------------------------------------------- /examples/advanced/event_publisher_sample.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import logging 4 | import os 5 | import sys 6 | 7 | from dxlclient.client import DxlClient 8 | from dxlclient.client_config import DxlClientConfig 9 | from dxlclient.message import Event 10 | 11 | # Import common logging and configuration 12 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 13 | from common import * 14 | 15 | # Configure local logger 16 | logger = logging.getLogger(__name__) 17 | 18 | # Sample topic to publish DXL Events on 19 | EVENT_TOPIC = "/isecg/sample/event" 20 | 21 | # Sample Event Publisher 22 | try: 23 | # Create DxlClientConfig from expected sample configuration file 24 | logger.info("Event Publisher - Load DXL config from: %s", CONFIG_FILE) 25 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 26 | 27 | # Initialize DXL client using our configuration 28 | logger.info("Event Publisher - Creating DXL Client") 29 | with DxlClient(config) as client: 30 | 31 | # Connect to DXL Broker 32 | logger.info("Event Publisher - Connecting to Broker") 33 | client.connect() 34 | 35 | # Prompt user for input to publish DXL Events 36 | while True: 37 | print(" Enter 1 to publish a DXL Event") 38 | print(" Enter 9 to quit") 39 | option = prompt(" Enter value: ").strip() 40 | 41 | # Option: DXL Event 42 | if option == "1": 43 | # Create the Event 44 | logger.info("Event Publisher - Creating Event for Topic %s", EVENT_TOPIC) 45 | event = Event(EVENT_TOPIC) 46 | 47 | # Encode string payload as UTF-8 48 | event.payload = "Sample Event Payload".encode() 49 | 50 | # Publish the Event to the DXL Fabric on the Topic 51 | logger.info("Event Publisher - Publishing Event to %s", EVENT_TOPIC) 52 | client.send_event(event) 53 | 54 | # Option: Exit the loop 55 | elif option == "9": 56 | break 57 | 58 | # Invalid input 59 | else: 60 | logger.info("Event Publisher - Invalid input: %s", option) 61 | # End Prompt Loop 62 | 63 | except Exception as e: 64 | logger.exception("Event Publisher - Exception") 65 | exit(1) 66 | -------------------------------------------------------------------------------- /examples/advanced/event_subscriber_sample.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import logging 4 | import os 5 | import sys 6 | 7 | from dxlclient.callbacks import EventCallback 8 | from dxlclient.client import DxlClient 9 | from dxlclient.client_config import DxlClientConfig 10 | 11 | # Import common logging and configuration 12 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 13 | from common import * 14 | 15 | # Configure local logger 16 | logger = logging.getLogger(__name__) 17 | 18 | # Sample topic to fire Events on 19 | EVENT_TOPIC = "/isecg/sample/event" 20 | 21 | # Sample Event Subscriber 22 | try: 23 | # Create DxlClientConfig from expected sample configuration file 24 | logger.info("Event Subscriber - Load DXL config from: %s", CONFIG_FILE) 25 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 26 | 27 | # Initialize DXL client using our configuration 28 | logger.info("Event Subscriber - Creating DXL Client") 29 | with DxlClient(config) as client: 30 | 31 | # Connect to DXL Broker 32 | logger.info("Event Subscriber - Connecting to Broker") 33 | client.connect() 34 | 35 | # Event callback class to handle incoming DXL Events 36 | class MyEventCallback(EventCallback): 37 | def on_event(self, event): 38 | # Extract information from Event payload, in this sample we 39 | # expect it is UTF-8 encoded 40 | logger.info("Event Subscriber - Event received:\n Topic: %s\n Payload: %s", 41 | event.destination_topic, event.payload.decode()) 42 | 43 | # Add Event callback to DXL client 44 | logger.info("Adding Event callback function to Topic: %s", EVENT_TOPIC) 45 | client.add_event_callback(EVENT_TOPIC, MyEventCallback()) 46 | 47 | # Wait for DXL Events 48 | while True: 49 | print(" Enter 9 to quit") 50 | option = prompt(" Enter value: ").strip() 51 | 52 | # Option: Exit the loop 53 | if option == "9": 54 | break 55 | 56 | # Invalid input 57 | else: 58 | logger.info("Event Subscriber - Invalid input: %s", option) 59 | 60 | except Exception as e: 61 | logger.exception("Event Subscriber - Exception") 62 | exit(1) 63 | -------------------------------------------------------------------------------- /examples/advanced/service_invoker_sample.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import logging 4 | import os 5 | import sys 6 | 7 | from dxlclient.callbacks import ResponseCallback 8 | from dxlclient.client import DxlClient 9 | from dxlclient.client_config import DxlClientConfig 10 | from dxlclient.message import Message, Request 11 | 12 | # Import common logging and configuration 13 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 14 | from common import * 15 | 16 | # Configure local logger 17 | logger = logging.getLogger(__name__) 18 | 19 | # Sample topic to fire Requests on 20 | SERVICE_TOPIC = "/isecg/sample/service" 21 | 22 | # Sample Synchronous Service Invoker 23 | try: 24 | # Create DxlClientConfig from expected sample configuration file 25 | logger.info("Service Invoker - Load DXL config from: %s", CONFIG_FILE) 26 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 27 | 28 | # Initialize DXL client using our configuration 29 | logger.info("Service Invoker - Creating DXL Client") 30 | with DxlClient(config) as client: 31 | 32 | # Connect to DXL Broker 33 | logger.info("Service Invoker - Connecting to Broker") 34 | client.connect() 35 | 36 | # Prompt user for input to publish DXL Requests 37 | while True: 38 | print(" Press 1 to send a Synchronous Request") 39 | print(" Press 2 to send an Asynchronous Request") 40 | print(" Press 9 to quit") 41 | option = prompt(" Enter value: ").strip() 42 | 43 | # Option: Synchronous DXL Request 44 | if option == "1": 45 | # Create the Request 46 | logger.info( 47 | "Service Invoker - Creating Synchronous Request for topic %s", 48 | SERVICE_TOPIC) 49 | request = Request(SERVICE_TOPIC) 50 | 51 | # Encode string payload as UTF-8 52 | request.payload = \ 53 | ("Sample Synchronous Request Payload - Request ID: " + 54 | str(request.message_id)).encode() 55 | 56 | # Send Synchronous Request with default timeout and wait for 57 | # Response 58 | logger.info( 59 | "Service Invoker - Sending Synchronous Request to %s", 60 | SERVICE_TOPIC) 61 | response = client.sync_request(request) 62 | 63 | # Check that the Response is not an Error Response, then extract 64 | if response.message_type != Message.MESSAGE_TYPE_ERROR: 65 | # Extract information from Response payload, in this sample 66 | # we expect it is UTF-8 encoded 67 | logger.info( 68 | "Service Invoker - Synchronous Response received:\n" + 69 | " Topic: %s\n Payload: %s", 70 | response.destination_topic, 71 | response.payload.decode()) 72 | else: 73 | logger.info( 74 | "Service Invoker - Synchronous Error Response received:\n" + 75 | " Topic: %s\n Error: %s", 76 | response.destination_topic, response.error_message) 77 | 78 | # Option: Asynchronous DXL Request 79 | elif option == "2": 80 | # Response callback class to handle DXL Responses from a Service 81 | # to our Asynchronous Requests 82 | class MyResponseCallback(ResponseCallback): 83 | def on_response(self, response): 84 | # Check that the Response is not an Error Response, then 85 | # extract 86 | if response.message_type != Message.MESSAGE_TYPE_ERROR: 87 | # Extract information from Response payload, in this 88 | # sample we expect it is UTF-8 encoded 89 | logger.info( 90 | "Service Invoker - " + 91 | "Asynchronous Response received:\n " + 92 | "Topic: %s\n Request ID: %s\n Payload: %s", 93 | response.destination_topic, 94 | response.request_message_id, 95 | response.payload.decode()) 96 | else: 97 | logger.info( 98 | "Service Invoker - " + 99 | "Asynchronous Error Response received:\n " + 100 | "Topic: %s\n Request ID: %s\n Error: %s", 101 | response.destination_topic, 102 | response.request_message_id, 103 | response.error_message) 104 | 105 | # Create the Request 106 | logger.info( 107 | "Service Invoker - " + 108 | "Creating Asynchronous Request for topic %s", SERVICE_TOPIC) 109 | request = Request(SERVICE_TOPIC) 110 | 111 | # Encode string payload as UTF-8 112 | request.payload = 'Sample Asynchronous Request Payload'.encode() 113 | 114 | #Send Asynchronous Request with a timeout of 5 seconds 115 | logger.info( 116 | "Service Invoker - Sending Asynchronous Request:\n " + 117 | "Request ID: %s\n Topic: %s", 118 | request.message_id, SERVICE_TOPIC) 119 | client.async_request(request, MyResponseCallback()) 120 | 121 | # Option: Exit the loop 122 | elif option == "9": 123 | break 124 | 125 | # Invalid input 126 | else: 127 | logger.info("Service Invoker - Invalid input: %s", option) 128 | 129 | except Exception as e: 130 | logger.exception("Service Invoker - Exception") 131 | exit(1) 132 | -------------------------------------------------------------------------------- /examples/advanced/service_provider_sample.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import logging 4 | import os 5 | import sys 6 | 7 | from dxlclient.callbacks import RequestCallback 8 | from dxlclient.client import DxlClient 9 | from dxlclient.client_config import DxlClientConfig 10 | from dxlclient.message import Response 11 | from dxlclient.service import ServiceRegistrationInfo 12 | 13 | # Import common logging and configuration 14 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 15 | from common import * 16 | 17 | # Configure local logger 18 | logger = logging.getLogger(__name__) 19 | 20 | # Sample topic to fire Requests on 21 | SERVICE_TOPIC = "/isecg/sample/service" 22 | 23 | # Sample Service Provider 24 | try: 25 | # Create DxlClientConfig from expected sample configuration file 26 | logger.info("Service Provider - Load DXL config from: %s", CONFIG_FILE) 27 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 28 | 29 | # Initialize DXL client using our configuration 30 | logger.info("Service Provider - Creating DXL Client") 31 | with DxlClient(config) as client: 32 | 33 | # Connect to DXL Broker 34 | logger.info("Service Provider - Connecting to Broker") 35 | client.connect() 36 | 37 | # Response callback class to handle DXL Responses to our Asynchronous Requests 38 | class MyRequestCallback(RequestCallback): 39 | def on_request(self, request): 40 | # Extract information from Response payload, in this sample we 41 | # expect it is UTF-8 encoded 42 | logger.info( 43 | "Service Provider - Request received:\n " + 44 | "Topic: %s\n Request ID: %s\n Payload: %s", 45 | request.destination_topic, 46 | request.message_id, 47 | request.payload.decode()) 48 | 49 | # Create the Response message 50 | logger.info( 51 | "Service Provider - Creating Response for Request ID %s on %s", 52 | request.message_id, request.destination_topic) 53 | response = Response(request) 54 | 55 | # Encode string payload as UTF-8 56 | response.payload = "Sample Response Payload".encode() 57 | 58 | # Send the Response back 59 | logger.info( 60 | "Service Provider - Sending Response to Request ID: %s on %s", 61 | response.request_message_id, request.destination_topic) 62 | client.send_response(response) 63 | 64 | # Create DXL Service Registration object 65 | service_registration_info = ServiceRegistrationInfo(client, "/mycompany/myservice") 66 | 67 | # Add a topic for the service to respond to 68 | service_registration_info.add_topic(SERVICE_TOPIC, MyRequestCallback()) 69 | 70 | # Register the service with the DXL fabric (with a wait up to 10 seconds 71 | # for registration to complete) 72 | logger.info("Registering service.") 73 | client.register_service_sync(service_registration_info, 10) 74 | 75 | # Wait for DXL Requests 76 | while True: 77 | print(" Enter 9 to quit") 78 | option = prompt(" Enter value: ").strip() 79 | 80 | # Option: Exit the loop 81 | if option == "9": 82 | break 83 | 84 | # Invalid input 85 | else: 86 | logger.info("Service Provider - Invalid input: %s", option) 87 | 88 | except Exception as e: 89 | logger.exception("Service Provider - Exception") 90 | exit(1) 91 | -------------------------------------------------------------------------------- /examples/basic/event_example.py: -------------------------------------------------------------------------------- 1 | # This sample demonstrates how to register a callback to receive Event messages 2 | # from the DXL fabric. Once the callback is registered, the sample sends a 3 | # set number of Event messages to the fabric and waits for them all to be 4 | # received by the callback. 5 | 6 | from __future__ import absolute_import 7 | from __future__ import print_function 8 | import logging 9 | import os 10 | import sys 11 | import time 12 | from threading import Condition 13 | 14 | from dxlclient.callbacks import EventCallback 15 | from dxlclient.client import DxlClient 16 | from dxlclient.client_config import DxlClientConfig 17 | from dxlclient.message import Event 18 | 19 | # Import common logging and configuration 20 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 21 | from common import * 22 | 23 | # Configure local logger 24 | logging.getLogger().setLevel(logging.ERROR) 25 | logger = logging.getLogger(__name__) 26 | 27 | # The topic to publish to 28 | EVENT_TOPIC = "/isecg/sample/basicevent" 29 | 30 | # The total number of events to send 31 | TOTAL_EVENTS = 1000 32 | 33 | # Condition/lock used to protect changes to counter 34 | event_count_condition = Condition() 35 | 36 | # The events received (use an array so we can modify in callback) 37 | event_count = [0] 38 | 39 | # Create DXL configuration from file 40 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 41 | 42 | # Create the client 43 | with DxlClient(config) as client: 44 | 45 | # Connect to the fabric 46 | client.connect() 47 | 48 | # 49 | # Register callback and subscribe 50 | # 51 | 52 | # Create and add event listener 53 | class MyEventCallback(EventCallback): 54 | def on_event(self, event): 55 | with event_count_condition: 56 | # Print the payload for the received event 57 | print("Received event: " + event.payload.decode()) 58 | # Increment the count 59 | event_count[0] += 1 60 | # Notify that the count was increment 61 | event_count_condition.notify_all() 62 | 63 | # Register the callback with the client 64 | client.add_event_callback(EVENT_TOPIC, MyEventCallback()) 65 | 66 | # 67 | # Send events 68 | # 69 | 70 | # Record the start time 71 | start = time.time() 72 | 73 | # Loop and send the events 74 | for event_id in range(TOTAL_EVENTS): 75 | # Create the event 76 | event = Event(EVENT_TOPIC) 77 | # Set the payload 78 | event.payload = str(event_id).encode() 79 | # Send the event 80 | client.send_event(event) 81 | 82 | # Wait until all events have been received 83 | print("Waiting for events to be received...") 84 | with event_count_condition: 85 | while event_count[0] < TOTAL_EVENTS: 86 | event_count_condition.wait() 87 | 88 | # Print the elapsed time 89 | print("Elapsed time (ms): " + str((time.time() - start) * 1000)) 90 | -------------------------------------------------------------------------------- /examples/basic/service_example.py: -------------------------------------------------------------------------------- 1 | # This sample demonstrates how to register a DXL service to receive Request 2 | # messages and send Response messages back to an invoking client. 3 | 4 | from __future__ import absolute_import 5 | from __future__ import print_function 6 | import logging 7 | import os 8 | import sys 9 | 10 | from dxlclient.callbacks import RequestCallback 11 | from dxlclient.client import DxlClient 12 | from dxlclient.client_config import DxlClientConfig 13 | from dxlclient.message import Message, Request, Response 14 | from dxlclient.service import ServiceRegistrationInfo 15 | 16 | # Import common logging and configuration 17 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 18 | from common import * 19 | 20 | # Configure local logger 21 | logging.getLogger().setLevel(logging.ERROR) 22 | logger = logging.getLogger(__name__) 23 | 24 | # The topic for the service to respond to 25 | SERVICE_TOPIC = "/isecg/sample/basicservice" 26 | 27 | # Create DXL configuration from file 28 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 29 | 30 | # Create the client 31 | with DxlClient(config) as client: 32 | 33 | # Connect to the fabric 34 | client.connect() 35 | 36 | # 37 | # Register the service 38 | # 39 | 40 | # Create incoming request callback 41 | class MyRequestCallback(RequestCallback): 42 | def on_request(self, request): 43 | # Extract information from request 44 | print("Service received request payload: " + request.payload.decode()) 45 | # Create the response message 46 | res = Response(request) 47 | # Populate the response payload 48 | res.payload = "pong".encode() 49 | # Send the response 50 | client.send_response(res) 51 | 52 | # Create service registration object 53 | info = ServiceRegistrationInfo(client, "myService") 54 | 55 | # Add a topic for the service to respond to 56 | info.add_topic(SERVICE_TOPIC, MyRequestCallback()) 57 | 58 | # Register the service with the fabric (wait up to 10 seconds for registration to complete) 59 | client.register_service_sync(info, 10) 60 | 61 | # 62 | # Invoke the service (send a request) 63 | # 64 | 65 | # Create the request message 66 | req = Request(SERVICE_TOPIC) 67 | 68 | # Populate the request payload 69 | req.payload = "ping".encode() 70 | 71 | # Send the request and wait for a response (synchronous) 72 | res = client.sync_request(req) 73 | 74 | # Extract information from the response (if an error did not occur) 75 | if res.message_type != Message.MESSAGE_TYPE_ERROR: 76 | print("Client received response payload: " + res.payload.decode()) 77 | -------------------------------------------------------------------------------- /examples/common.py: -------------------------------------------------------------------------------- 1 | """ 2 | Common definitions for the DXL Python SDK samples. 3 | 4 | This includes the defining the path to the configuration file used to initialize the DXL client 5 | in addition to setting up the logger appropriately. 6 | """ 7 | 8 | from __future__ import absolute_import 9 | import os 10 | import logging 11 | 12 | # Config file name. 13 | CONFIG_FILE_NAME = "dxlclient.config" 14 | CONFIG_FILE = os.path.dirname(os.path.abspath(__file__)) + "/" + CONFIG_FILE_NAME 15 | 16 | # Enable logging, this will also direct built-in DXL log messages. 17 | # See - https://docs.python.org/2/howto/logging-cookbook.html 18 | log_formatter = logging.Formatter('%(asctime)s %(name)s - %(levelname)s - %(message)s') 19 | 20 | console_handler = logging.StreamHandler() 21 | console_handler.setFormatter(log_formatter) 22 | 23 | logger = logging.getLogger() 24 | logger.addHandler(console_handler) 25 | logger.setLevel(logging.INFO) 26 | 27 | # pylint: disable=unused-import 28 | try: 29 | from builtins import input as prompt 30 | except ImportError: 31 | from __builtin__ import raw_input as prompt 32 | -------------------------------------------------------------------------------- /examples/dxlclient.config: -------------------------------------------------------------------------------- 1 | [Certs] 2 | BrokerCertChain= 3 | CertFile= 4 | PrivateKey= 5 | 6 | [Brokers] 7 | unique_broker_id_1=broker_id_1;broker_port_1;broker_hostname_1;broker_ip_1 8 | unique_broker_id_2=broker_id_2;broker_port_2;broker_hostname_2;broker_ip_2 9 | 10 | [BrokersWebSockets] 11 | unique_broker_id_1=broker_id_1;broker_websockets_port_1;broker_hostname_1;broker_ip_1 12 | unique_broker_id_2=broker_id_2;broker_websockets_port_2;broker_hostname_2;broker_ip_2 -------------------------------------------------------------------------------- /examples/mar/search_example.py: -------------------------------------------------------------------------------- 1 | # This sample queries McAfee Active Response for the IP addresses of hosts 2 | # that have an Active Response client installed. 3 | 4 | from __future__ import absolute_import 5 | from __future__ import print_function 6 | import os 7 | import sys 8 | import json 9 | import time 10 | 11 | from dxlclient.client import DxlClient 12 | from dxlclient.client_config import DxlClientConfig 13 | from dxlclient.message import Message, Request 14 | 15 | # Import common logging and configuration 16 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 17 | from common import * 18 | 19 | # Configure local logger 20 | logging.getLogger().setLevel(logging.ERROR) 21 | logger = logging.getLogger(__name__) 22 | 23 | # The topic to create a search 24 | CREATE_SEARCH_TOPIC = "/mcafee/mar/service/api/search" 25 | 26 | # Create DXL configuration from file 27 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 28 | 29 | 30 | def execute_mar_search_api(client, payload_dict): 31 | """ 32 | Executes a query against the MAR search api 33 | 34 | :param client: The DXL client 35 | :param payload_dict: The payload 36 | :return: A dictionary containing the results of the query 37 | """ 38 | # Create the request message 39 | req = Request(CREATE_SEARCH_TOPIC) 40 | # Set the payload 41 | req.payload = json.dumps(payload_dict).encode(encoding="UTF-8") 42 | 43 | # Display the request that is going to be sent 44 | print("Request:\n" + json.dumps(payload_dict, sort_keys=True, indent=4, separators=(',', ': '))) 45 | 46 | # Send the request and wait for a response (synchronous) 47 | res = client.sync_request(req, timeout=30) 48 | 49 | # Return a dictionary corresponding to the response payload 50 | if res.message_type != Message.MESSAGE_TYPE_ERROR: 51 | resp_dict = json.loads(res.payload.decode(encoding="UTF-8")) 52 | # Display the response 53 | print("Response:\n" + json.dumps(resp_dict, sort_keys=True, 54 | indent=4, separators=(',', ': '))) 55 | if "code" in resp_dict: 56 | code = resp_dict['code'] 57 | if code < 200 or code >= 300: 58 | if "body" in resp_dict and "applicationErrorList" in resp_dict["body"]: 59 | error = resp_dict["body"]["applicationErrorList"][0] 60 | raise Exception(error["message"] + ": " + str(error["code"])) 61 | if "body" in resp_dict: 62 | raise Exception(resp_dict["body"] + ": " + str(code)) 63 | raise Exception("Error: Received failure response code: " + str(code)) 64 | else: 65 | raise Exception("Error: unable to find response code") 66 | return resp_dict 67 | 68 | raise Exception("Error: " + res.error_message + " (" + str(res.error_code) + ")") 69 | 70 | # Create the client 71 | with DxlClient(config) as client: 72 | 73 | # Connect to the fabric 74 | client.connect() 75 | 76 | # Create the search 77 | response_dict = execute_mar_search_api( 78 | client, 79 | { 80 | "target": "/v1/simple", 81 | "method": "POST", 82 | "parameters": {}, 83 | "body": { 84 | "aggregated": True, 85 | "projections": [ 86 | { 87 | "name": "HostInfo", 88 | "outputs": ["ip_address"] 89 | } 90 | ] 91 | } 92 | } 93 | ) 94 | 95 | # Get the search identifier 96 | search_id = response_dict["body"]["id"] 97 | 98 | # Start the search 99 | execute_mar_search_api( 100 | client, 101 | { 102 | "target": "/v1/" + search_id + "/start", 103 | "method": "PUT", 104 | "parameters": {}, 105 | "body": {} 106 | } 107 | ) 108 | 109 | # Wait until the search finishes 110 | finished = False 111 | while not finished: 112 | response_dict = execute_mar_search_api( 113 | client, 114 | { 115 | "target": "/v1/" + search_id + "/status", 116 | "method": "GET", 117 | "parameters": {}, 118 | "body": {} 119 | } 120 | ) 121 | finished = response_dict["body"]["status"] == "FINISHED" 122 | if not finished: 123 | time.sleep(5) 124 | 125 | # Get the search results 126 | # Results limited to 10, the API does support paging 127 | response_dict = execute_mar_search_api( 128 | client, 129 | { 130 | "target": "/v1/" + search_id + "/results", 131 | "method": "GET", 132 | "parameters": { 133 | "$offset": 0, 134 | "$limit": 10, 135 | "filter": "", 136 | "sortBy": "count", 137 | "sortDirection": "desc" 138 | }, 139 | "body": {} 140 | } 141 | ) 142 | 143 | # Loop and display the results 144 | print("Results:") 145 | for result in response_dict['body']['items']: 146 | print(" " + result['output']['HostInfo|ip_address']) 147 | -------------------------------------------------------------------------------- /examples/servicewrapper/openweather_common.py: -------------------------------------------------------------------------------- 1 | # This sample demonstrates wrapping an existing service and exposing it on the 2 | # DXL fabric. 3 | # 4 | # In this particular case, the "openweathermap.org" current weather data is 5 | # exposed as a DXL service. This service wrapper delegates to the 6 | # OpenWeatherMap REST API. 7 | 8 | # The API key for invoking the weather service 9 | API_KEY = "" 10 | 11 | # The name of the OpenWeatherMap service 12 | SERVICE_NAME = "/openweathermap/service/openweathermap" 13 | 14 | # The "current weather" topic 15 | SERVICE_CURRENT_WEATHER_TOPIC = SERVICE_NAME + "/current" 16 | -------------------------------------------------------------------------------- /examples/servicewrapper/openweather_service_invoker.py: -------------------------------------------------------------------------------- 1 | # This sample demonstrates wrapping an existing service and exposing it on the 2 | # DXL fabric. 3 | # 4 | # In this particular case, the "openweathermap.org" current weather data is 5 | # exposed as a DXL service. This service wrapper delegates to the 6 | # OpenWeatherMap REST API. 7 | # 8 | # openweather_common.py must be edited to include the OpenWeatherMap API 9 | # key (see http://openweathermap.org/appid) 10 | 11 | from __future__ import absolute_import 12 | from __future__ import print_function 13 | import json 14 | import logging 15 | import os 16 | import sys 17 | 18 | from dxlclient.client import DxlClient 19 | from dxlclient.client_config import DxlClientConfig 20 | from dxlclient.message import Message, Request 21 | 22 | from openweather_common import * 23 | 24 | # Import common logging and configuration 25 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 26 | from common import * 27 | 28 | # Configure local logger 29 | logging.getLogger().setLevel(logging.INFO) 30 | logger = logging.getLogger(__name__) 31 | 32 | # Create DXL configuration from file 33 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 34 | 35 | # Create the client 36 | with DxlClient(config) as client: 37 | 38 | # Connect to the fabric 39 | client.connect() 40 | 41 | # 42 | # Invoke the service (send a request) 43 | # 44 | 45 | # Create the "Current Weather" request 46 | req = Request(SERVICE_CURRENT_WEATHER_TOPIC) 47 | # Populate the request payload 48 | # Examples include: 49 | # By ZIP code: zip=97140,us 50 | # By geographic coordinates: lat=35&lon=139 51 | # By city name: q=London,uk 52 | req.payload = "zip=97140,us".encode() 53 | 54 | # Send the request and wait for a response (synchronous) 55 | res = client.sync_request(req) 56 | 57 | # Extract information from the response (if an error did not occur) 58 | if res.message_type != Message.MESSAGE_TYPE_ERROR: 59 | response_dict = json.loads(res.payload.decode(encoding="UTF-8")) 60 | print("Client received response payload: \n" + \ 61 | json.dumps(response_dict, sort_keys=True, indent=4, separators=(',', ': '))) 62 | else: 63 | logger.error("Error: %s (%s)", res.error_message, res.error_code) 64 | -------------------------------------------------------------------------------- /examples/servicewrapper/openweather_service_wrapper.py: -------------------------------------------------------------------------------- 1 | # This sample demonstrates wrapping an existing service and exposing it on the 2 | # DXL fabric. 3 | # 4 | # In this particular case, the "openweathermap.org" current weather data is 5 | # exposed as a DXL service. This service wrapper delegates to the 6 | # OpenWeatherMap REST API. 7 | # 8 | # openweather_common.py must be edited to include the OpenWeatherMap API 9 | # key (see http://openweathermap.org/appid) 10 | 11 | from __future__ import absolute_import 12 | from __future__ import print_function 13 | import logging 14 | import os 15 | import sys 16 | import time 17 | 18 | try: 19 | from urllib.request import urlopen 20 | from urllib.request import Request as URLRequest 21 | except ImportError: 22 | from urllib2 import urlopen 23 | from urllib2 import Request as URLRequest 24 | 25 | from dxlclient.callbacks import RequestCallback 26 | from dxlclient.client import DxlClient 27 | from dxlclient.client_config import DxlClientConfig 28 | from dxlclient.message import ErrorResponse, Response 29 | from dxlclient.service import ServiceRegistrationInfo 30 | 31 | from openweather_common import * 32 | 33 | # Import common logging and configuration 34 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 35 | from common import * 36 | 37 | # Configure local logger 38 | logging.getLogger().setLevel(logging.INFO) 39 | logger = logging.getLogger(__name__) 40 | 41 | # The OpenWeatherMap "Current Weather" URL (see http://openweathermap.org/current) 42 | CURRENT_WEATHER_URL = "http://api.openweathermap.org/data/2.5/weather?{0}&APPID={1}" 43 | 44 | # Create DXL configuration from file 45 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 46 | 47 | # Create the client 48 | with DxlClient(config) as client: 49 | 50 | # Connect to the fabric 51 | client.connect() 52 | 53 | # 54 | # Register the service 55 | # 56 | 57 | # Create "Current Weather" incoming request callback 58 | class CurrentWeatherCallback(RequestCallback): 59 | def on_request(self, request): 60 | try: 61 | # Extract information from request 62 | query = request.payload.decode(encoding="UTF-8") 63 | logger.info("Service received request payload: " + query) 64 | 65 | # Send HTTP request to OpenWeatherMap 66 | req = URLRequest( 67 | CURRENT_WEATHER_URL.format(query, API_KEY), None, 68 | {'Content-Type': 'text/json'}) 69 | f = urlopen(req) 70 | weather_response = f.read() 71 | f.close() 72 | 73 | # Create the response message 74 | response = Response(request) 75 | # Populate the response payload 76 | response.payload = weather_response 77 | # Send the response 78 | client.send_response(response) 79 | 80 | except Exception as ex: 81 | print(str(ex)) 82 | # Send error response 83 | client.send_response(ErrorResponse( 84 | request, error_message=str(ex).encode(encoding="UTF-8"))) 85 | 86 | # Create service registration object 87 | info = ServiceRegistrationInfo(client, SERVICE_NAME) 88 | 89 | # Add a topic for the service to respond to 90 | info.add_topic(SERVICE_CURRENT_WEATHER_TOPIC, CurrentWeatherCallback()) 91 | 92 | # Register the service with the fabric (wait up to 10 seconds for registration to complete) 93 | client.register_service_sync(info, 10) 94 | 95 | logger.info("Weather service is running...") 96 | 97 | # Wait forever 98 | while True: 99 | time.sleep(60) 100 | -------------------------------------------------------------------------------- /examples/tie/file_rep_sample.py: -------------------------------------------------------------------------------- 1 | # This sample demonstrates invoking the McAfee Threat Intelligence Exchange 2 | # (TIE) DXL service to retrieve the reputation of files (as identified 3 | # by their hashes) 4 | 5 | from __future__ import absolute_import 6 | from __future__ import print_function 7 | import logging 8 | import os 9 | import sys 10 | import json 11 | import base64 12 | 13 | from dxlclient.client import DxlClient 14 | from dxlclient.client_config import DxlClientConfig 15 | from dxlclient.message import Message, Request 16 | 17 | # Import common logging and configuration 18 | sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") 19 | from common import * 20 | 21 | # Configure local logger 22 | logging.getLogger().setLevel(logging.ERROR) 23 | logger = logging.getLogger(__name__) 24 | 25 | # The topic for requesting file reputations 26 | FILE_REP_TOPIC = "/mcafee/service/tie/file/reputation" 27 | 28 | # Create DXL configuration from file 29 | config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) 30 | 31 | 32 | def base64_from_hex(hexstr): 33 | """ 34 | Returns the base64 string for the hex string specified 35 | 36 | :param hexstr: The hex string to convert to base64 37 | :return: The base64 value for the specified hes string 38 | """ 39 | return base64.b64encode(bytes(bytearray.fromhex(hexstr))).decode("utf8") 40 | 41 | 42 | def get_tie_file_reputation(client, md5_hex, sha1_hex): 43 | """ 44 | Returns a dictionary containing the results of a TIE file reputation request 45 | 46 | :param client: The DXL client 47 | :param md5_hex: The MD5 Hex string for the file 48 | :param sha1_hex: The SHA-1 Hex string for the file 49 | :return: A dictionary containing the results of a TIE file reputation request 50 | """ 51 | # Create the request message 52 | req = Request(FILE_REP_TOPIC) 53 | 54 | # Create a dictionary for the payload 55 | payload_dict = { 56 | "hashes": [ 57 | {"type": "md5", "value": base64_from_hex(md5_hex)}, 58 | {"type": "sha1", "value": base64_from_hex(sha1_hex)} 59 | ] 60 | } 61 | 62 | # Set the payload 63 | req.payload = json.dumps(payload_dict).encode() 64 | 65 | # Send the request and wait for a response (synchronous) 66 | res = client.sync_request(req) 67 | 68 | # Return a dictionary corresponding to the response payload 69 | if res.message_type != Message.MESSAGE_TYPE_ERROR: 70 | return json.loads(res.payload.decode(encoding="UTF-8")) 71 | raise Exception("Error: " + res.error_message + " (" + str(res.error_code) + ")") 72 | 73 | # Create the client 74 | with DxlClient(config) as client: 75 | 76 | # Connect to the fabric 77 | client.connect() 78 | 79 | # 80 | # Request and display reputation for notepad.exe 81 | # 82 | response_dict = get_tie_file_reputation(client=client, 83 | md5_hex="f2c7bb8acc97f92e987a2d4087d021b1", 84 | sha1_hex="7eb0139d2175739b3ccb0d1110067820be6abd29") 85 | print("Notepad.exe reputation:") 86 | print(json.dumps(response_dict, sort_keys=True, indent=4, separators=(',', ': ')) + "\n") 87 | 88 | # 89 | # Request and display reputation for EICAR 90 | # 91 | response_dict = get_tie_file_reputation(client=client, 92 | md5_hex="44d88612fea8a8f36de82e1278abb02f", 93 | sha1_hex="3395856ce81f2b7382dee72602f798b642f14140") 94 | print("EICAR reputation:") 95 | print(json.dumps(response_dict, sort_keys=True, indent=4, separators=(',', ': '))) 96 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | verbosity=2 3 | attr=!manual 4 | exclude=test_wildcard_performance 5 | exe=1 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ Setup script for the dxlclient package """ 2 | 3 | # pylint: disable=no-member, no-name-in-module, import-error, wrong-import-order 4 | # pylint: disable=missing-docstring, no-self-use 5 | 6 | from __future__ import absolute_import 7 | import glob 8 | import os 9 | from setuptools import Command, setup 10 | import setuptools.command.sdist 11 | import distutils.command.sdist 12 | import distutils.log 13 | import subprocess 14 | 15 | 16 | # Patch setuptools' sdist behaviour with distutils' sdist behaviour 17 | setuptools.command.sdist.sdist.run = distutils.command.sdist.sdist.run 18 | 19 | PRODUCT_PROPS = {} 20 | CWD = os.path.abspath(os.path.dirname(__file__)) 21 | with open(os.path.join(CWD, "dxlclient", "_product_props.py")) as f: 22 | exec(f.read(), PRODUCT_PROPS) # pylint: disable=exec-used 23 | 24 | class LintCommand(Command): 25 | """ 26 | Custom setuptools command for running lint 27 | """ 28 | description = 'run lint against project source files' 29 | user_options = [] 30 | def initialize_options(self): 31 | pass 32 | def finalize_options(self): 33 | pass 34 | def run(self): 35 | self.announce("Running pylint for library source files and tests", 36 | level=distutils.log.INFO) 37 | subprocess.check_call(["pylint", "dxlclient"] + glob.glob("*.py")) 38 | self.announce("Running pylint for examples", level=distutils.log.INFO) 39 | subprocess.check_call(["pylint"] + glob.glob("examples/*.py") + 40 | glob.glob("examples/**/*.py") + 41 | ["--rcfile", ".pylintrc.examples"]) 42 | 43 | class CiCommand(Command): 44 | """ 45 | Custom setuptools command for running steps that are performed during 46 | Continuous Integration testing. 47 | """ 48 | description = 'run CI steps (lint, test, etc.)' 49 | user_options = [] 50 | def initialize_options(self): 51 | pass 52 | def finalize_options(self): 53 | pass 54 | def run(self): 55 | self.run_command("lint") 56 | self.run_command("test") 57 | 58 | TEST_REQUIREMENTS = [ 59 | 'futures; python_version == "3.7"', 60 | "mock", 61 | "nose", 62 | "parameterized", 63 | 'astroid<2.3.0; python_version == "3.7"', 64 | 'astroid==2.3.3; python_version > "3.7"', 65 | "pylint<=2.3.1", 66 | "requests-mock" 67 | ] 68 | 69 | DEV_REQUIREMENTS = TEST_REQUIREMENTS + ["sphinx"] 70 | 71 | setup( 72 | # Application name: 73 | name="dxlclient", 74 | 75 | # Version number: 76 | version=PRODUCT_PROPS["__version__"], 77 | 78 | # Application author details: 79 | author="McAfee, Inc.", 80 | 81 | # License 82 | license="Apache License 2.0", 83 | 84 | keywords=['opendxl', 'dxl', 'mcafee', 'client'], 85 | 86 | # Custom Paho MQTT Python client with proxy support added as a git submodule 87 | package_dir={ 88 | 'pahoproxy': 'paho_mqtt_dxl/src/paho/mqtt', 89 | 'oscrypto': 'oscrypto/oscrypto' 90 | }, 91 | 92 | # Packages 93 | packages=[ 94 | "dxlclient", 95 | "dxlclient._cli", 96 | "pahoproxy", 97 | "oscrypto", 98 | "oscrypto._openssl", 99 | "oscrypto._mac", 100 | "oscrypto._win", 101 | "oscrypto._linux_bsd" 102 | ], 103 | 104 | # Include additional files into the package 105 | include_package_data=True, 106 | 107 | install_requires=[ 108 | "asn1crypto", 109 | "configobj", 110 | "msgpack>=0.5,<1.0.0", 111 | "requests", 112 | "PySocks" 113 | ], 114 | 115 | tests_require=TEST_REQUIREMENTS, 116 | 117 | extras_require={ 118 | "dev": DEV_REQUIREMENTS, 119 | "test": TEST_REQUIREMENTS 120 | }, 121 | 122 | test_suite="nose.collector", 123 | 124 | # Details 125 | url="http://www.mcafee.com/", 126 | 127 | description="McAfee Data Exchange Layer Client", 128 | 129 | long_description=open('README').read(), 130 | 131 | python_requires='>=3.7', 132 | 133 | classifiers=[ 134 | "Topic :: Software Development :: Libraries :: Python Modules", 135 | "License :: OSI Approved :: Apache Software License", 136 | "Programming Language :: Python :: 3.7", 137 | "Programming Language :: Python :: 3.8", 138 | "Programming Language :: Python :: 3.9", 139 | ], 140 | 141 | cmdclass={ 142 | 'ci': CiCommand, 143 | 'lint': LintCommand 144 | }, 145 | 146 | entry_points={ 147 | 'console_scripts': [ 148 | 'dxlclient = dxlclient._cli:cli_run' 149 | ], 150 | } 151 | ) 152 | --------------------------------------------------------------------------------