├── .github ├── CODEOWNERS ├── commitlintrc.yml ├── renovate.json └── workflows │ ├── commitlint.yml │ ├── linters.yml │ ├── opi-lab.yml │ ├── poc-integration-skip.yml │ └── poc-integration.yml ├── .gitignore ├── .golangci.yml ├── .pre-commit-config.yaml ├── LICENSE ├── Meeting.md ├── README.md ├── demos ├── README.md ├── network │ ├── Presentation-opi-api.pptx │ ├── README.md │ └── opi-api-demo-pr128.pdf ├── redfish │ └── README.md ├── security │ ├── README.md │ ├── intel │ │ └── README.md │ ├── marvell │ │ └── README.md │ └── nvidia │ │ ├── README.md │ │ ├── RESTasV3.py │ │ ├── Statistics.py │ │ ├── cyperf-ipsec-config.zip │ │ ├── deployment │ │ ├── .env │ │ ├── conf │ │ │ ├── k.swanctl.conf │ │ │ └── vici.conf │ │ └── cyperf_with_ipsec.yml │ │ ├── opi-ipsec-demo-1.rxf │ │ ├── presentation-opi-api.pptx │ │ ├── requirements.txt │ │ └── test_ipsec.py ├── storage │ ├── README.md │ ├── intel │ │ └── README.md │ ├── marvell │ │ └── README.md │ └── nvidia │ │ └── README.md ├── testbed.py └── tgen │ ├── README.md │ ├── deployment │ ├── .env │ ├── conf │ │ └── client │ │ │ ├── k.swanctl.conf │ │ │ └── vici.conf │ ├── ipsec.yml │ └── tgen.yml │ ├── images │ ├── opi-lab-cabling-reduced.drawio.svg │ ├── opi-lab-cabling.drawio.svg │ ├── opi-lab-host2host.drawio.svg │ ├── opi-lab-scenarios-ipsec.drawio.svg │ ├── opi-lab-scenarios-with-dpu.drawio.svg │ ├── opi-lab-scenarios.drawio.svg │ ├── opi-lab-server2host.drawio.svg │ └── opi-lab-server2server.drawio.svg │ ├── requirements.txt │ └── test_tgen_demo.py ├── integration ├── .env ├── README.md ├── docker-compose.host.yml ├── docker-compose.networks.yml ├── docker-compose.otel.yml ├── docker-compose.pxe.yml ├── docker-compose.spdk.yml ├── docker-compose.xpu.yml ├── gateway │ ├── Dockerfile │ ├── Dockerfile.network │ ├── Dockerfile.storage │ ├── README.md │ ├── errors.grpc_conf │ └── grpc_gateway.conf ├── host-cpu │ ├── Dockerfile │ ├── Dockerfile.telegraf │ ├── README.md │ └── telegraf-redfish.conf ├── otel │ ├── README.md │ ├── otel-collector-config.yaml │ └── prometheus.yaml ├── scripts │ ├── deploy.sh │ └── integration.sh ├── strongswan │ ├── server │ │ ├── eap.conf │ │ ├── ecdsa │ │ │ └── serverKey.pem │ │ ├── psk.conf │ │ ├── rw.conf │ │ ├── swanctl.conf │ │ ├── x509 │ │ │ └── serverCert.pem │ │ └── x509ca │ │ │ └── caCert.pem │ └── strongswan.conf ├── sztp │ ├── dhclient.conf │ ├── dhcpd.conf.template │ └── my-boot-image.img ├── xPU-Integration-Blocks.png └── xpu-cpu │ ├── Dockerfile │ ├── Dockerfile.telegraf │ ├── README.md │ └── telegraf-redfish.conf ├── lab └── README.md ├── openoffload ├── README.md └── host_xpu.png ├── security └── README.md ├── setup ├── README.md ├── debian.yml ├── redhat.yml ├── setup.yml └── suse.yml └── storage └── README.md /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @opiproject/opi-maintainers 2 | -------------------------------------------------------------------------------- /.github/commitlintrc.yml: -------------------------------------------------------------------------------- 1 | extends: 2 | - "@commitlint/config-conventional" 3 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base", 5 | ":enablePreCommit" 6 | ], 7 | "schedule": [ 8 | "every weekend" 9 | ], 10 | "packageRules": [ 11 | { 12 | "matchUpdateTypes": ["digest"], 13 | "automerge": true 14 | }, 15 | { 16 | "matchPackagePatterns": ["opi-sztp-*"], 17 | "groupName": "OPI sztp dockers" 18 | } 19 | ], 20 | "regexManagers": [ 21 | { 22 | "fileMatch": ["^Dockerfile.network$"], 23 | "matchStrings": ["ARG TAG=(?.*?)\\n"], 24 | "datasourceTemplate": "github-tags", 25 | "depNameTemplate": "grpc/grpc", 26 | "versioningTemplate": "loose" 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | name: Commitlint 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main ] 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | call: 15 | uses: opiproject/actions/.github/workflows/commitlint.yml@main 16 | secrets: inherit 17 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: Linters 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main ] 9 | 10 | jobs: 11 | markdown-lint: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | - name: Run tests 16 | run: docker run -v $PWD:/workdir ghcr.io/igorshubovych/markdownlint-cli:latest --ignore=minutes --disable=MD013 "**/*.md" 17 | 18 | # - uses: avto-dev/markdown-lint@v1.5.0 19 | # with: 20 | # args: './*.md' 21 | 22 | docker-lint: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v4 26 | - uses: hadolint/hadolint-action@v3.1.0 27 | with: 28 | recursive: true 29 | ignore: DL3041 30 | 31 | shellcheck: 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v4 35 | - uses: azohra/shell-linter@v0.6.0 36 | 37 | YAMLlint: 38 | runs-on: ubuntu-latest 39 | steps: 40 | - uses: actions/checkout@v4 41 | - uses: ibiqlik/action-yamllint@v3 42 | with: 43 | file_or_dir: integration setup 44 | config_data: "{extends: default, rules: {line-length: disable}}" 45 | 46 | golangci: 47 | runs-on: ubuntu-latest 48 | steps: 49 | - uses: actions/checkout@v4 50 | - uses: actions/setup-go@v5 51 | -------------------------------------------------------------------------------- /.github/workflows/opi-lab.yml: -------------------------------------------------------------------------------- 1 | name: opi-lab-regression 2 | on: workflow_dispatch 3 | jobs: 4 | opi-tgen: 5 | runs-on: self-hosted 6 | steps: 7 | - uses: actions/checkout@v1 8 | - name: pytest-run0 9 | run: id 10 | - name: pytest-run1 11 | run: ls 12 | - name: pytest-run2 13 | run: pwd 14 | - name: pytest-run3 15 | run: pip install -r ./demos/tgen/requirements.txt 16 | - name: pytest-run4 17 | run: pytest -s ./tgen/test_tgen_demo.py 18 | working-directory: ./demos 19 | env: 20 | PYTHONPATH: ${PYTHONPATH}:${{ github.workspace }}/demos 21 | 22 | opi-ipsec: 23 | runs-on: self-hosted 24 | steps: 25 | - uses: actions/checkout@v1 26 | - name: Install python requirements 27 | run: pip install -r ./demos/security/nvidia/requirements.txt 28 | - name: Run CyPerf ipsec test automation 29 | run: pytest -s ./demos/security/nvidia/test_ipsec.py 30 | 31 | opi-ansible: 32 | runs-on: self-hosted 33 | steps: 34 | - name: nvidia bf-2 fw update using ansible 35 | run: | 36 | docker run --rm --pull=always \ 37 | ghcr.io/opiproject/ansible-opi-dpu:main \ 38 | all \ 39 | --module-name include_role \ 40 | --args name=bmc_fw_update \ 41 | -vvv -i "172.22.4.2," \ 42 | -e dpu_bmc_username='root' \ 43 | -e dpu_bmc_password='NvidiaBf2#Pass' \ 44 | -e bmc_fw_update_inventory_name='38a78c0e_bmc_firmware' \ 45 | -e bmc_fw_update_image_file='/tmp/bf2-bmc-ota-24.04-5-opn.tar' 46 | -------------------------------------------------------------------------------- /.github/workflows/poc-integration-skip.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'OPI Integration CI' 3 | 4 | on: 5 | push: 6 | branches: [ main ] 7 | paths-ignore: 8 | - 'integration/**' 9 | - '.github/workflows/poc-integration.yml' 10 | pull_request: 11 | branches: [ main ] 12 | paths-ignore: 13 | - 'integration/**' 14 | - '.github/workflows/poc-integration.yml' 15 | 16 | concurrency: 17 | # if workflow for PR or push is already running stop it, and start new one 18 | group: poc-integartion-skip-${{ github.ref }} 19 | cancel-in-progress: true 20 | 21 | jobs: 22 | run_integration_poc: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - run: 'echo "No integration test run required"' 26 | -------------------------------------------------------------------------------- /.github/workflows/poc-integration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'OPI Integration CI' 3 | 4 | on: 5 | workflow_dispatch: 6 | push: 7 | branches: [ main ] 8 | paths: 9 | - 'integration/**' 10 | - '.github/workflows/poc-integration.yml' 11 | pull_request: 12 | branches: [ main ] 13 | paths: 14 | - 'integration/**' 15 | - '.github/workflows/poc-integration.yml' 16 | 17 | concurrency: 18 | # if workflow for PR or push is already running stop it, and start new one 19 | group: poc-integration-${{ github.ref }} 20 | cancel-in-progress: true 21 | 22 | jobs: 23 | run_integration_poc: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: configure HUGE pages 27 | run: | 28 | sync 29 | echo 1 | sudo tee /proc/sys/vm/drop_caches 30 | sudo mkdir -p /mnt/huge 31 | grep hugetlbfs /proc/mounts || sudo mount -t hugetlbfs nodev /mnt/huge 32 | echo 1024 | sudo tee /proc/sys/vm/nr_hugepages 33 | echo "Check and fail if not enough" 34 | grep 1024 /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages 35 | 36 | - name: Checkout repository 37 | uses: actions/checkout@v4 38 | 39 | - name: Set up QEMU 40 | uses: docker/setup-qemu-action@v3 41 | 42 | - name: Set up Docker Buildx 43 | uses: docker/setup-buildx-action@v3 44 | 45 | - name: Start containers 46 | run: ./scripts/integration.sh start 47 | working-directory: integration 48 | 49 | - name: Run Integration Tests 50 | run: ./scripts/integration.sh tests 51 | working-directory: integration 52 | 53 | - name: Logs 54 | if: failure() 55 | run: ./scripts/integration.sh logs 56 | working-directory: integration 57 | 58 | - name: Stop containers 59 | if: always() 60 | run: ./scripts/integration.sh stop 61 | working-directory: integration 62 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | setup/inventory 2 | **/.idea 3 | **/*.iml 4 | *.pyc 5 | demos/tgen/passwords.py 6 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # This file contains all available configuration options 2 | # with their default values. 3 | 4 | # options for analysis running 5 | run: 6 | 7 | # output configuration options 8 | output: 9 | # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" 10 | format: colored-line-number 11 | 12 | linters-settings: 13 | funlen: 14 | lines: 120 15 | statements: 120 16 | dupl: 17 | threshold: 200 18 | gocognit: 19 | min-complexity: 32 20 | 21 | linters: 22 | enable: 23 | - bodyclose 24 | - depguard 25 | - dogsled 26 | - errcheck 27 | - exportloopref 28 | - funlen 29 | - gochecknoinits 30 | - gocognit 31 | - goconst 32 | - gocritic 33 | - gocyclo 34 | - gofmt 35 | - goimports 36 | - gosec 37 | - gosimple 38 | - govet 39 | - ineffassign 40 | - misspell 41 | - nakedret 42 | - prealloc 43 | - revive 44 | - staticcheck 45 | - stylecheck 46 | - unconvert 47 | - unparam 48 | - whitespace 49 | 50 | issues: 51 | # Independently from option `exclude` we use default exclude patterns, 52 | # it can be disabled by this option. To list all 53 | # excluded by default patterns execute `golangci-lint run --help`. 54 | # Default value for this option is true. 55 | exclude-use-default: false 56 | 57 | # Maximum issues count per one linter. Set to 0 to disable. Default is 50. 58 | max-issues-per-linter: 0 59 | 60 | # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. 61 | max-same-issues: 0 -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.5.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: check-yaml 10 | - id: check-added-large-files 11 | - repo: https://github.com/renovatebot/pre-commit-hooks 12 | rev: 37.131.0 13 | hooks: 14 | - id: renovate-config-validator 15 | - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook 16 | rev: v9.11.0 17 | hooks: 18 | - id: commitlint 19 | stages: [commit-msg] 20 | additional_dependencies: ['@commitlint/config-conventional'] 21 | - repo: https://github.com/golangci/golangci-lint 22 | rev: v1.55.2 23 | hooks: 24 | - id: golangci-lint 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Meeting.md: -------------------------------------------------------------------------------- 1 | # Platform/PoC/Reference Architecture Meeting Minutes 2 | 3 | ## 10/5/2021 4 | 5 | Talk about goals: Define a PoC 6 | 7 | PoC ideas: 8 | 9 | * Nginx already runs on BF2 (but slowly) 10 | * Nginx on Phantom Lake? 11 | * Mt Evans, Oak Springs Canyon complex, likely not MVP 12 | 13 | Actions: 14 | 15 | * Set up meeting with Marvell: Octeon <- Kris 16 | * Talk to Intel Phantom Lake people about Diamond Bluff <- Tim 17 | 18 | Definition of MVP: 19 | 20 | * OS - free 21 | * Free app in container: eg Squid Proxy 22 | * Ability to swap out container 23 | * Freely downloadable 24 | * Blog or other publishing 25 | 26 | ## 11/1/2021 27 | 28 | Agenda: 29 | 30 | * Refresh on previous discussion 31 | * Follow up on card availability: Tim and Kris 32 | * F5 has Intel Phantom Lake cards incoming 33 | * Nginx dpdk support provided by vendor (Silicom?) 34 | * Intel Big Springs Canyon suggested, available now, Xeon, with IPDK 35 | * FPGA works out of the box 36 | * Runtime programming all open source, P4 compiler backend closed 37 | * ACTION: Tom to investigate hardware 38 | * ACTION: Tim to verify nginx dpdk support can be made available 39 | * Intel Mt Evans early access possible as well 40 | * Requires 5.10 and 5.14 for the two cores 41 | * Get input from new members 42 | * Discuss direction and next steps 43 | * ACTION: Steve to create github repo for PoC working group 44 | * OS Selection discussion 45 | * RHEL with dev sub for initial PoC 46 | * Figure out OS strategy later 47 | * Meeting cadence and date/time 48 | * Weekly Wed 1-3pm slot 49 | 50 | ## 11/12/2021 51 | 52 | * Dan: Building cluster 53 | * Big Springs Canyon on Broadwell with 2x25GbE 54 | * Mt Evans on Sapphire Rapids with 2x100GbE 55 | * What OS to deploy? Start with RHEL 8.5, but Red Hat to investigate OCP 56 | options. 57 | 58 | ## 12/1/2021 59 | 60 | * Introductions to Kyle Mestery and Maxime Coquelin 61 | * Brief review of the goals of the working group and prior discussions 62 | 63 | ## 12/15/2021 64 | 65 | Merging with the dev platform group 66 | 67 | * OS requirements on device 68 | * Looked at Ubuntu, Fedora, CentOS Stream, and RHEL 69 | * CentOS Stream 9 with help from the community sounding possible 70 | * [CentOS SIG](https://wiki.centos.org/SpecialInterestGroup) 71 | * Canceling 12/22/2021 meeting 72 | 73 | ## 1/5/2022 74 | 75 | Light discussion today 76 | 77 | * Brief discussion about the Dell presentation, touching on SONiC and whether 78 | that applies to DB or not (not general purpose OS) 79 | * Kyle mentioned attempting to get IPDK environment working, will present the 80 | process next week. 81 | 82 | ## 1/12/2022 83 | 84 | * CentOS SIG discussion 85 | * [Meeting recording](https://drive.google.com/file/d/1gPSXwK7xHvFcfWUL3vay7zEaMkjBJQPU/view?usp=sharing) 86 | 87 | ## 1/19/2022 88 | 89 | * CentOS SIG discussion continuation 90 | 91 | ## 1/26/2022 92 | 93 | Dan Daly and Tim Worsley attended 94 | 95 | * Dan update on lab: all parts are in 96 | * Putting CentOS 8.2 on Mt Evans 97 | * Putting CentOS 7.x? on Big Springs Canyon 98 | * Tim update on PoC workload 99 | * Simple learning switch based on nginx 100 | * Pure software, no acceleration 101 | * How do we generate test load in the Intel lab? 102 | 103 | For next session: 104 | 105 | * Expecting to not have made progress with hardware, so focus on the test 106 | * Create document describing scenarios, test network, traffic generation, 107 | traffic targets 108 | * Goal is to be able to limit test the solution, which F5 hasn’t done yet 109 | * Also look into open source tools or languages for describing the test, so 110 | diamond bluff doesn’t have to invent something. Example would be the open 111 | ddos (proposed) standard 112 | 113 | ## 2/1/2022 114 | 115 | Attendees: Tim Worsley, Tom Rix, Mark Sanders, Steven Royer 116 | 117 | * Discussion on virtual development environment: 118 | * Similar to the ipdk virtual environment 119 | * Need volunteers to define and implement virtual development environment 120 | * Tom Rix volunteered! 121 | * Prefer to start with arm64 122 | * Start with RHEL for developer (free) 123 | * OS Image requirements 124 | * DPDK 125 | * docker/podman 126 | * App requirements 127 | * Looking to start with open source nginx in a container 128 | * Use pure CPU cores initially 129 | * Tim Worsley has agreed to provide this once a platform exists 130 | * Deliveries: 131 | * General agreement that DPUs aren’t build systems 132 | * Go into github: recipes, etc 133 | * Artifactory??? For container images: Prefer quay.io. 134 | 135 | ## 2/9/2022 136 | 137 | Attendees: Tim Worsley, Steven Royer 138 | 139 | * PoC idea 140 | * Open source nginx with modules like modsec in a container 141 | * Generate data streams where at least one is “malicious” 142 | * Container for generating traffic 143 | * Container as traffic target 144 | 145 | ## 2/16/2022 146 | 147 | Attendees: Kyle Mestery, Maxime Coquelin, Mark Sanders, Michal Kalderon, Laura 148 | JH, Steven Royer 149 | 150 | * Review 151 | * Michal Kalderon to look at providing access to Marvell hardware 152 | * Artifacts: look at integrating with github actions + github docker registry 153 | 154 | ## 2/23/2022 155 | 156 | Attendees: Kyle Mestery, Maxime Coquelin, Mark Sanders, Tom Rix, Dan Daly, 157 | Steven Royer, Michal Kalderon 158 | 159 | * Dev environment: 160 | * Provide VM image with some OS (CentOS Stream 9, consider future Debian 11) 161 | * Tom to talk to Steve about this 162 | * Steve to find out if Red Hat could host some vendor hardware for Diamond Bluff 163 | * Michal to see if something like 164 | [RH and NVIDIA](https://www.redhat.com/en/blog/optimizing-server-utilization-datacenters-offloading-network-functions-nvidia-bluefield-2-dpus) 165 | can be done on Marvell hardware 166 | 167 | ## 3/2/2022 168 | 169 | Attendees: Tim Worsely, Michal Kalderon, Laura JH, Steven Royer 170 | 171 | * Dev environment follow up: concerns around will adapter vendors support CentOS 172 | Stream 9 and Debian 11 (drivers etc…) 173 | * Marvell: yes, drivers are upstream 174 | * Intel: ??? no attendees 175 | * Hosting vendor hardware by Red Hat: 176 | * Answer is no, the plan should be to work this through the foundation when 177 | that is set up. 178 | * Action: 179 | * Tim: put up PR for current PoC plan 180 | 181 | ## 3/9/2022 182 | 183 | Attendees: Steven Royer, Ted Streete, Michal Kalderon, Mark Sanders, Dan Daly 184 | 185 | * Ted leaving the subgroup to focus on other areas 186 | * Tim Worsely out sick 187 | * Kyle on vacation 188 | * Actions: 189 | * Steve: start OS discussion in slack 190 | * Steve: update working group description 191 | * Steve: migrate minutes to github 192 | * Dan: create document about hardware hosting requirements/options 193 | * Dan: hardware update 194 | * Big Springs Canyon and Mt Evans in place 195 | * Access control in place 196 | * Targeting two weeks for schedule 197 | 198 | ## 3/23/2022 199 | 200 | Attendees: Steven Royer, Gene Bagwell, Shafiq Abedin, Tim Worsley, Michal 201 | Kalderon, Lionel P 202 | 203 | * Follow-ups: 204 | * Steve: Started OS discussion [in slack](https://opi-project.slack.com/archives/C033E418VCK/p1648048161645019) 205 | * Steve: Update working group description: in progress 206 | * Steve: Migrate minutes to [github](https://github.com/Diamond-Bluff/dbluff-poc/pull/1) 207 | * Tim: Still working on providing firewall PoC documentation 208 | * What's the difference between this group and the use cases group? 209 | * Uses cases group to define use cases 210 | * PoC group to implement them 211 | * New actions: 212 | * Tim to provide Dockerfiles for firewall PoC application and tests 213 | * Tim to publish containers to dockerhub or equivalent 214 | * Michal to investigate defining VM emulating DPU 215 | * Steve to add people to the github org for access to private repos 216 | * Task list to be managed via github issues in the poc repo 217 | * Suggestion to move to meeting every other week instead of weekly 218 | 219 | ## 3/30/2022 220 | 221 | Attendees: Steven Royer, Venkat Pullela, Lionel P, Laura JH, Michal Kalderon, 222 | Shafiq Abedin, Mark Sanders 223 | 224 | * Meeting cadence 225 | * Moving to every other week meetings 226 | * Meeting April 6 227 | * [Review](https://github.com/Diamond-Bluff/dbluff-poc/pull/2) 228 | 229 | ## 4/6/2022 230 | 231 | Attendees: Steven Royer, Venkat Pullela, Shafiq Abedin, Michal Kaderon, Mark 232 | Sanders, Harry Quackenboss 233 | 234 | * Venkat volunteers to be involved in setting up open infrastructure, open 235 | testing that keeps high performance in mind for PoCs 236 | * What additional PoCs should we be thinking about? 237 | * Layering of function: network, storage, ai, etc implies an ordering 238 | * Once pure software firewall is done, look at accelerating it 239 | * Look into storage PoC 240 | * Shafiq volunteers to work on a storage PoC 241 | * There was a brief discussion where interest was expressed on hearing more 242 | about the IPDK container environment to use as a starting point for us 243 | * Ask Kyle for IPDK container presentation at next meeting 244 | * Reminder meetings are now every other week, next meeting is 4/20/2022 245 | 246 | ## 4/20/2022 247 | 248 | Attendees: Steven Royer, Kyle Mestery, Mark Sanders, Dan Daly, Tim Worsley, 249 | Venkat Pullela 250 | 251 | * Tim shared his near term plans on the software firewall: 252 | * PRs for containers over the next weeks 253 | * Kyle shared demo of the new p4-eBPF IPDK container environment 254 | * [IPDK P4-eBPF](https://github.com/ipdk-io/ipdk/tree/main/build/networking_ebpf) 255 | * Should be able to clone and replicate easily 256 | * Should be able to build a version of Tim's software firewall on top of this 257 | container environment 258 | * p4-DPDK is currently limited in ways that it is inconvenient to use 259 | * Dan to work with the team to improve this 260 | 261 | ## 5/4/2022 262 | 263 | Attendees: Steven Royer, Venkat Pullela, Kyle Mestery, Shafiq Abedin 264 | 265 | * Work with Mark Sanders and the API group on basic 2 node PoC 266 | * Can API group build on top of the 267 | [networking](https://github.com/opiproject/opi-poc/tree/main/networking) 268 | PoC? 269 | * Can still layer Tim's nginx firewall on top of existing PoC 270 | 271 | ## 6/15/2022 272 | 273 | Attendees: Yuval Caduri, Venkat Pullela, Tim Worsley, Dan Daly, Steven Royer 274 | 275 | * Refresh and welcome to Yuval! 276 | * Dan brought up concerns that it's too early to invest much in PoC before the 277 | other subgroups have produced much 278 | * General agreements, but there is still value in laying the framework. 279 | * Firewall PoC 280 | * Tim to update existing networking PoC to include his firewall bits 281 | 282 | ## 6/29/2022 283 | 284 | Attendees: Steven Royer, Dan Daly, Kyle Mestery, Tim Worsley, Anh Thu, Venkat 285 | Pullela 286 | 287 | * Talk about OS again 288 | * Device OS development can occur on any OS, but there needs to be a 289 | "supported" option, e.g. Red Hat 290 | * Hope is that applications can be developed in containers with base image of 291 | their choice. TBD on how libraries/APIs work across specific base image and 292 | device OS types. 293 | * New organization discussion 294 | * Brief discussion follow-on to main meeting topic related to expanding the 295 | role of this group to include hosting vendor code. TBD 296 | * Firewall 297 | * Tim to replace nginx in existing PoC 298 | * Dan: Testing discussion 299 | * To post slides around CI/CD: slack and github 300 | * Venkat to bring Keysight people to talk about test pipeline 301 | 302 | ## 7/13/2022 303 | 304 | Attendees: Steven Royer, Kyle Mestery, Boris Glimcher, Timothy Worsley, Dan Daly, Renato Recio, Venkat Pullela, Anh Thu 305 | 306 | * Boris presented the current state of the developer platform 307 | * Dan to work with IPDK group for how to merge storage test 308 | * Need IPDK/SPDK to publish SPDK container image 309 | 310 | ## 7/27/2022 311 | 312 | * Boris presented the current state of the developer platform 313 | 314 | ## 8/10/2022 315 | 316 | * Decided to end the regularly scheduled meetings for this group. We will use 317 | github and slack (#poc-dev-platform) for further discussions. 318 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OPI Project PoC 2 | 3 | [![Integration](https://github.com/opiproject/opi-poc/actions/workflows/poc-integration.yml/badge.svg)](https://github.com/opiproject/opi-poc/actions/workflows/poc-integration.yml) 4 | [![Linters](https://github.com/opiproject/opi-poc/actions/workflows/linters.yml/badge.svg)](https://github.com/opiproject/opi-poc/actions/workflows/linters.yml) 5 | [![GitHub stars](https://img.shields.io/github/stars/opiproject/opi-poc.svg?style=flat-square&label=github%20stars)](https://github.com/opiproject/opi-poc) 6 | [![GitHub Contributors](https://img.shields.io/github/contributors/opiproject/opi-poc.svg?style=flat-square)](https://github.com/opiproject/opi-poc/graphs/contributors) 7 | 8 | This repo hosts OPI proofs of concept. These PoCs are used to demonstrate that 9 | the OPI project can work for some set of use cases. As a result, OPI CI is 10 | built on top of these PoCs. 11 | 12 | ## High level requirements 13 | 14 | A PoC should demonstrate some aspect or aspects of OPI, for example a networking 15 | PoC could demonstrate some networking application using OPI APIs, provisioning, 16 | and lifecycle management. 17 | 18 | In most cases, the PoC should 19 | 20 | * be implemented by some set of containers 21 | * include tests 22 | * run on a variety of hardware configurations including virtual and multiple 23 | vendors' devices 24 | 25 | ## Current PoCs 26 | 27 | * [Developer Platform / Integration Platform](integration/README.md) aka The CI 28 | * [Storage](storage/README.md) 29 | * [Security](security/README.md) 30 | * [Demos](demos/README.md) 31 | 32 | ## LAB 33 | 34 | Moved to 35 | 36 | ## I Want To Contribute 37 | 38 | This project welcomes contributions and suggestions. We are happy to have the 39 | Community involved via submission of **Issues and Pull Requests** (with 40 | substantive content or even just fixes). We are hoping for the documents, 41 | test framework, etc. to become a community process with active engagement. 42 | PRs can be reviewed by any number of people, and a maintainer may accept. 43 | 44 | See [CONTRIBUTING](https://github.com/opiproject/opi/blob/main/CONTRIBUTING.md) 45 | and [GitHub Basic Process](https://github.com/opiproject/opi/blob/main/doc-github-rules.md) 46 | for more details. 47 | -------------------------------------------------------------------------------- /demos/README.md: -------------------------------------------------------------------------------- 1 | # demos practical examples 2 | 3 | ## Networking API 4 | 5 | [API protobufs](https://github.com/opiproject/opi-api/tree/main/network/cloud) 6 | 7 | [Additionl API proposal](https://github.com/opiproject/opi-api/pull/128) 8 | 9 | [how to](./network/) 10 | 11 | clients and 12 | 13 | [Video recording](https://youtu.be/UWIGLShe8d0) 14 | 15 | ## IPSec 16 | 17 | [API protobufs](https://github.com/opiproject/opi-api/tree/main/security) 18 | 19 | [how to](./security/) 20 | 21 | clients and 22 | 23 | [Video recording](https://youtu.be/uf6xXRXKlQo) 24 | 25 | ## Storage 26 | 27 | [API protobufs](https://github.com/opiproject/opi-api/tree/main/storage) 28 | 29 | [how to](./storage/) 30 | 31 | clients and 32 | 33 | [Video recording](tbd) 34 | -------------------------------------------------------------------------------- /demos/network/Presentation-opi-api.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opiproject/opi-poc/afd4bca7aa9c093edeb538e4119e6b7a11c0d899/demos/network/Presentation-opi-api.pptx -------------------------------------------------------------------------------- /demos/network/README.md: -------------------------------------------------------------------------------- 1 | # Network Cloud demo 2 | 3 | ## Video 4 | 5 | [Recording](https://youtu.be/UWIGLShe8d0) 6 | 7 | ## API 8 | 9 | [API protobufs](https://github.com/opiproject/opi-api/tree/main/network/cloud) 10 | 11 | [Additionl API proposal](https://github.com/opiproject/opi-api/pull/128) 12 | 13 | ## Server 14 | 15 | TBD 16 | 17 | ## Client 18 | 19 | see and 20 | -------------------------------------------------------------------------------- /demos/network/opi-api-demo-pr128.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opiproject/opi-poc/afd4bca7aa9c093edeb538e4119e6b7a11c0d899/demos/network/opi-api-demo-pr128.pdf -------------------------------------------------------------------------------- /demos/redfish/README.md: -------------------------------------------------------------------------------- 1 | # Redfish 2 | 3 | see and 4 | 5 | Example: 6 | 7 | ```bash 8 | docker run --rm --pull=always ghcr.io/opiproject/ansible-opi-dpu:main all \ 9 | --module-name include_role \ 10 | --args name=bmc_fw_update \ 11 | -vvv -i "172.22.4.2," \ 12 | -e dpu_bmc_username='root' \ 13 | -e dpu_bmc_password='NvidiaBf2#Pass' \ 14 | -e bmc_fw_update_inventory_name='a5e46d02_running' \ 15 | -e bmc_fw_update_image_file='/tmp/bf2-bmc-ota-24.01-5-opn.tar' 16 | ``` 17 | -------------------------------------------------------------------------------- /demos/security/README.md: -------------------------------------------------------------------------------- 1 | # IPSec demo 2 | 3 | - Nvidia 4 | - Marvell 5 | - Intel 6 | -------------------------------------------------------------------------------- /demos/security/intel/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opiproject/opi-poc/afd4bca7aa9c093edeb538e4119e6b7a11c0d899/demos/security/intel/README.md -------------------------------------------------------------------------------- /demos/security/marvell/README.md: -------------------------------------------------------------------------------- 1 | # Demos 2 | 3 | ## IPSec offload on Marvell DPU with gRPC 4 | 5 | This demo shows configuration of Marvell DPU for IPsec offload using gRPC. 6 | The data path is based on HW accelerated VPP and leverages Ligato for off-chip 7 | configuration. 8 | 9 | ### hardware 10 | 11 | - server with Ubuntu 18.04 (host) 12 | - Marvell Octeon CN10K (DPU) 13 | - external server 14 | 15 | ### configuration 16 | 17 | #### host 18 | 19 | - Configure Mellanox 100G nic towards DPU 20 | 21 | #### external server 22 | 23 | - Run etcd 24 | - Run external Agent on external server 25 | 26 | ### dpu 27 | 28 | - Run IPSec offload container 29 | This will launch vpp and Ligato (vpp-agent) 30 | 31 | ### Video Recording 32 | 33 | see 34 | 35 | --------------------------------------------------------------------- 36 | 37 | ## Strongswan integration into Marvell DPU using opi-strongswan-bridge 38 | 39 | This demo shows strongswan configuration on Marvell DPU using 40 | opi-strongswan-bridge. opi-strongswan-bridge is a secure server and allows 41 | IPSec off-chip configuration using gRPC based OPI security APIs. It will write 42 | this configuration to Strongswan via vici socket interface. Strongswan in Linux 43 | control plane handles IKE negotiation. IKE packets received in data path by VPP 44 | are transferred to Linux control plane using lcp plugin in VPP. After IKE 45 | negotiation, VPP imports the ip xfrm config from Linux and encrypts traffic. 46 | 47 | ### remote client 48 | 49 | - Run ipsec-config.py from opi-poc on external server 50 | 51 | ### dpu applications 52 | 53 | - Run opi-strongswan-bridge 54 | - Run VPP 55 | - Run Strongswan 56 | 57 | ### Demo Link 58 | 59 | see 60 | -------------------------------------------------------------------------------- /demos/security/nvidia/README.md: -------------------------------------------------------------------------------- 1 | # IPSec demo 2 | 3 | ## hardware 4 | 5 | - server with Ubuntu 22.04 6 | - Nvidia BlueField2 or BlueField3 7 | - Keysight CloudStorm 100G 8 | 9 | ## configuration 10 | 11 | ### host (the server holding the DPU) 12 | 13 | - install Ubuntu 22.04 server 14 | - install `MLNX_OFED_LINUX-24.04-0.6.6.0-ubuntu22.04-x86_64.iso` 15 | - install `bf-bundle-2.7.0-33_24.04_ubuntu-22.04_prod.bfb` on the BlueField2 16 | - set BlueField2 in SEPARATED_HOST mode to make things easier 17 | 18 | ```Shell 19 | mlxconfig -d /dev/mst/mt41686_pciconf0 s INTERNAL_CPU_MODEL=0 20 | mlxconfig -d /dev/mst/mt41686_pciconf0.1 s INTERNAL_CPU_MODEL=0 21 | ``` 22 | 23 | ### dpu (BlueField2) 24 | 25 | - enable ip forwarding `sysctl net.ipv4.ip_forward=1` 26 | - set ip addresses on the phisical links 27 | 28 | ```Shell 29 | ip addr add 200.0.0.1/24 dev enp3s0f0s0 30 | ip addr add 201.0.0.1/24 dev enp3s0f1s0 31 | ``` 32 | 33 | - install Docker (see [Docker manual](https://docs.docker.com/engine/install/ubuntu/) ) 34 | - set the secrets (TODO: see if it is posible to do it via opi-api) 35 | 36 | ```Shell 37 | cat /etc/swanctl/conf.d/k.swanctl.conf 38 | secrets { 39 | ike-tun1_0_0 { 40 | secret = "ipsec" 41 | } 42 | ike-tun1_0_1 { 43 | secret = "ipsec" 44 | } 45 | ``` 46 | 47 | - start OPI server 48 | 49 | see 50 | 51 | ```Shell 52 | docker pull ghcr.io/opiproject/opi-strongswan-bridge:main 53 | 54 | sudo sed -i "s/# socket = unix:\/\/\${piddir}\/charon\.vici/socket = unix:\/\/\/var\/run\/charon\.vici/g" /etc/strongswan.d/charon/vici.conf 55 | 56 | docker run --rm --network host \ 57 | --mount src=/var/run,target=/var/run,type=bind \ 58 | -d ghcr.io/opiproject/opi-strongswan-bridge:main /opi-vici-bridge -port=50151 59 | ``` 60 | 61 | - to debug or see what is happening open 2 ssh connections to the DPU 62 | 63 | ```Shell 64 | # in SSH1, instead of -d use -it flag when starting the container 65 | docker run --rm --network host \ 66 | --mount src=/var/run,target=/var/run,type=bind \ 67 | -it ghcr.io/opiproject/opi-strongswan-bridge:main /opi-vici-bridge -port=50151 68 | 69 | # in SSH2, it will show how ipsec tunnels are initiated and terminated 70 | swanctl --log 71 | ``` 72 | 73 | ### anywhere (on your pc/laptop on the host server.....) 74 | 75 | - run the OPI client application done for ipsec demo 76 | 77 | see and 78 | 79 | ```Shell 80 | # add python api lib to path 81 | git clone https://github.com/opiproject/opi-poc.git 82 | cd ./opi-poc/ 83 | pip3 install -r demos/security/nvidia/requirements.txt 84 | pytest -s -v demos/security/nvidia/test_ipsec.py 85 | ``` 86 | -------------------------------------------------------------------------------- /demos/security/nvidia/Statistics.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import pandas as pd 5 | from tabulate import tabulate 6 | 7 | 8 | class JSONObject: 9 | def __init__(self, dict): 10 | vars(self).update(dict) 11 | 12 | 13 | def json_to_class(path): 14 | """ 15 | Converts a json into a class 16 | """ 17 | json_file = open(path) 18 | s = json_file.read() 19 | return json.loads(s, object_hook=JSONObject) 20 | 21 | 22 | class Statistics: 23 | criteria_message = '' 24 | 25 | def __init__(self, csvs_path): 26 | """ 27 | Takes in the path to csv folder 28 | """ 29 | self.csvs_path = csvs_path 30 | self.headers = ['Condition', 'Status'] 31 | self.table = [] 32 | self.stats_failures = [] 33 | self.config_type = None 34 | self.stats = {} 35 | self.include_baseline_file = True 36 | for csv in os.listdir(csvs_path): 37 | if csv.endswith(".csv"): 38 | self.stats[csv[:-4]] = self.make_dataframe(os.path.join(csvs_path, csv)) 39 | 40 | def make_dataframe(self, csv_file_path): 41 | ''' 42 | Creates a data frame from a csv found at that path 43 | :csv_file_path 44 | ''' 45 | with open(csv_file_path, encoding='utf-8') as csvf: 46 | try: 47 | csv = pd.read_csv(csvf) 48 | except pd.errors.EmptyDataError: 49 | raise Exception("{} is empty".format(csv_file_path)) 50 | except pd.errors.ParserError: 51 | raise Exception("{} is corupt".format(csv_file_path)) 52 | return csv 53 | 54 | @staticmethod 55 | def last(df): 56 | df = df[df['Timestamp epoch ms'] == max(df['Timestamp epoch ms'])] 57 | return df 58 | 59 | def preform_validation(self, validation_entry): 60 | stats = self.stats 61 | last = Statistics.last 62 | try: 63 | validation_ok = eval(validation_entry.condition) 64 | except : 65 | raise Exception("This validation is not written correctly: {}".format(validation_entry.condition)) 66 | if validation_ok: 67 | self.table.append([validation_entry.description, 'Pass']) 68 | else: 69 | self.table.append([validation_entry.description, 'Fail']) 70 | self.stats_failures.append(validation_entry.description) 71 | 72 | def validate_criteria_file(self, criteria_path): 73 | """ 74 | Preforms specific validation for a config and decides if the baseline validation needs to be added 75 | criteria path: path to the json criteria 76 | """ 77 | validator = json_to_class(criteria_path) 78 | self.include_baseline_file = validator.include_baseline 79 | if self.config_type['dut']: 80 | validator = validator.DUT 81 | else: 82 | validator = validator.B2B 83 | for validation_entry in validator: 84 | self.preform_validation(validation_entry) 85 | 86 | def validate_baseline_file(self, criteria_path): 87 | """ 88 | Checks what type of profiles are present in the test and preforms general validation 89 | criteria path: path to the json criteria 90 | config_type: A dictionary that flags the types of profiles present inside the test 91 | """ 92 | validator = json_to_class(criteria_path) 93 | if self.config_type['dut']: 94 | validator = validator.DUT 95 | else: 96 | validator = validator.B2B 97 | if self.config_type['traffic'] or self.config_type['attack']: 98 | for validation_entry in validator.general: 99 | self.preform_validation(validation_entry) 100 | else: 101 | raise Exception('The config does not have an attack or traffic profile') 102 | if self.config_type['traffic']: 103 | for validation_entry in validator.traffic: 104 | self.preform_validation(validation_entry) 105 | if self.config_type['attack']: 106 | for validation_entry in validator.attack: 107 | self.preform_validation(validation_entry) 108 | 109 | def validate_mdw_stats(self, config_type, config_path=""): 110 | """ 111 | Using a the criteria json and the baseline json, validates the resources returned after the run. 112 | config_type: A dictionary that flags the types of profiles present inside the test 113 | config_name: same name as the test that ran. 114 | """ 115 | 116 | self.config_type = config_type 117 | if os.path.exists(config_path): 118 | config_name = os.path.basename(config_path) 119 | print("Config: {}\n\n".format(config_name)) 120 | criteria_path = os.path.join(config_path, 'validation.json') 121 | print(criteria_path) 122 | print('Running Validations for {}'.format(config_name)) 123 | 124 | if os.path.exists(criteria_path): 125 | 126 | try: 127 | self.validate_criteria_file(criteria_path) 128 | except AttributeError as e: 129 | print('Criteria {} could not be applied due to: {}'.format(config_name, e)) 130 | else: 131 | self.include_baseline_file = True 132 | if self.include_baseline_file: 133 | print('Baseline validation applied') 134 | criteria_path = os.path.join("./resources", "baseline_validation.json") 135 | self.validate_baseline_file(criteria_path) 136 | else: 137 | print('Baseline validation skipped') 138 | print(tabulate(self.table, self.headers, tablefmt="grid")) 139 | return "; ".join(self.stats_failures) 140 | 141 | -------------------------------------------------------------------------------- /demos/security/nvidia/cyperf-ipsec-config.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opiproject/opi-poc/afd4bca7aa9c093edeb538e4119e6b7a11c0d899/demos/security/nvidia/cyperf-ipsec-config.zip -------------------------------------------------------------------------------- /demos/security/nvidia/deployment/.env: -------------------------------------------------------------------------------- 1 | DOCKER_REGISTRY=ghcr.io/open-traffic-generator 2 | CONTROLLER_VERSION=latest 3 | TRAFFIC_ENGINE_VERSION=latest 4 | AUR_VERSION=latest 5 | IFC1=enp129s0f0np0 6 | IFC2=enp129s0f1np1 7 | TCP_PORT_IFC1=5555 8 | TCP_PORT_IFC2=5556 9 | CPU_CORES_IFC1="0,1,2" 10 | CPU_CORES_IFC2="0,3,4" 11 | OPT_ENABLE_IPv6="No" 12 | -------------------------------------------------------------------------------- /demos/security/nvidia/deployment/conf/k.swanctl.conf: -------------------------------------------------------------------------------- 1 | secrets { 2 | ike-tun1_0_0 { 3 | secret = "ipsec" 4 | } 5 | ike-tun1_0_1 { 6 | secret = "ipsec" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /demos/security/nvidia/deployment/conf/vici.conf: -------------------------------------------------------------------------------- 1 | vici { 2 | 3 | # Whether to load the plugin. Can also be an integer to increase the 4 | # priority of this plugin. 5 | load = yes 6 | 7 | # Socket the vici plugin serves clients. 8 | socket = unix:///var/run/charon.vici 9 | 10 | } 11 | -------------------------------------------------------------------------------- /demos/security/nvidia/deployment/cyperf_with_ipsec.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | client: 5 | image: public.ecr.aws/keysight/cyperf-agent:latest 6 | container_name: ClientAgent 7 | #network_mode: "host" 8 | restart: always 9 | privileged: true 10 | cpuset: ${CPU_CORES_IFC1:-"0"} 11 | #command: sh -c "ip link set enp129s0f0np0 netns {} name enp129s0f0np0 && cyperfagent interface test set enp129s0f0np0" 12 | environment: 13 | - AGENT_TAGS="Dockers-Group=CyPerf-Agent-Client,node-owner=KB" 14 | - AGENT_MANAGEMENT_INTERFACE=br1 15 | - AGENT_TEST_INTERFACE=enp129s0f0np0 16 | cap_add: 17 | - NET_ADMIN 18 | - IPC_LOCK 19 | - NET_RAW 20 | networks: 21 | cyperf-mgmt-net: 22 | ipv4_address: 192.168.0.10 23 | server: 24 | image: public.ecr.aws/keysight/cyperf-agent:latest 25 | container_name: ServerAgent 26 | #network_mode: "host" 27 | restart: always 28 | privileged: true 29 | cpuset: ${CPU_CORES_IFC2:-"1"} 30 | #command: sh -c "ip link set enp129s0f1np1 netns {} name enp129s0f1np1 && cyperfagent interface test set enp129s0f1np1" 31 | environment: 32 | - AGENT_TAGS="Dockers-Group=CyPerf-Agent-Server,node-owner=KB" 33 | - AGENT_MANAGEMENT_INTERFACE=br1 34 | - AGENT_TEST_INTERFACE=enp129s0f1np1 35 | cap_add: 36 | - NET_ADMIN 37 | - IPC_LOCK 38 | - NET_RAW 39 | 40 | networks: 41 | cyperf-mgmt-net: 42 | ipv4_address: 192.168.0.11 43 | 44 | networks: 45 | cyperf-mgmt-net: 46 | name: mgmt-net 47 | ipam: 48 | config: 49 | - subnet: "192.168.0.0/24" 50 | -------------------------------------------------------------------------------- /demos/security/nvidia/presentation-opi-api.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opiproject/opi-poc/afd4bca7aa9c093edeb538e4119e6b7a11c0d899/demos/security/nvidia/presentation-opi-api.pptx -------------------------------------------------------------------------------- /demos/security/nvidia/requirements.txt: -------------------------------------------------------------------------------- 1 | pypdu 2 | simplejson 3 | tabulate 4 | -------------------------------------------------------------------------------- /demos/security/nvidia/test_ipsec.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | import netmiko 3 | import pydpu 4 | import pytest 5 | import snappi 6 | import time 7 | from pprint import pprint as pp 8 | import os 9 | import sys 10 | 11 | sys.path.insert(0,os.path.join(os.getcwd(), "demos")) 12 | sys.path.insert(1,os.path.join(os.getcwd(), "demos/security/nvidia")) 13 | sys.path.insert(2,'..') 14 | 15 | import datetime 16 | from testbed import * 17 | from RESTasV3 import RESTasV3 18 | from tabulate import tabulate 19 | from pydpu.proto.v1 import ipsec_pb2 20 | from pydpu.proto.v1 import ipsec_pb2_grpc 21 | 22 | 23 | @pytest.fixture(scope='session') 24 | def dpu(): 25 | dpu_info = { 26 | 'device_type': 'linux', 27 | 'host': BF2_IP, 28 | 'username': 'root', 29 | 'use_keys': True, 30 | 'key_file': '/home/opi/.ssh/id_rsa.pub' 31 | } 32 | dpu_connect = netmiko.ConnectHandler(**dpu_info) 33 | dpu_connect.send_command('ip addr add 200.0.0.1/24 dev %s' % BF2_INTERFACES[0]) 34 | dpu_connect.send_command('ip addr add 201.0.0.1/24 dev %s' % BF2_INTERFACES[1]) 35 | dpu_connect.send_command('sysctl net.ipv4.ip_forward=1') 36 | dpu_connect.send_command('iptables -P FORWARD ACCEPT') 37 | dpu_connect.send_command('systemctl stop firewalld') 38 | dpu_connect.send_command('systemctl disable firewalld') 39 | dpu_connect.send_command('systemctl start docker') 40 | 41 | command = 'docker run --rm --network host --mount src=/var/run,target=/var/run,type=bind --name opi-strongswan-bridge -d ghcr.io/opiproject/opi-strongswan-bridge:main /opi-vici-bridge -port=50151' 42 | print(command) 43 | output = dpu_connect.send_command(command) 44 | print(output) 45 | 46 | # idealy replace as many of those ssh commands with OPI API as they become available 47 | 48 | yield dpu_connect 49 | 50 | dpu_connect.send_command('ip addr del 200.0.0.1/24 dev %s' % BF2_INTERFACES[0]) 51 | dpu_connect.send_command('ip addr del 201.0.0.1/24 dev %s' % BF2_INTERFACES[1]) 52 | dpu_connect.send_command('docker stop opi-strongswan-bridge') 53 | dpu_connect.send_command('ip neigh flush 200.0.0.0/8') 54 | dpu_connect.send_command('ip neigh flush 201.0.0.0/8') 55 | 56 | 57 | @pytest.fixture(scope='session') 58 | def server(): 59 | 60 | server_info = { 61 | 'device_type': 'linux', 62 | 'host': TGEN1_IP, 63 | 'username': 'root', 64 | 'use_keys': True, 65 | 'key_file': '/home/opi/.ssh/id_rsa.pub' 66 | } 67 | server_connect = netmiko.ConnectHandler(**server_info) 68 | 69 | print(os.getcwd()) 70 | command = 'docker compose --progress plain -f {} up -d'.format(os.path.join(os.getcwd(), 'demos/security/nvidia/deployment/cyperf_with_ipsec.yml')) 71 | print(command) 72 | output = server_connect.send_command(command, read_timeout=30) 73 | print(output) 74 | 75 | command = "docker top ClientAgent | grep startup.sh | sed -n '1p' | awk '{ print $2 }' | xargs -I{} sudo ip link set %s netns {} name %s" % (TGEN1_INTERFACES[0],TGEN1_INTERFACES[0]) 76 | print(command) 77 | output = server_connect.send_command(command) 78 | print(output) 79 | command = "docker top ServerAgent | grep startup.sh | sed -n '1p' | awk '{ print $2 }' | xargs -I{} sudo ip link set %s netns {} name %s" % (TGEN1_INTERFACES[1], TGEN1_INTERFACES[1]) 80 | print(command) 81 | output = server_connect.send_command(command) 82 | print(output) 83 | 84 | 85 | command = 'docker exec ClientAgent bash -c "cyperfagent controller set {}"'.format(CYPERF_IP) 86 | print(command) 87 | output = server_connect.send_command(command, read_timeout=45) 88 | print(output) 89 | command = 'docker exec ServerAgent bash -c "cyperfagent controller set {};"'.format(CYPERF_IP) 90 | print(command) 91 | output = server_connect.send_command(command, read_timeout=45) 92 | print(output) 93 | 94 | 95 | command = 'docker exec ClientAgent bash -c "cyperfagent interface test set {};ip link set up dev {}"'.format(TGEN1_INTERFACES[0],TGEN1_INTERFACES[0]) 96 | print(command) 97 | output = server_connect.send_command(command, read_timeout=45) 98 | print(output) 99 | command = 'docker exec ServerAgent bash -c "cyperfagent interface test set {};ip link set up dev {}"'.format(TGEN1_INTERFACES[1], TGEN1_INTERFACES[1]) 100 | print(command) 101 | output = server_connect.send_command(command, read_timeout=45) 102 | print(output) 103 | time.sleep(30) 104 | 105 | yield server_connect 106 | command = 'docker compose --progress plain -f {} down'.format(os.path.join(os.getcwd(), 'demos/security/nvidia/deployment/cyperf_with_ipsec.yml')) 107 | print(command) 108 | output = server_connect.send_command(command,read_timeout=30) 109 | print(output) 110 | 111 | 112 | def test_server_to_server_via_ipsec_and_dpu(dpu, server): 113 | 114 | print('connecting to opi') 115 | channel = grpc.insecure_channel('%s:50151' % BF2_IP) 116 | stub = ipsec_pb2_grpc.IPsecServiceStub(channel) 117 | 118 | stub.IPsecVersion(ipsec_pb2.IPsecVersionRequest()) 119 | 120 | stub.IPsecStats(ipsec_pb2.IPsecStatsRequest()) 121 | 122 | print('configuring the tunnel') 123 | tun1_0_0 = ipsec_pb2.IPsecLoadConnRequest( 124 | connection=ipsec_pb2.Connection( 125 | name='tun1_0_0', 126 | version='2', 127 | local_addrs=[ipsec_pb2.Addrs(addr='200.0.0.1')], 128 | remote_addrs=[ipsec_pb2.Addrs(addr='200.0.0.2')], 129 | local_auth=ipsec_pb2.LocalAuth( 130 | auth=ipsec_pb2.AUTH_TYPE_PSK, 131 | id='200.0.0.1' 132 | ), 133 | remote_auth=ipsec_pb2.RemoteAuth( 134 | auth=ipsec_pb2.AUTH_TYPE_PSK, 135 | id='200.0.0.2' 136 | ), 137 | children=[ipsec_pb2.Child( 138 | name='tun1_0_0', 139 | esp_proposals=ipsec_pb2.Proposals( 140 | crypto_alg=[ipsec_pb2.CRYPTO_ALGORITHM_AES128], 141 | integ_alg=[ipsec_pb2.INTEG_ALGORITHM_SHA1] 142 | ), 143 | remote_ts=ipsec_pb2.TrafficSelectors( 144 | ts=[ipsec_pb2.TrafficSelectors.TrafficSelector( 145 | cidr='40.0.0.0/24' 146 | )] 147 | ), 148 | local_ts=ipsec_pb2.TrafficSelectors( 149 | ts=[ipsec_pb2.TrafficSelectors.TrafficSelector( 150 | cidr='201.0.0.0/24' 151 | )] 152 | ), 153 | )], 154 | proposals=ipsec_pb2.Proposals( 155 | crypto_alg=[ipsec_pb2.CRYPTO_ALGORITHM_AES128], 156 | integ_alg=[ipsec_pb2.INTEG_ALGORITHM_SHA384], 157 | dhgroups=[ipsec_pb2.DH_GROUPS_MODP1536] 158 | ) 159 | ) 160 | ) 161 | 162 | connection_1 = stub.IPsecLoadConn(tun1_0_0) 163 | 164 | rest = RESTasV3(ipAddress=CYPERF_IP) 165 | rest.wait_agents_connect(agents_nr=2) 166 | response = rest.get_agents() 167 | for res in response: 168 | print(res['AgentTags'], type(res['AgentTags']),res['IP']) 169 | 170 | agents_IPs = (response[1]['IP'],response[0]['IP']) if 'Server' in response[0]['AgentTags'][0] else (response[0]['IP'],response[1]['IP']) 171 | print(agents_IPs) 172 | 173 | def wait_test_finished(timeout=300): 174 | def collect_stats(stats_to_get): 175 | os.system('clear') 176 | for stat,transpose in stats_to_get: 177 | stats = rest._RESTasV3__sendGet('/api/v2/results/{}/stats/{}'.format(result_id, stat), 200, debug=False).json() 178 | headers = stats['columns'] 179 | #import pdb;pdb.set_trace() 180 | if 'snapshots' not in stats:continue 181 | rows = stats['snapshots'][-1]['values'] 182 | if transpose: 183 | rows.insert(0,headers) 184 | rows = list(zip(*rows)) 185 | headers=[] 186 | print("\n\n",stat,"\n") 187 | print(tabulate(rows, headers=headers, tablefmt="github")) 188 | time.sleep(5) 189 | 190 | response = rest.get_test_status() 191 | actual_duration = 0 192 | counter = 1 193 | while actual_duration < rest.testDuration + timeout: 194 | response = rest.get_test_status() 195 | if response['status'] == 'STOPPING' and not rest.stopTrafficTime: 196 | rest.stopTrafficTime = rest._RESTasV3__getEpochTime() 197 | if response['status'] == 'STOPPED': 198 | if response['testElapsed'] >= response['testDuration']: 199 | print('Test gracefully finished') 200 | rest.stopTime = rest._RESTasV3__getEpochTime() 201 | return rest.stopTime 202 | else: 203 | raise Exception("Error! Test stopped before reaching the configured duration = {}; Elapsed = {}" 204 | .format(response['testDuration'], response['testElapsed'])) 205 | else: 206 | print('Test duration = {}; Elapsed = {}'.format(response['testDuration'], response['testElapsed'])) 207 | actual_duration += counter 208 | collect_stats([('client-action-statistics',False),('ipsec-tunnels-total',True)]) 209 | time.sleep(counter) 210 | else: 211 | print("Test did not stop after timeout {}s. Test status= {}. Force stopping the test!".format(timeout,response['status'])) 212 | rest.stop_test() 213 | raise Exception("Error! Test failed to stop after timeout {}s.".format(timeout)) 214 | 215 | def verify_stats(): 216 | istat = rest._RESTasV3__sendGet('/api/v2/results/{}/stats/{}'.format(result_id, 'ipsec-tunnels-total'), 200, debug=False).json() 217 | indx_f, indx_s, indx_ss = istat['columns'].index('Sessions Failed'), istat['columns'].index('Sessions Initiated'), istat['columns'].index('Sessions Succeeded') 218 | rows = istat['snapshots'][-1]['values'] 219 | if not (rows[0][indx_f]=='0' and rows[0][indx_s] == rows[0][indx_ss]): 220 | rest.delete_current_session() 221 | raise Exception("Error! Please check Ipsec Tunnels Total Statistics") 222 | 223 | cstat = rest._RESTasV3__sendGet('/api/v2/results/{}/stats/{}'.format(result_id, 'client-action-statistics'), 200, debug=False).json() 224 | indx_f, indx_s, indx_ss = cstat['columns'].index('Action Failed'), cstat['columns'].index('Action Started'), cstat['columns'].index('Action Succeeded') 225 | rows = cstat['snapshots'][-1]['values'] 226 | 227 | passed = True 228 | for row in rows: 229 | if (row[indx_f]=='0' and row[indx_s] == row[indx_ss]): 230 | continue 231 | else: 232 | passed = False 233 | if not passed: 234 | rest.delete_current_session() 235 | raise Exception("Error! Please check Client Action Statistics") 236 | 237 | 238 | # CyPerf API test with 1 imported test config. This is where we keep all the test configs. 239 | rest.setup(os.path.join(os.getcwd(), "demos/security/nvidia","cyperf-ipsec-config.zip")) 240 | rest.assign_agents_by_ip(agents_ips=agents_IPs[0], network_segment=1) 241 | rest.assign_agents_by_ip(agents_ips=agents_IPs[1], network_segment=2) 242 | rest.set_test_duration(30) 243 | #print("SLEEP FOR 30 MINS") 244 | #time.sleep(180) 245 | rest.start_test() 246 | result_id = rest.get_test_id() 247 | wait_test_finished() 248 | verify_stats() 249 | rest.delete_current_session() 250 | -------------------------------------------------------------------------------- /demos/storage/README.md: -------------------------------------------------------------------------------- 1 | # Storage demo 2 | 3 | - Nvidia 4 | - Intel 5 | - Marvell 6 | -------------------------------------------------------------------------------- /demos/storage/intel/README.md: -------------------------------------------------------------------------------- 1 | # Storage demo 2 | 3 | ## Video 4 | 5 | See 6 | 7 | ## API 8 | 9 | [API protobufs](https://github.com/opiproject/opi-api/tree/main/storage) 10 | 11 | ## Server 12 | 13 | see 14 | 15 | ```Shell 16 | $ docker run --rm -it -v /var/tmp/:/var/tmp/ -p 50051:50051 ghcr.io/opiproject/opi-intel-bridge:main 17 | 2022/11/29 00:03:55 plugin serevr is &{{}} 18 | 2022/11/29 00:03:55 server listening at [::]:50051 19 | ``` 20 | 21 | ## Client 22 | 23 | see and 24 | -------------------------------------------------------------------------------- /demos/storage/marvell/README.md: -------------------------------------------------------------------------------- 1 | # Storage demo 2 | 3 | ## Video 4 | 5 | See 6 | 7 | ## API 8 | 9 | [API protobufs](https://github.com/opiproject/opi-api/tree/main/storage) 10 | 11 | ## Server 12 | 13 | see 14 | 15 | ```Shell 16 | $ docker run --rm -it -v /var/tmp/:/var/tmp/ -p 50051:50051 ghcr.io/opiproject/opi-marvell-bridge:main 17 | 2022/11/29 00:03:55 plugin serevr is &{{}} 18 | 2022/11/29 00:03:55 server listening at [::]:50051 19 | ``` 20 | 21 | ## Client 22 | 23 | see and 24 | -------------------------------------------------------------------------------- /demos/storage/nvidia/README.md: -------------------------------------------------------------------------------- 1 | # Storage demo 2 | 3 | ## Video 4 | 5 | tbd 6 | 7 | ## API 8 | 9 | [API protobufs](https://github.com/opiproject/opi-api/tree/main/storage) 10 | 11 | ## Server 12 | 13 | see 14 | 15 | ```Shell 16 | $ docker run --rm -it -v /var/tmp/:/var/tmp/ -p 50051:50051 ghcr.io/opiproject/opi-nvidia-bridge:main 17 | 2022/11/29 00:03:55 plugin serevr is &{{}} 18 | 2022/11/29 00:03:55 server listening at [::]:50051 19 | ``` 20 | 21 | ## Client 22 | 23 | see and 24 | -------------------------------------------------------------------------------- /demos/testbed.py: -------------------------------------------------------------------------------- 1 | # from https://github.com/opiproject/lab/blob/main/ips.md 2 | 3 | # DPU HOST 1 4 | DH1_IP = '172.22.1.1' 5 | DH1_INTERFACES = ['ens1f0np0', 'ens1f1np1'] 6 | 7 | # AMD/Pensando 8 | PEN_IP = '172.22.3.1' 9 | PEN_INTERFACES = ['enp3s0f1s0', 'enp3s0f0s0'] 10 | 11 | # DPU HOST 2 12 | DH2_IP = '172.22.1.2' 13 | DH2_INTERFACES = ['ens1f0np0', 'ens1f1np1'] 14 | 15 | # Nvidia BlueField2 16 | BF2_IP = '172.22.3.2' 17 | BF2_INTERFACES = ['enp3s0f1s0', 'enp3s0f0s0'] 18 | 19 | # DPU HOST 3 20 | DH3_IP = '172.22.1.3' 21 | #DH3_INTERFACES = ['ens1f0np0', 'ens1f1np1'] 22 | 23 | # TGEN SERVER 1 24 | TGEN1_IP = '172.22.1.100' 25 | 26 | TGEN1_INTERFACES = ['ens1f0np0', 'ens1f1np1'] 27 | #TGEN1_INTERFACES = ['enp138s0f0np0', 'enp138s0f1np1'] 28 | 29 | # CYPERF SERVER 30 | CYPERF_IP = '172.22.222.9' 31 | -------------------------------------------------------------------------------- /demos/tgen/README.md: -------------------------------------------------------------------------------- 1 | # OPI testing demo/poc 2 | 3 | ## hardware 4 | 5 | - server with Ubuntu 22.04 6 | - Nvidia BlueField2 7 | - server with Ubuntu 22.04 8 | - 100G NIC 9 | 10 | ## topologies 11 | 12 | ### covered 13 | 14 | any combination of host or server traffic originated to any of the 4 ports and targeted to any of the 4 ports (it could come back to itself) 15 | ![testbed diagram](./images/opi-lab-scenarios.drawio.svg) 16 | 17 | server to dpu, dpu forwording to the other port and back to the server 18 | ![test_server_to_server_via_dpu](./images/opi-lab-server2server.drawio.svg) 19 | 20 | server to dpu and dpu to the host 21 | ![test_server_to_dpuhost_via_dpu](./images/opi-lab-server2host.drawio.svg) 22 | 23 | host to the dpu, dpu forwording to the other vf port and back to the host 24 | ![test_dpuhost_to_dpuhost_via_dpu](./images/opi-lab-host2host.drawio.svg) 25 | 26 | same scenarios as above but having the traffic inside the ipsec tunnel 27 | ![test_server_to_server_via_ipsec_and_dpu](./images/opi-lab-scenarios-ipsec.drawio.svg) 28 | 29 | ### not covered 30 | 31 | same scenarios as above but having traffic generator endpoints also on the dpu (posible if dpu cores are x86 or arm and docker can be installed) 32 | ![tgen on dpu](./images/opi-lab-scenarios-with-dpu.drawio.svg) 33 | 34 | ## testing framework 35 | 36 | - to automate the opi cases it is prefered to use [python](https://www.python.org) language 37 | - as test framework it is prefered to use [pytest](https://docs.pytest.org) 38 | - for ssh conectivity it is sugested to use [netmiko](https://github.com/ktbyers/netmiko) package instead of [paramiko](https://www.paramiko.org), it uses paramiko and provides drivers for many devices + ability to write plugins 39 | 40 | ## traffic generator 41 | 42 | Open Traffic Generator (OTG) is an open standard, specifying declarative and vendor neutral API for testing Layer 2-7 network devices and applications (at any scale). 43 | 44 | - [open traffic generator website](https://otg.dev) 45 | - [open traffic generator implementation](https://github.com/open-traffic-generator) 46 | - [snappi api library for otg](https://pypi.org/project/snappi) 47 | 48 | ## to run the test 49 | 50 | ```Shell 51 | 52 | git clone https://github.com/opiproject/opi-poc.git 53 | 54 | pip install --no-cache-dir -r ./opi-poc/demos/tgen/requirements.txt 55 | 56 | pytest -s ./opi-poc/demos/tgen/test_tgen_demo.py 57 | 58 | ``` 59 | 60 | ## notes, todo, etc 61 | 62 | to debug traffic flow 63 | 64 | ```Shell 65 | tcpdump -n -e -i enp3s0f1s0 66 | ``` 67 | 68 | setting the way DPU interacts with host requires a reboot 69 | 70 | ```Shell 71 | mst start 72 | mlxconfig -d /dev/mst/mt41686_pciconf0 s INTERNAL_CPU_MODEL=1 73 | mlxconfig -d /dev/mst/mt41686_pciconf0.1 s INTERNAL_CPU_MODEL=1 74 | ``` 75 | 76 | all paths are absolute paths from my demo testbed, those shoudl be fixed to be more flexible and map the git repo better 77 | 78 | ```Shell 79 | example: in test_tgen_demo.py 80 | docker compose -f /root/opi/tgen/deployment/tgen.yml up -d 81 | ``` 82 | 83 | there is an assumption the files exist on host or dpu or the server like docker compose files 84 | 85 | ```Shell 86 | example: in test_tgen_demo.py 87 | docker compose -f /root/opi/tgen/deployment/tgen.yml up -d 88 | ``` 89 | 90 | there is an assumption that docker is installed on the dpu 91 | 92 | ```Shell 93 | example: in test_tgen_with_ipsec_demo.py 94 | systemctl start docker 95 | docker run --rm --network host --mount src=/var/run,target=/var/run,type=bind --name opi-strongswan-bridge -d ghcr.io/opiproject/opi-strongswan-bridge:main /opi-vici-bridge -port=50151 96 | ``` 97 | 98 | care must be taken to prevent test traffic going out on management interface 99 | 100 | pydpu needs to be integrate din ipsec test 101 | -------------------------------------------------------------------------------- /demos/tgen/deployment/.env: -------------------------------------------------------------------------------- 1 | DOCKER_REGISTRY=ghcr.io/open-traffic-generator 2 | CONTROLLER_VERSION=latest 3 | TRAFFIC_ENGINE_VERSION=latest 4 | AUR_VERSION=latest 5 | IFC1=ens1f0np0 6 | IFC2=ens1f1np1 7 | TCP_PORT_IFC1=5555 8 | TCP_PORT_IFC2=5556 9 | CPU_CORES_IFC1="0,1,2" 10 | CPU_CORES_IFC2="0,3,4" 11 | OPT_ENABLE_IPv6="No" 12 | -------------------------------------------------------------------------------- /demos/tgen/deployment/conf/client/k.swanctl.conf: -------------------------------------------------------------------------------- 1 | connections { 2 | tun1_0_0 { 3 | #include /etc/swanctl/conf.d/ike_sa_default.conf 4 | #pfs=no 5 | version=2 6 | remote_addrs=200.0.0.1 7 | local_addrs=200.0.0.2 8 | proposals=aes128-sha384-modp1536 9 | children { 10 | tun1_0_0 { 11 | #include /etc/swanctl/conf.d/child_sa_default.conf 12 | start_action=trap 13 | esp_proposals=aes128-sha1 14 | local_ts=40.0.0.0/24 15 | remote_ts=201.0.0.0/24 16 | } 17 | } 18 | local-0 { 19 | auth = psk 20 | id = 200.0.0.2 21 | } 22 | remote-0 { 23 | auth = psk 24 | id = 200.0.0.1 25 | } 26 | } 27 | tun1_0_1 { 28 | #include /etc/swanctl/conf.d/ike_sa_default.conf 29 | #pfs=no 30 | version=2 31 | remote_addrs=200.0.0.1 32 | local_addrs=200.0.0.3 33 | proposals=aes128-sha384-modp1536 34 | children { 35 | tun1_0_1 { 36 | #include /etc/swanctl/conf.d/child_sa_default.conf 37 | start_action=trap 38 | esp_proposals=aes128-sha1 39 | local_ts=40.0.1.0/24 40 | remote_ts=201.0.0.0/24 41 | } 42 | } 43 | local-0 { 44 | auth = psk 45 | id = 200.0.0.3 46 | } 47 | remote-0 { 48 | auth = psk 49 | id = 200.0.0.1 50 | } 51 | } 52 | } 53 | pools { 54 | } 55 | authorities { 56 | } 57 | secrets { 58 | ike-tun1_0_0 { 59 | secret = "ipsec" 60 | } 61 | ike-tun1_0_1 { 62 | secret = "ipsec" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /demos/tgen/deployment/conf/client/vici.conf: -------------------------------------------------------------------------------- 1 | vici { 2 | 3 | # Whether to load the plugin. Can also be an integer to increase the 4 | # priority of this plugin. 5 | load = yes 6 | 7 | # Socket the vici plugin serves clients. 8 | socket = unix:///var/run/charon.vici 9 | 10 | } 11 | -------------------------------------------------------------------------------- /demos/tgen/deployment/ipsec.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | 5 | traffic_controller: 6 | image: ${DOCKER_REGISTRY}/ixia-c-controller:${CONTROLLER_VERSION:-latest} 7 | container_name: ixia-c-controller-${USER} 8 | command: --accept-eula 9 | network_mode: "host" 10 | restart: always 11 | 12 | traffic_engine_1: 13 | image: ${DOCKER_REGISTRY}/ixia-c-traffic-engine:${TRAFFIC_ENGINE_VERSION:-latest} 14 | container_name: ixia-c-traffic-engine1-${USER} 15 | network_mode: "service:vpn-client" 16 | depends_on: 17 | - vpn-client 18 | restart: always 19 | privileged: true 20 | cpuset: ${CPU_CORES_IFC1:-"0"} 21 | environment: 22 | - OPT_LISTEN_PORT=${TCP_PORT_IFC1:-5555} 23 | - ARG_IFACE_LIST=virtual@af_packet,${IFC1} 24 | - OPT_NO_HUGEPAGES=Yes 25 | - OPT_ENABLE_IPv6=${OPT_ENABLE_IPv6} 26 | 27 | traffic_engine_2: 28 | image: ${DOCKER_REGISTRY}/ixia-c-traffic-engine:${TRAFFIC_ENGINE_VERSION:-latest} 29 | container_name: ixia-c-traffic-engine2-${USER} 30 | network_mode: "host" 31 | restart: always 32 | privileged: true 33 | cpuset: ${CPU_CORES_IFC2:-"1"} 34 | environment: 35 | - OPT_LISTEN_PORT=${TCP_PORT_IFC2:-5556} 36 | - ARG_IFACE_LIST=virtual@af_packet,${IFC2} 37 | - OPT_NO_HUGEPAGES=Yes 38 | - OPT_ENABLE_IPv6=${OPT_ENABLE_IPv6} 39 | 40 | vpn-client: 41 | image: strongx509/strongswan:5.9.9 42 | container_name: vpn-client-${USER} 43 | network_mode: "host" 44 | cap_add: 45 | - NET_ADMIN 46 | - SYS_ADMIN 47 | - SYS_MODULE 48 | stdin_open: true 49 | tty: true 50 | volumes: 51 | - ./conf/client/k.swanctl.conf:/etc/swanctl/conf.d/k.swanctl.conf 52 | - ./conf/client/vici.conf:/etc/strongswan.d/charon/vici.conf 53 | #- ./conf/strongswan-client.conf:/etc/strongswan.conf 54 | command: 55 | - './charon' 56 | -------------------------------------------------------------------------------- /demos/tgen/deployment/tgen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.9' 3 | 4 | services: 5 | 6 | controller: 7 | image: ${DOCKER_REGISTRY}/keng-controller:${CONTROLLER_VERSION:-latest} 8 | container_name: keng-controller-${USER} 9 | command: --accept-eula 10 | network_mode: "host" 11 | restart: always 12 | traffic_engine_1: 13 | image: ${DOCKER_REGISTRY}/ixia-c-traffic-engine:${TRAFFIC_ENGINE_VERSION:-latest} 14 | container_name: ixia-c-traffic-engine1-${USER} 15 | network_mode: "host" 16 | restart: always 17 | privileged: true 18 | cpuset: ${CPU_CORES_IFC1:-"0,1,2"} 19 | environment: 20 | - OPT_LISTEN_PORT=${TCP_PORT_IFC1:-5555} 21 | - ARG_IFACE_LIST=virtual@af_packet,${IFC1} 22 | - OPT_NO_HUGEPAGES=Yes 23 | - OPT_ENABLE_IPv6=${OPT_ENABLE_IPv6} 24 | 25 | traffic_engine_2: 26 | image: ${DOCKER_REGISTRY}/ixia-c-traffic-engine:${TRAFFIC_ENGINE_VERSION:-latest} 27 | container_name: ixia-c-traffic-engine2-${USER} 28 | network_mode: "host" 29 | restart: always 30 | privileged: true 31 | cpuset: ${CPU_CORES_IFC2:-"0,3,4"} 32 | environment: 33 | - OPT_LISTEN_PORT=${TCP_PORT_IFC2:-5556} 34 | - ARG_IFACE_LIST=virtual@af_packet,${IFC2} 35 | - OPT_NO_HUGEPAGES=Yes 36 | - OPT_ENABLE_IPv6=${OPT_ENABLE_IPv6} 37 | aur: 38 | image: ghcr.io/open-traffic-generator/keng-app-usage-reporter:${AUR_VERSION:-latest} 39 | network_mode: "host" 40 | restart: always -------------------------------------------------------------------------------- /demos/tgen/images/opi-lab-host2host.drawio.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
IP
IP
VGA
VGA
USB
USB
USB
USB
P1
P1
P2
P2
DPU
DPU
IP
IP
SERVER DPU HOST 1
SERVER DPU HOST 1
tgen container 3
tgen container 3
tgen container 4
tgen container 4
scripts / controller container
scripts / controller container
tgen controller
tgen controllerText is not SVG - cannot display -------------------------------------------------------------------------------- /demos/tgen/images/opi-lab-server2host.drawio.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
IP
IP
VGA
VGA
USB
USB
USB
USB
P1
P1
P2
P2
NIC
NIC
SERVER SW TGEN 1
SERVER SW TGEN 1
tgen container 1
tgen container 1
tgen controller
tgen controller
IP
IP
VGA
VGA
USB
USB
USB
USB
P1
P1
P2
P2
DPU
DPU
IP
IP
SERVER DPU HOST 1
SERVER DPU HOST 1
tgen container 3
tgen container 3
scripts / controller container
scripts / controller containerText is not SVG - cannot display -------------------------------------------------------------------------------- /demos/tgen/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio 2 | netmiko 3 | pydpu 4 | pytest 5 | snappi -------------------------------------------------------------------------------- /demos/tgen/test_tgen_demo.py: -------------------------------------------------------------------------------- 1 | import os 2 | import grpc 3 | import netmiko 4 | import pydpu 5 | import pytest 6 | import snappi 7 | import requests 8 | 9 | from testbed import * 10 | 11 | #HOME_FOLDER = r'/home/opi/actions-runner/_work/opi-poc/opi-poc' 12 | #HOME_FOLDER = r'/home/opi/opi-poc' 13 | HOME_FOLDER = os.environ['GITHUB_WORKSPACE'] 14 | 15 | 16 | @pytest.fixture 17 | def dpu(): 18 | dpu_info = { 19 | 'device_type': 'linux', 20 | 'host': BF2_IP, 21 | 'username': 'root', 22 | 'use_keys': True, 23 | 'key_file': '/home/opi/.ssh/id_rsa.pub' 24 | } 25 | dpu_connect = netmiko.ConnectHandler(**dpu_info) 26 | #prompt = dpu_connect.find_prompt() 27 | dpu_connect.send_command('ip addr add 200.0.0.1/24 dev %s' % BF2_INTERFACES[0]) 28 | dpu_connect.send_command('ip addr add 201.0.0.1/24 dev %s' % BF2_INTERFACES[1]) 29 | dpu_connect.send_command('ip addr add 202.0.0.1/24 dev pf0hpf') 30 | dpu_connect.send_command('ip addr add 203.0.0.1/24 dev pf1hpf') 31 | dpu_connect.send_command('sysctl net.ipv4.ip_forward=1') 32 | dpu_connect.send_command('iptables -P FORWARD ACCEPT') 33 | dpu_connect.send_command('arp -s 200.0.0.2 80:09:02:02:00:01') 34 | dpu_connect.send_command('arp -s 201.0.0.2 80:09:02:02:00:02') 35 | dpu_connect.send_command('arp -s 202.0.0.2 80:09:02:02:00:03') 36 | dpu_connect.send_command('arp -s 203.0.0.2 80:09:02:02:00:04') 37 | # idealy replace as many of those ssh commands with OPI API as they become available 38 | 39 | # iptables -P INPUT ACCEPT 40 | # iptables -P FORWARD ACCEPT 41 | # iptables -P OUTPUT ACCEPT 42 | # iptables -t nat -F 43 | # iptables -t mangle -F 44 | # iptables -F 45 | # iptables -X 46 | 47 | yield dpu_connect 48 | 49 | dpu_connect.send_command('ip addr del 200.0.0.1/24 dev %s' % BF2_INTERFACES[0]) 50 | dpu_connect.send_command('ip addr del 201.0.0.1/24 dev %s' % BF2_INTERFACES[1]) 51 | dpu_connect.send_command('ip addr del 202.0.0.1/24 dev pf0hpf') 52 | dpu_connect.send_command('ip addr del 203.0.0.1/24 dev pf1hpf') 53 | dpu_connect.send_command('ip neigh flush 200.0.0.0/8') 54 | dpu_connect.send_command('ip neigh flush 201.0.0.0/8') 55 | dpu_connect.send_command('ip neigh flush 202.0.0.0/8') 56 | dpu_connect.send_command('ip neigh flush 203.0.0.0/8') 57 | 58 | 59 | @pytest.fixture 60 | def server(): 61 | server_info = { 62 | 'device_type': 'linux', 63 | 'host': TGEN1_IP, 64 | 'username': 'root', 65 | 'use_keys': True, 66 | 'key_file': '/home/opi/.ssh/id_rsa.pub' 67 | } 68 | server_connect = netmiko.ConnectHandler(**server_info) 69 | output = server_connect.send_command('docker compose -f %s/demos/tgen/deployment/tgen.yml up -d' % HOME_FOLDER, read_timeout=30) 70 | print(output) 71 | 72 | yield server_connect 73 | 74 | output = server_connect.send_command('docker compose -f %s/demos/tgen/deployment/tgen.yml down' % HOME_FOLDER) 75 | print(output) 76 | 77 | 78 | @pytest.fixture 79 | def host(): 80 | # needs reboot 81 | # mst start 82 | # mlxconfig -d /dev/mst/mt41686_pciconf0 s INTERNAL_CPU_MODEL=1 83 | # mlxconfig -d /dev/mst/mt41686_pciconf0.1 s INTERNAL_CPU_MODEL=1 84 | # mlxconfig -d /dev/mst/mt41686_pciconf0 s INTERNAL_CPU_MODEL=0 85 | # mlxconfig -d /dev/mst/mt41686_pciconf0.1 s INTERNAL_CPU_MODEL=0 86 | host_info = { 87 | 'device_type': 'linux', 88 | 'host': DH2_IP, 89 | 'username': 'root', 90 | 'use_keys': True, 91 | 'key_file': '/home/opi/.ssh/id_rsa.pub' 92 | } 93 | host_connect = netmiko.ConnectHandler(**host_info) 94 | 95 | scp_conn = netmiko.SCPConn(host_connect) 96 | scp_conn.scp_put_file('%s/demos/tgen/deployment/tgen.yml' % HOME_FOLDER, '~/tgen.yaml') 97 | scp_conn.scp_put_file('%s/demos/tgen/deployment/.env' % HOME_FOLDER, '~/.env') 98 | 99 | output = host_connect.send_command('docker compose -f ~/tgen.yaml up -d', read_timeout=30) 100 | print(output) 101 | 102 | yield host_connect 103 | 104 | output = host_connect.send_command('docker compose -f ~/tgen.yaml down') 105 | print(output) 106 | 107 | 108 | def test_server_to_server_via_dpu(dpu, server): 109 | tgen = snappi.api(location=f'https://{TGEN1_IP}:8443', verify=False) 110 | cfg = tgen.config() 111 | p1 = cfg.ports.port(name='server:p1', location=f'{TGEN1_IP}:5555')[-1] 112 | p2 = cfg.ports.port(name='server:p2', location=f'{TGEN1_IP}:5556')[-1] 113 | 114 | # add layer 1 property to configure same speed on both ports 115 | ly = cfg.layer1.layer1(name='ly')[-1] 116 | ly.port_names = [p1.name, p2.name] 117 | ly.speed = ly.SPEED_10_GBPS 118 | 119 | # enable packet capture on both ports 120 | cp = cfg.captures.capture(name='cp')[-1] 121 | cp.port_names = [p1.name, p2.name] 122 | 123 | flow = cfg.flows.flow(name='flow server:p2 -> server:p1')[-1] 124 | flow.tx_rx.port.tx_name = p2.name 125 | flow.tx_rx.port.rx_name = p1.name 126 | flow.size.fixed = 256 127 | flow.duration.fixed_packets.packets = 1000 128 | flow.rate.pps = 1000 129 | eth2, ip2, udp2 = flow.packet.ethernet().ipv4().udp() 130 | eth2.src.value = '00:1B:6E:00:00:01' 131 | eth2.dst.value = '02:6a:82:5d:da:2c' 132 | ip2.src.value = '201.0.0.2' 133 | ip2.dst.value = '200.0.0.2' 134 | inc2 = udp2.src_port.increment 135 | inc2.start, inc2.step, inc2.count = 6000, 4, 10 136 | udp2.dst_port.values = [8000, 8044, 8060, 8074, 8082, 8084] 137 | 138 | print('Pushing traffic configuration ...') 139 | tgen.set_config(cfg) 140 | 141 | print('Starting transmit on all configured flows ...') 142 | cs = tgen.control_state() 143 | cs.traffic.flow_transmit.state = cs.traffic.flow_transmit.START 144 | tgen.set_control_state(cs) 145 | 146 | #import pdb 147 | #pdb.set_trace() 148 | 149 | print('Checking metrics on all configured ports ...') 150 | print('Expected\tTotal Tx\tTotal Rx') 151 | assert wait_for(lambda: metrics_ok(tgen)), 'Metrics validation failed!' 152 | 153 | 154 | def test_server_to_dpuhost_via_dpu(dpu, server, host): 155 | tgen = snappi.api(location=f'https://{TGEN1_IP}:8443', verify=False) 156 | cfg = tgen.config() 157 | p1 = cfg.ports.port(name='host:p1', location=f'{DH2_IP}:5555')[-1] 158 | p2 = cfg.ports.port(name='server:p1', location=f'{TGEN1_IP}:5556')[-1] 159 | 160 | # add layer 1 property to configure same speed on both ports 161 | ly = cfg.layer1.layer1(name='ly')[-1] 162 | ly.port_names = [p1.name, p2.name] 163 | ly.speed = ly.SPEED_10_GBPS 164 | 165 | # enable packet capture on both ports 166 | cp = cfg.captures.capture(name='cp')[-1] 167 | cp.port_names = [p1.name, p2.name] 168 | 169 | flow = cfg.flows.flow(name='flow server:p2 -> host:p1')[-1] 170 | flow.tx_rx.port.tx_name = p2.name 171 | flow.tx_rx.port.rx_name = p1.name 172 | flow.size.fixed = 256 173 | flow.duration.fixed_packets.packets = 1000 174 | flow.rate.pps = 1000 175 | eth2, ip2, udp2 = flow.packet.ethernet().ipv4().udp() 176 | eth2.src.value = '00:1B:6E:00:00:01' 177 | eth2.dst.value = '02:6a:82:5d:da:2c' 178 | ip2.src.value = '200.0.0.2' 179 | ip2.dst.value = '202.0.0.2' 180 | inc2 = udp2.src_port.increment 181 | inc2.start, inc2.step, inc2.count = 6000, 4, 10 182 | udp2.dst_port.values = [8000, 8044, 8060, 8074, 8082, 8084] 183 | 184 | print('Pushing traffic configuration ...') 185 | tgen.set_config(cfg) 186 | 187 | print('Starting transmit on all configured flows ...') 188 | cs = tgen.control_state() 189 | cs.traffic.flow_transmit.state = cs.traffic.flow_transmit.START 190 | tgen.set_control_state(cs) 191 | 192 | print('Checking metrics on all configured ports ...') 193 | print('Expected\tTotal Tx\tTotal Rx') 194 | assert wait_for(lambda: metrics_ok(tgen)), 'Metrics validation failed!' 195 | 196 | 197 | def test_dpuhost_to_dpuhost_via_dpu(dpu, host): 198 | tgen = snappi.api(location=f'https://{DH2_IP}:8443', verify=False) 199 | cfg = tgen.config() 200 | p1 = cfg.ports.port(name='host:p1', location=f'{DH2_IP}:5555')[-1] 201 | p2 = cfg.ports.port(name='host:p2', location=f'{DH2_IP}:5556')[-1] 202 | 203 | # add layer 1 property to configure same speed on both ports 204 | ly = cfg.layer1.layer1(name='ly')[-1] 205 | ly.port_names = [p1.name, p2.name] 206 | ly.speed = ly.SPEED_10_GBPS 207 | 208 | # enable packet capture on both ports 209 | cp = cfg.captures.capture(name='cp')[-1] 210 | cp.port_names = [p1.name, p2.name] 211 | 212 | flow = cfg.flows.flow(name='flow host:p2 -> host:p1')[-1] 213 | flow.tx_rx.port.tx_name = p2.name 214 | flow.tx_rx.port.rx_name = p1.name 215 | flow.size.fixed = 256 216 | flow.duration.fixed_packets.packets = 1000 217 | flow.rate.pps = 1000 218 | eth2, ip2, udp2 = flow.packet.ethernet().ipv4().udp() 219 | eth2.src.value = '00:1B:6E:00:00:01' 220 | eth2.dst.value = '02:6a:82:5d:da:2c' 221 | ip2.src.value = '203.0.0.2' 222 | ip2.dst.value = '202.0.0.2' 223 | inc2 = udp2.src_port.increment 224 | inc2.start, inc2.step, inc2.count = 6000, 4, 10 225 | udp2.dst_port.values = [8000, 8044, 8060, 8074, 8082, 8084] 226 | 227 | print('Pushing traffic configuration ...') 228 | tgen.set_config(cfg) 229 | 230 | print('Starting transmit on all configured flows ...') 231 | cs = tgen.control_state() 232 | cs.traffic.flow_transmit.state = cs.traffic.flow_transmit.START 233 | tgen.set_control_state(cs) 234 | 235 | print('Checking metrics on all configured ports ...') 236 | print('Expected\tTotal Tx\tTotal Rx') 237 | assert wait_for(lambda: metrics_ok(tgen)), 'Metrics validation failed!' 238 | 239 | 240 | def metrics_ok(api): 241 | # create a port metrics request and filter based on port names 242 | cfg = api.get_config() 243 | 244 | req = api.metrics_request() 245 | req.port.port_names = [p.name for p in cfg.ports] 246 | # include only sent and received packet counts 247 | req.port.column_names = [req.port.FRAMES_TX, req.port.FRAMES_RX] 248 | # fetch port metrics 249 | res = api.get_metrics(req) 250 | # calculate total frames sent and received across all configured ports 251 | total_tx = sum([m.frames_tx for m in res.port_metrics]) 252 | total_rx = sum([m.frames_rx for m in res.port_metrics]) 253 | expected = sum([f.duration.fixed_packets.packets for f in cfg.flows]) 254 | 255 | print('%d\t\t%d\t\t%d' % (expected, total_tx, total_rx)) 256 | 257 | return expected == total_tx and total_rx >= expected 258 | 259 | 260 | def wait_for(func, timeout=10, interval=0.2): 261 | ''' 262 | Keeps calling the `func` until it returns true or `timeout` occurs 263 | every `interval` seconds. 264 | ''' 265 | import time 266 | 267 | start = time.time() 268 | 269 | while time.time() - start <= timeout: 270 | if func(): 271 | return True 272 | time.sleep(interval) 273 | 274 | print('Timeout occurred !') 275 | return False 276 | -------------------------------------------------------------------------------- /integration/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PATH_SEPARATOR=: 2 | COMPOSE_FILE=docker-compose.host.yml:docker-compose.xpu.yml:docker-compose.otel.yml:docker-compose.spdk.yml:docker-compose.pxe.yml:docker-compose.networks.yml 3 | -------------------------------------------------------------------------------- /integration/README.md: -------------------------------------------------------------------------------- 1 | # Integration 2 | 3 | ## Diagram 4 | 5 | ![DPU SW Components](xPU-Integration-Blocks.png) 6 | 7 | ## Prereqs 8 | 9 | [Setup via Ansible](../setup/README.md) 10 | 11 | ## Start 12 | 13 | This pulls the latest images and only builds those it cannot find. 14 | 15 | ```bash 16 | ./scripts/integration.sh start 17 | ``` 18 | 19 | If you are making changes to the container images, you can `build` them before 20 | running `start`. **Note** This does not work for images pulled from a cr like 21 | the spdk-target image. 22 | 23 | ```bash 24 | ./scripts/integration.sh build 25 | ./scripts/integration.sh start 26 | ``` 27 | 28 | ### Start - Red Hat 29 | 30 | **Note** Root-less podman is not supported. So run the integration script as 31 | root: 32 | 33 | ```bash 34 | sudo ./scripts/integration.sh start 35 | ``` 36 | 37 | ## Test 38 | 39 | To manually check the run, execute the following: 40 | 41 | 42 | * Check Prometheus at 43 | * Check Platform/Host BMC redfish server 44 | * Check NIC/DPU/IPU BMC redfish server 45 | * Check PXE server 46 | * Log into Host CPU: `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2210 host@127.0.0.1` 47 | * Log into Host BMC: `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2208 bmc@127.0.0.1` 48 | * Log into xPU CPU: `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2207 xpu@127.0.0.1` 49 | * Log into xPU BMC: `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2209 bmc@127.0.0.1` 50 | 51 | 52 | You can also run the CI tests and log collection as follows: 53 | 54 | ```bash 55 | ./scripts/integration.sh tests 56 | ./scripts/integration.sh logs 57 | ``` 58 | 59 | ## Stop 60 | 61 | ```bash 62 | ./scripts/integration.sh stop 63 | ``` 64 | 65 | ### Stop - Red Hat 66 | 67 | **Note** `stop` currently has an issue in this environment where you need to 68 | run it twice to fully clean up. 69 | 70 | ```bash 71 | sudo ./scripts/integration.sh stop 72 | sudo ./scripts/integration.sh stop 73 | ``` 74 | -------------------------------------------------------------------------------- /integration/docker-compose.host.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | services: 5 | 6 | host-cpu-ssh: 7 | image: linuxserver/openssh-server:8.8_p1-r1-ls84 8 | hostname: host-cpu 9 | environment: 10 | - PUID=1000 11 | - PGID=1000 12 | - TZ=Europe/London 13 | - PASSWORD_ACCESS=true 14 | - USER_PASSWORD=123456 15 | - USER_NAME=host 16 | ports: 17 | - '2210:2222' # ssh 18 | networks: 19 | high-speed-external: 20 | ipv4_address: 10.129.129.10 21 | host-ncsi: 22 | ipv4_address: 10.130.130.10 23 | healthcheck: 24 | test: ["CMD-SHELL", "timeout 5 bash -c ' /etc/dhcp/dhcpd.conf && dhcpd -d' 25 | 26 | nmap: 27 | image: docker.io/instrumentisto/nmap:7.94 28 | networks: 29 | - xpu-cpu 30 | command: --script broadcast-dhcp-discover 31 | 32 | web: 33 | image: docker.io/library/httpd:2.4.57-alpine3.17 34 | ports: 35 | - 8082:80 36 | volumes: 37 | - ./sztp/my-boot-image.img:/usr/local/apache2/htdocs/my-boot-image.img 38 | networks: 39 | xpu-cpu: 40 | ipv4_address: 10.127.127.16 41 | healthcheck: 42 | test: wget --silent --fail http://localhost:80 || exit 1 43 | 44 | bootstrap: 45 | image: ghcr.io/opiproject/opi-sztp-server:main@sha256:b50556916978fb6782bf71b7a40a1831f63cc1f34a432049ddfe34634cd2cc9d 46 | environment: 47 | SZTPD_INIT_PORT: 6080 48 | SZTPD_NBI_PORT: 7080 49 | SZTPD_SBI_PORT: 9090 50 | SZTPD_INIT_MODE: 1 51 | SZTPD_ACCEPT_CONTRACT: "Yes" 52 | SZTPD_INIT_ADDR: 0.0.0.0 53 | SZTPD_OPI_MODE: "running" 54 | SZTPD_RETRY_ATTEMPTS: 30 55 | networks: 56 | xpu-cpu: 57 | ipv4_address: 10.127.127.11 58 | healthcheck: 59 | test: ["CMD-SHELL", "curl --fail -H Accept:application/yang-data+json http://127.0.0.1:$$SZTPD_NBI_PORT/.well-known/host-meta || exit 1"] 60 | 61 | redirecter: 62 | image: ghcr.io/opiproject/opi-sztp-server:main@sha256:b50556916978fb6782bf71b7a40a1831f63cc1f34a432049ddfe34634cd2cc9d 63 | environment: 64 | SZTPD_INIT_PORT: 6080 65 | SZTPD_NBI_PORT: 7080 66 | SZTPD_SBI_PORT: 8080 67 | SZTPD_INIT_MODE: 1 68 | SZTPD_ACCEPT_CONTRACT: "Yes" 69 | SZTPD_INIT_ADDR: 0.0.0.0 70 | BOOTSVR_PORT: 9090 71 | BOOTSVR_ADDR: "bootstrap" 72 | SZTPD_OPI_MODE: "redirect" 73 | SZTPD_RETRY_ATTEMPTS: 30 74 | networks: 75 | xpu-cpu: 76 | ipv4_address: 10.127.127.12 77 | healthcheck: 78 | test: ["CMD-SHELL", "curl --fail -H Accept:application/yang-data+json http://127.0.0.1:$$SZTPD_NBI_PORT/.well-known/host-meta || exit 1"] 79 | 80 | dhclient: 81 | image: docker.io/modularitycontainers/dhcp-client:latest 82 | cap_add: 83 | - CAP_NET_RAW 84 | volumes: 85 | - dhcp-leases-folder:/var/lib/dhclient/ 86 | - ./sztp/dhclient.conf:/etc/dhcp/dhclient.conf 87 | network_mode: service:web 88 | command: dhclient -d -v 89 | 90 | agent: 91 | image: ghcr.io/opiproject/opi-sztp-client:main@sha256:3755b693cf268cc3d72829b92dc6dd9e47693285d0bffeaa631feb99be920676 92 | depends_on: 93 | bootstrap: 94 | condition: service_healthy 95 | redirecter: 96 | condition: service_healthy 97 | volumes_from: 98 | - dhclient:ro 99 | networks: 100 | - xpu-cpu 101 | command: ['/opi-sztp-agent', 'daemon'] 102 | 103 | volumes: 104 | dhcp-leases-folder: 105 | 106 | networks: 107 | xpu-cpu: 108 | external: true 109 | -------------------------------------------------------------------------------- /integration/docker-compose.spdk.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: "3.7" 3 | 4 | services: 5 | 6 | spdk-target: 7 | image: ghcr.io/opiproject/spdk:main@sha256:7b6ae4c0af27244525aa99cc66d96947baf206fe10475a2e8af095633801e906 8 | volumes: 9 | - /dev/hugepages:/dev/hugepages 10 | - /dev/shm:/dev/shm 11 | - /proc:/proc 12 | - /var/tmp/ 13 | privileged: true 14 | networks: 15 | high-speed-external: 16 | ipv4_address: 10.129.129.4 17 | working_dir: /usr/libexec/spdk/scripts 18 | command: | 19 | sh -x -c 'sync; echo 1 > /proc/sys/vm/drop_caches && \ 20 | mkdir -p /mnt/huge && \ 21 | grep hugetlbfs /proc/mounts || mount -t hugetlbfs nodev /mnt/huge && \ 22 | echo 1024 > /proc/sys/vm/nr_hugepages && \ 23 | grep "" /sys/kernel/mm/hugepages/hugepages-*/nr_hugepages && \ 24 | /usr/local/bin/spdk_tgt -m 0x1 -s 512 --no-pci 2>&1 & \ 25 | for i in `seq 1 10`; do ./rpc.py spdk_get_version && break || sleep 1; done && \ 26 | ./rpc.py bdev_malloc_create -b Malloc0 64 512 && \ 27 | ./rpc.py bdev_malloc_create -b Malloc1 64 512 && \ 28 | ./rpc.py nvmf_create_transport -t TCP -u 8192 -m 4 -c 0 && \ 29 | ./rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -d SPDK_Controller1 && \ 30 | ./rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t tcp -a 10.129.129.4 -s 4420 && \ 31 | ./rpc.py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 -n 1 && \ 32 | ./rpc.py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 -n 2 && \ 33 | wait' 34 | healthcheck: 35 | test: ["CMD-SHELL", "python3 /usr/libexec/spdk/scripts/rpc.py spdk_get_version || exit 1"] 36 | interval: 6s 37 | retries: 5 38 | start_period: 20s 39 | timeout: 10s 40 | 41 | spdk-target-web: 42 | image: ghcr.io/opiproject/spdk:main@sha256:7b6ae4c0af27244525aa99cc66d96947baf206fe10475a2e8af095633801e906 43 | volumes_from: 44 | - spdk-target:rw 45 | ports: 46 | - "9004:9009" 47 | networks: 48 | high-speed-external: 49 | ipv4_address: 10.129.129.8 50 | working_dir: /usr/libexec/spdk/scripts 51 | command: | 52 | sh -x -c 'sync && \ 53 | for i in `seq 1 10`; do ./rpc.py spdk_get_version && break || sleep 1; done && \ 54 | ./rpc_http_proxy.py 0.0.0.0 9009 spdkuser spdkpass' 55 | healthcheck: 56 | test: ["CMD-SHELL", "curl --fail --insecure --user spdkuser:spdkpass -X POST -H 'Content-Type: application/json' -d '{\"id\": 1, \"method\": \"bdev_get_bdevs\"}' http://localhost:9009 || exit 1"] 57 | interval: 6s 58 | retries: 5 59 | start_period: 20s 60 | timeout: 10s 61 | 62 | networks: 63 | high-speed-external: 64 | external: true 65 | -------------------------------------------------------------------------------- /integration/docker-compose.xpu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | services: 5 | 6 | xpu-cpu-ssh: 7 | image: linuxserver/openssh-server:8.8_p1-r1-ls84 8 | hostname: xpu-cpu 9 | environment: 10 | - PUID=1000 11 | - PGID=1000 12 | - TZ=Europe/London 13 | - PASSWORD_ACCESS=true 14 | - USER_PASSWORD=123456 15 | - USER_NAME=xpu 16 | ports: 17 | - '2207:2222' # ssh 18 | - "9009:9009" # spdk 19 | - "127.0.0.1:8011:8001/tcp" 20 | - "127.0.0.1:8444:8444/tcp" 21 | networks: 22 | xpu-cpu: 23 | ipv4_address: 10.127.127.7 24 | high-speed-external: 25 | ipv4_address: 10.129.129.7 26 | xpu-ncsi: 27 | ipv4_address: 10.131.131.7 28 | healthcheck: 29 | test: ["CMD-SHELL", "timeout 5 bash -c ' /proc/sys/vm/drop_caches && \ 72 | mkdir -p /mnt/huge && \ 73 | grep hugetlbfs /proc/mounts || mount -t hugetlbfs nodev /mnt/huge && \ 74 | echo 1024 > /proc/sys/vm/nr_hugepages && \ 75 | grep "" /sys/kernel/mm/hugepages/hugepages-*/nr_hugepages && \ 76 | /usr/local/bin/spdk_tgt -m 0x1 -s 512 --no-pci 2>&1 & \ 77 | for i in `seq 1 10`; do ./rpc.py spdk_get_version && break || sleep 1; done && \ 78 | ./rpc.py bdev_malloc_create -b Malloc0 64 512 && \ 79 | ./rpc.py bdev_malloc_create -b Malloc1 64 512 && \ 80 | echo ./rpc.py bdev_nvme_attach_controller -b opi-bdev -t tcp -a spdk-target -f ipv4 -s 4420 -n nqn.2016-06.io.spdk:cnode1 && \ 81 | wait' 82 | healthcheck: 83 | test: ["CMD-SHELL", "python3 /usr/libexec/spdk/scripts/rpc.py spdk_get_version || exit 1"] 84 | interval: 6s 85 | retries: 5 86 | start_period: 20s 87 | timeout: 10s 88 | 89 | xpu-spdk-web: 90 | image: ghcr.io/opiproject/spdk:main@sha256:7b6ae4c0af27244525aa99cc66d96947baf206fe10475a2e8af095633801e906 91 | volumes_from: 92 | - xpu-spdk:rw 93 | network_mode: service:xpu-cpu-ssh 94 | working_dir: /usr/libexec/spdk/scripts 95 | command: | 96 | sh -x -c 'sync && \ 97 | for i in `seq 1 10`; do ./rpc.py spdk_get_version && break || sleep 1; done && \ 98 | ./rpc_http_proxy.py 0.0.0.0 9009 spdkuser spdkpass' 99 | healthcheck: 100 | test: ["CMD-SHELL", "curl --fail --insecure --user spdkuser:spdkpass -X POST -H 'Content-Type: application/json' -d '{\"id\": 1, \"method\": \"bdev_get_bdevs\"}' http://localhost:9009 || exit 1"] 101 | interval: 6s 102 | retries: 5 103 | start_period: 20s 104 | timeout: 10s 105 | 106 | strongswan: 107 | image: strongx509/strongswan:5.9.13 108 | cap_add: 109 | - NET_ADMIN 110 | - SYS_ADMIN 111 | - SYS_MODULE 112 | stdin_open: true 113 | tty: true 114 | volumes: 115 | - ./strongswan/server:/etc/swanctl:z 116 | - ./strongswan/strongswan.conf:/etc/strongswan.conf:z 117 | network_mode: service:xpu-cpu-ssh 118 | command: './charon' 119 | healthcheck: 120 | test: swanctl --stats || exit 1 121 | 122 | xpu-bmc-ssh: 123 | image: linuxserver/openssh-server:8.8_p1-r1-ls84 124 | hostname: xpu-bmc 125 | environment: 126 | - PUID=1000 127 | - PGID=1000 128 | - TZ=Europe/London 129 | - PASSWORD_ACCESS=true 130 | - USER_PASSWORD=123456 131 | - USER_NAME=bmc 132 | ports: 133 | - '2209:2222' # ssh 134 | - '8002:8000' # redfish 135 | networks: 136 | bmc: 137 | ipv4_address: 10.128.128.9 138 | xpu-ncsi: 139 | ipv4_address: 10.131.131.9 140 | healthcheck: 141 | test: ["CMD-SHELL", "timeout 5 bash -c ' 5 | 6 | And two example services: network (in python) and storage (in go) 7 | 8 | And two example clients that send gRPC request via `nginx` gRPC API gataway to network and storage services. 9 | 10 | ## Run 11 | 12 | :exclamation: `docker-compose` is deprecated. For details, see [Migrate to Compose V2](https://docs.docker.com/compose/migrate/). 13 | 14 | ```text 15 | docker-compose up --build example-storage-client example-network-client 16 | ``` 17 | 18 | ## Scale 19 | 20 | TBD: how to update ngnix configuration. 21 | 22 | ```text 23 | docker-compose up --scale example-network=3 --scale example-storage=2 gateway 24 | ``` 25 | 26 | ## Security 27 | 28 | TBD 29 | 30 | ## Authentication 31 | 32 | TBD 33 | 34 | ## Monitoring 35 | 36 | TBD via OTEL 37 | -------------------------------------------------------------------------------- /integration/gateway/errors.grpc_conf: -------------------------------------------------------------------------------- 1 | # Standard HTTP-to-gRPC status code mappings 2 | # Ref: https://github.com/grpc/grpc/blob/master/doc/http-grpc-status-mapping.md 3 | # 4 | error_page 400 = @grpc_internal; 5 | error_page 401 = @grpc_unauthenticated; 6 | error_page 403 = @grpc_permission_denied; 7 | error_page 404 = @grpc_unimplemented; 8 | error_page 429 = @grpc_unavailable; 9 | error_page 502 = @grpc_unavailable; 10 | error_page 503 = @grpc_unavailable; 11 | error_page 504 = @grpc_unavailable; 12 | 13 | # NGINX-to-gRPC status code mappings 14 | # Ref: https://github.com/grpc/grpc/blob/master/doc/statuscodes.md 15 | # 16 | error_page 405 = @grpc_internal; # Method not allowed 17 | error_page 408 = @grpc_deadline_exceeded; # Request timeout 18 | error_page 413 = @grpc_resource_exhausted; # Payload too large 19 | error_page 414 = @grpc_resource_exhausted; # Request URI too large 20 | error_page 415 = @grpc_internal; # Unsupported media type; 21 | error_page 426 = @grpc_internal; # HTTP request was sent to HTTPS port 22 | error_page 495 = @grpc_unauthenticated; # Client certificate authentication error 23 | error_page 496 = @grpc_unauthenticated; # Client certificate not presented 24 | error_page 497 = @grpc_internal; # HTTP request was sent to mutual TLS port 25 | error_page 500 = @grpc_internal; # Server error 26 | error_page 501 = @grpc_internal; # Not implemented 27 | 28 | # gRPC error responses 29 | # Ref: https://github.com/grpc/grpc-go/blob/master/codes/codes.go 30 | # 31 | location @grpc_deadline_exceeded { 32 | add_header grpc-status 4; 33 | add_header grpc-message 'deadline exceeded'; 34 | return 204; 35 | } 36 | 37 | location @grpc_permission_denied { 38 | add_header grpc-status 7; 39 | add_header grpc-message 'permission denied'; 40 | return 204; 41 | } 42 | 43 | location @grpc_resource_exhausted { 44 | add_header grpc-status 8; 45 | add_header grpc-message 'resource exhausted'; 46 | return 204; 47 | } 48 | 49 | location @grpc_unimplemented { 50 | add_header grpc-status 12; 51 | add_header grpc-message unimplemented; 52 | return 204; 53 | } 54 | 55 | location @grpc_internal { 56 | add_header grpc-status 13; 57 | add_header grpc-message 'internal error'; 58 | return 204; 59 | } 60 | 61 | location @grpc_unavailable { 62 | add_header grpc-status 14; 63 | add_header grpc-message unavailable; 64 | return 204; 65 | } 66 | 67 | location @grpc_unauthenticated { 68 | add_header grpc-status 16; 69 | add_header grpc-message unauthenticated; 70 | return 204; 71 | } 72 | 73 | # vim: syntax=nginx -------------------------------------------------------------------------------- /integration/gateway/grpc_gateway.conf: -------------------------------------------------------------------------------- 1 | log_format grpc_json escape=json '{"timestamp":"$time_iso8601",' 2 | '"client":"$remote_addr","uri":"$uri","http-status":$status,' 3 | '"grpc-status":$grpc_status,"upstream":"$upstream_addr"' 4 | '"rx-bytes":$request_length,"tx-bytes":$bytes_sent}'; 5 | 6 | map $upstream_trailer_grpc_status $grpc_status { 7 | default $upstream_trailer_grpc_status; # grpc-status is usually a trailer 8 | '' $sent_http_grpc_status; # Else use the header, whatever its source 9 | } 10 | 11 | server { 12 | listen 50051 http2; # In production, comment out to disable plaintext port 13 | server_name grpc.example.com; 14 | access_log /var/log/nginx/grpc_log.json grpc_json; 15 | 16 | # Routing 17 | location /routeguide. { 18 | grpc_pass grpc://network_service; 19 | } 20 | location /helloworld. { 21 | grpc_pass grpc://storage_service; 22 | } 23 | 24 | # Error responses 25 | include conf.d/errors.grpc_conf; # gRPC-compliant error responses 26 | default_type application/grpc; # Ensure gRPC for all error responses 27 | } 28 | 29 | # Backend gRPC servers 30 | # 31 | upstream network_service { 32 | zone network_service 64k; 33 | server example-network:50051; 34 | } 35 | 36 | upstream storage_service { 37 | zone storage_service 64k; 38 | server example-storage:50051; 39 | } 40 | 41 | # vim: syntax=nginx -------------------------------------------------------------------------------- /integration/host-cpu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/fedora:37 2 | 3 | RUN dnf install -y fio nvme-cli nmap tftp && dnf clean all 4 | -------------------------------------------------------------------------------- /integration/host-cpu/Dockerfile.telegraf: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/telegraf:1.29.2 2 | COPY telegraf-redfish.conf /etc/telegraf/telegraf.conf 3 | -------------------------------------------------------------------------------- /integration/host-cpu/README.md: -------------------------------------------------------------------------------- 1 | # Host 2 | 3 | Host is the server where xPU is pugged into. 4 | 5 | Real user Apps/Vms/Containers are runnig on the host. 6 | 7 | In our case we runing iperf for networking and fio for storage, for exmaple. 8 | 9 | ## Host Main CPU 10 | 11 | Runs nvme-cli and fio and some DB, example: 12 | 13 | ```text 14 | $ nvme list 15 | Node SN Model Namespace Usage Format FW Rev 16 | --------------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- 17 | /dev/nvme0n1 71T0A0CETC88 Dell Ent NVMe CM6 MU 1.6TB 1 14.79 GB / 1.60 TB 512 B + 0 B 2.1.3 18 | /dev/nvme1n1 71T0A0BGTC88 Dell Ent NVMe CM6 MU 1.6TB 1 888.31 MB / 1.60 TB 512 B + 0 B 2.1.3 19 | /dev/nvme2n1 71T0A0C3TC88 Dell Ent NVMe CM6 MU 1.6TB 1 524.29 kB / 1.60 TB 512 B + 0 B 2.1.3 20 | /dev/nvme3n1 71T0A0C9TC88 Dell Ent NVMe CM6 MU 1.6TB 1 524.29 kB / 1.60 TB 512 B + 0 B 2.1.3 21 | ``` 22 | 23 | fio example: 24 | 25 | ```text 26 | sudo fio --direct=1 --prio=0 --norandommap=1 --group_reporting --cpus_allowed_policy=split --ioengine=libaio --rw=randrw --rwmixread=100 --bs=4096 --runtime=100 --numjobs=16 --iodepth=4 --name=filename1 --filename=/dev/nvme0n1 --name=filename2 --filename=/dev/nvme0n2 --name=filename3 --filename=/dev/nvme0n3 27 | ``` 28 | 29 | ssh example: 30 | 31 | ```text 32 | $ ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2210 host@127.0.0.1 33 | Warning: Permanently added '[127.0.0.1]:2210' (ECDSA) to the list of known hosts. 34 | host@127.0.0.1's password: 35 | Welcome to OpenSSH Server 36 | 37 | host-cpu:~$ exit 38 | logout 39 | Connection to 127.0.0.1 closed. 40 | ``` 41 | 42 | ## Host/Platform BMC 43 | 44 | Runs redfish server, example: 45 | 46 | ```text 47 | $ curl --fail http://127.0.0.1:8001/redfish/v1 { 48 | "@odata.id": "/redfish/v1/", 49 | "@odata.type": "#ServiceRoot.v1_6_0.ServiceRoot", 50 | "AccountService": { 51 | "@odata.id": "/redfish/v1/AccountService" 52 | }, 53 | "CertificateService": { 54 | "@odata.id": "/redfish/v1/CertificateService" 55 | }, 56 | "Chassis": { 57 | "@odata.id": "/redfish/v1/Chassis" 58 | }, 59 | "EventService": { 60 | "@odata.id": "/redfish/v1/EventService" 61 | }, 62 | "Id": "RootService", 63 | "Links": { 64 | "Sessions": { 65 | "@odata.id": "/redfish/v1/SessionService/Sessions" 66 | } 67 | }, 68 | "Managers": { 69 | "@odata.id": "/redfish/v1/Managers" 70 | }, 71 | "Name": "Root Service", 72 | "Oem": {}, 73 | "RedfishVersion": "1.6.0", 74 | "Registries": { 75 | "@odata.id": "/redfish/v1/Registries" 76 | }, 77 | "SessionService": { 78 | "@odata.id": "/redfish/v1/SessionService" 79 | }, 80 | "Systems": { 81 | "@odata.id": "/redfish/v1/Systems" 82 | }, 83 | "Tasks": { 84 | "@odata.id": "/redfish/v1/TaskService" 85 | }, 86 | "UUID": "92384634-2938-2342-8820-489239905423", 87 | "UpdateService": { 88 | "@odata.id": "/redfish/v1/UpdateService" 89 | } 90 | } 91 | ``` 92 | 93 | ssh example: 94 | 95 | ```text 96 | $ ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2208 bmc@127.0.0.1 97 | Warning: Permanently added '[127.0.0.1]:2208' (ECDSA) to the list of known hosts. 98 | bmc@127.0.0.1's password: 99 | Welcome to OpenSSH Server 100 | 101 | host-bmc:~$ exit 102 | logout 103 | Connection to 127.0.0.1 closed. 104 | ``` 105 | -------------------------------------------------------------------------------- /integration/host-cpu/telegraf-redfish.conf: -------------------------------------------------------------------------------- 1 | [[inputs.redfish]] 2 | address = "http://host-bmc-ssh:8000" 3 | username = "root" 4 | password = "password123456" 5 | computer_system_id="437XR1138R2" 6 | 7 | [[inputs.cpu]] 8 | percpu = true 9 | totalcpu = true 10 | collect_cpu_time = false 11 | report_active = false 12 | 13 | [[inputs.mem]] 14 | # no configuration 15 | 16 | [[inputs.net]] 17 | ignore_protocol_stats = false 18 | 19 | [[outputs.file]] 20 | files = ["stdout"] 21 | data_format = "influx" 22 | 23 | [[outputs.opentelemetry]] 24 | service_address = "otel-gw-collector:4317" 25 | -------------------------------------------------------------------------------- /integration/otel/README.md: -------------------------------------------------------------------------------- 1 | # Open Telemetry 2 | 3 | Telegraf (agent) still runs inside DPU/IPU and connects to OTEL gateway. Later on replace it with OTEL agent. 4 | 5 | OTEL gateway collector runs outside of DPU/IPU and connects to backends like ELK, Jaeger, Zipkin and Prometheus back-ends... 6 | 7 | ## Docs 8 | 9 | * 10 | * 11 | 12 | * Dockers used 13 | * 14 | * 15 | 16 | ## Getting started 17 | 18 | :exclamation: `docker-compose` is deprecated. For details, see [Migrate to Compose V2](https://docs.docker.com/compose/migrate/). 19 | 20 | Run `docker-compose -f docker-compose.otel.yml up` 21 | 22 | ## Example 23 | 24 | * Prometheus at 25 | -------------------------------------------------------------------------------- /integration/otel/otel-collector-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | receivers: 3 | otlp: 4 | protocols: 5 | grpc: 6 | 7 | exporters: 8 | prometheus: 9 | endpoint: "0.0.0.0:8889" 10 | const_labels: 11 | label1: value1 12 | 13 | logging: 14 | 15 | processors: 16 | batch: 17 | 18 | extensions: 19 | health_check: 20 | pprof: 21 | endpoint: :1888 22 | zpages: 23 | endpoint: :55679 24 | 25 | service: 26 | extensions: [pprof, zpages, health_check] 27 | pipelines: 28 | traces: 29 | receivers: [otlp] 30 | processors: [batch] 31 | exporters: [logging] 32 | metrics: 33 | receivers: [otlp] 34 | processors: [batch] 35 | exporters: [logging, prometheus] 36 | -------------------------------------------------------------------------------- /integration/otel/prometheus.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | scrape_configs: 3 | - job_name: 'otel-collector' 4 | scrape_interval: 10s 5 | static_configs: 6 | - targets: ['otel-gw-collector:8889'] 7 | - targets: ['otel-gw-collector:8888'] 8 | -------------------------------------------------------------------------------- /integration/scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # SPDX-License-Identifier: Apache-2.0 3 | # Copyright (c) 2022 Intel Corporation 4 | # 5 | # This script is used to deploy the OPI PoC into one of two 6 | # environments: 7 | # 8 | # * Developer emulated xPU setup 9 | # * Split setup between an actual xPU and a laptop/VM/server 10 | # 11 | # This script assumes you have Docker installed on the xPU and laptop/VM/server 12 | # instances. 13 | # 14 | 15 | set -eo pipefail 16 | 17 | # Enable debug if running in CI 18 | if [ -n "${CI}" ] && [ "${CI}" == "true" ] 19 | then 20 | set -x 21 | fi 22 | 23 | # DOCKER_COMPOSE setup 24 | DC=docker-compose 25 | 26 | if [ "$(command -v ${DC})" == "" ] 27 | then 28 | DC="docker compose" 29 | fi 30 | 31 | usage() { 32 | echo "" 33 | echo "Usage: $0 -m [dev | xpu] -b [BMC IP address] -h [HOST IP address] -i [IP address of the infra server] -x [IP address of the xPU]" 34 | echo "" 35 | exit 1 36 | } 37 | 38 | deploy_dev() { 39 | echo "Deploying development emulated environment" 40 | 41 | # Run the integration script to start things up 42 | ./scripts/integration.sh start 43 | } 44 | 45 | deploy_xpu() { 46 | echo "Deploying xPU environment" 47 | echo "BMC IP address: ${BMC_IP_ADDRESS}" 48 | echo "Host IP address: ${HOST_IP_ADDRESS}" 49 | echo "Infra IP address: ${INFRA_IP_ADDRESS}" 50 | echo "xPU IP address: ${XPU_IP_ADDRESS}" 51 | 52 | # Deploy to laptop/VM/server 53 | export COMPOSE_FILE=docker-compose.otel.yml:docker-compose.spdk.yml:docker-compose.pxe.yml:docker-compose.networks.yml 54 | [[ -n ${INFRA_IP_ADDRESS} ]] && export DOCKER_HOST="ssh://ubuntu@${INFRA_IP_ADDRESS}" 55 | bash -c "${DC} up -d" 56 | 57 | # Deploy to x86 HOST where xPU is plugged into (were actual apps are running like iperf, fio, ...) 58 | export COMPOSE_FILE=docker-compose.host.yml:docker-compose.networks.yml 59 | [[ -n ${HOST_IP_ADDRESS} ]] && export DOCKER_HOST="ssh://root@${HOST_IP_ADDRESS}" 60 | bash -c "${DC} up -d" 61 | 62 | # Deploy to XPU 63 | export COMPOSE_FILE=docker-compose.xpu.yml:docker-compose.networks.yml 64 | [[ -n ${XPU_IP_ADDRESS} ]] && export DOCKER_HOST="ssh://ubuntu@${XPU_IP_ADDRESS}" 65 | bash -c "${DC} up -d" 66 | } 67 | 68 | # Default mode is dev 69 | # Valid values: "dev" or "xpu" 70 | MODE=dev 71 | 72 | # IP addresses, only used for xpu mode 73 | XPU_IP_ADDRESS= 74 | INFRA_IP_ADDRESS= 75 | BMC_IP_ADDRESS= 76 | 77 | while getopts b:h:i:m:x: option 78 | do 79 | case "${option}" 80 | in 81 | b) 82 | BMC_IP_ADDRESS="${OPTARG}" 83 | echo "-b not in use for now" 84 | usage 85 | ;; 86 | h) 87 | HOST_IP_ADDRESS="${OPTARG}" 88 | ;; 89 | i) 90 | INFRA_IP_ADDRESS="${OPTARG}" 91 | ;; 92 | m) 93 | MODE="${OPTARG}" 94 | ;; 95 | x) 96 | XPU_IP_ADDRESS="${OPTARG}" 97 | ;; 98 | *) 99 | usage 100 | ;; 101 | esac 102 | done 103 | 104 | if [ "${MODE}" != "dev" ] && [ "${MODE}" != "xpu" ] 105 | then 106 | usage 107 | fi 108 | 109 | echo "Selected mode ${MODE}" 110 | 111 | if [ "${MODE}" == "xpu" ] 112 | then 113 | deploy_xpu 114 | elif [ "${MODE}" == "dev" ] 115 | then 116 | deploy_dev 117 | fi 118 | -------------------------------------------------------------------------------- /integration/scripts/integration.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # SPDX-License-Identifier: Apache-2.0 3 | # Copyright (c) 2022 Intel Corporation 4 | 5 | set -euxo pipefail 6 | 7 | INT_BUILD=build 8 | INT_START=start 9 | INT_TESTS=tests 10 | INT_LOGS=logs 11 | INT_STOP=stop 12 | 13 | # docker compose plugin 14 | command -v docker-compose || { shopt -s expand_aliases && alias docker-compose='docker compose'; } 15 | 16 | usage() { 17 | echo "" 18 | echo "Usage: integration.sh [${INT_BUILD} | ${INT_START} | ${INT_TESTS} | ${INT_LOGS} | ${INT_STOP}]" 19 | echo "" 20 | } 21 | 22 | build_containers() { 23 | docker-compose build --parallel 24 | } 25 | 26 | start_containers() { 27 | docker-compose down 28 | docker network prune --force 29 | docker-compose pull 30 | docker-compose up -d 31 | } 32 | 33 | run_integration_tests() { 34 | docker-compose ps 35 | # shellcheck disable=SC2046 36 | uniq -c <<< "$(sort <<< "$(docker inspect --format='{{json .State.Health.Status}}' $(docker-compose ps -q))")" 37 | 38 | # TODO: replace sleep with timeouted-wait for all services to become healthy 39 | echo wait 5s... && sleep 5s 40 | curl --fail http://127.0.0.1:8001/redfish/v1/Systems/437XR1138R2 41 | curl --fail http://127.0.0.1:8002/redfish/v1/Systems/437XR1138R2 42 | curl --fail http://127.0.0.1:8082/ 43 | 44 | for i in $(seq 1 20) 45 | do 46 | echo "${i}" 47 | if [[ "$(curl --fail --insecure --user spdkuser:spdkpass -X POST -H 'Content-Type: application/json' -d '{"id": 1, "method": "spdk_get_version"}' http://127.0.0.1:9004)" ]] 48 | then 49 | break 50 | else 51 | sleep 1 52 | fi 53 | done 54 | for i in $(seq 1 20) 55 | do 56 | echo "$i" 57 | if [[ "$(curl --fail --insecure --user spdkuser:spdkpass -X POST -H 'Content-Type: application/json' -d '{"id": 1, "method": "spdk_get_version"}' http://127.0.0.1:9009)" ]] 58 | then 59 | break 60 | else 61 | sleep 1 62 | fi 63 | done 64 | curl --fail --insecure --user spdkuser:spdkpass -X POST -H 'Content-Type: application/json' -d '{"id": 1, "method": "bdev_get_bdevs"}' http://127.0.0.1:9004 65 | curl --fail --insecure --user spdkuser:spdkpass -X POST -H 'Content-Type: application/json' -d '{"id": 1, "method": "bdev_get_bdevs"}' http://127.0.0.1:9009 66 | sshpass -p 123456 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2210 host@127.0.0.1 hostname 67 | sshpass -p 123456 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2208 bmc@127.0.0.1 hostname 68 | sshpass -p 123456 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2207 xpu@127.0.0.1 hostname 69 | sshpass -p 123456 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2209 bmc@127.0.0.1 hostname 70 | 71 | docker-compose exec -T dhcp cat /var/lib/dhcp/dhcpd.leases 72 | docker-compose run nmap 73 | docker-compose run nmap | grep "Server Identifier: 10.127.127.3" 74 | # docker-compose exec -w /tmp/sztpd-simulator -T bootstrap ./run-sztpd-test.sh 75 | docker-compose exec -T redirecter curl --fail -H 'Accept:application/yang-data+json' http://127.0.0.1:7080/.well-known/host-meta 76 | docker-compose exec -T bootstrap curl --fail -H 'Accept:application/yang-data+json' http://127.0.0.1:7080/.well-known/host-meta 77 | 78 | docker-compose exec -T spdk-target /usr/local/bin/spdk_nvme_identify -r 'traddr:10.129.129.4 trtype:TCP adrfam:IPv4 trsvcid:4420' 79 | docker-compose exec -T xpu-spdk /usr/local/bin/spdk_nvme_identify -r 'traddr:10.129.129.4 trtype:TCP adrfam:IPv4 trsvcid:4420' 80 | docker-compose exec -T spdk-target /usr/local/bin/spdk_nvme_perf -r 'traddr:10.129.129.4 trtype:TCP adrfam:IPv4 trsvcid:4420' -c 0x1 -q 1 -o 4096 -w randread -t 10 81 | docker-compose exec -T xpu-spdk /usr/local/bin/spdk_nvme_perf -r 'traddr:10.129.129.4 trtype:TCP adrfam:IPv4 trsvcid:4420' -c 0x1 -q 1 -o 4096 -w randread -t 10 82 | 83 | curl --fail http://127.0.0.1:9091/api/v1/query?query=mem_free | grep mem_free 84 | curl --fail http://127.0.0.1:9091/api/v1/query?query=cpu_usage_user | grep cpu_usage_user 85 | curl --fail http://127.0.0.1:9091/api/v1/query?query=xpu_num_blocks | grep xpu_num_blocks 86 | curl --fail http://127.0.0.1:9091/api/v1/query?query=net_bytes_recv_total | grep net_bytes_recv_total 87 | curl --fail http://127.0.0.1:9091/api/v1/query?query=redfish_thermal_fans_reading_rpm | grep redfish_thermal_fans_reading_rpm 88 | 89 | # TODO Workaround for agent getting connection refused from the redirecter 90 | SZTP_AGENT_NAME=$(docker-compose ps | grep agent | awk '{print $1}') 91 | for i in {1..2}; do 92 | SZTP_AGENT_RC=$(docker wait "${SZTP_AGENT_NAME}") 93 | if [ "${SZTP_AGENT_RC}" != "0" ]; then 94 | set +e +o pipefail 95 | docker-compose logs agent | grep -q 'connection refused' 96 | CONN_REFUSED=$? 97 | set -e -o pipefail 98 | if [ "$i" == "1" ] && [ "$CONN_REFUSED" == "0" ]; then 99 | # Do exactly one retry for connection refused errors 100 | echo "${SZTP_AGENT_NAME} got connection refused error, retry" 101 | docker-compose up -d agent 102 | continue 103 | fi 104 | echo "${SZTP_AGENT_NAME} failed:" 105 | docker logs "${SZTP_AGENT_NAME}" 106 | exit 1 107 | else 108 | break 109 | fi 110 | done 111 | 112 | NETWORK_CLIENT_NAME=$(docker-compose ps | grep example-network-client | awk '{print $1}') 113 | NETWORK_CLIENT_RC=$(docker inspect --format '{{.State.ExitCode}}' "${NETWORK_CLIENT_NAME}") 114 | if [ "${NETWORK_CLIENT_RC}" != "0" ]; then 115 | echo "example-network-client failed:" 116 | docker logs "${NETWORK_CLIENT_NAME}" 117 | exit 1 118 | fi 119 | STORAGE_CLIENT_NAME=$(docker-compose ps | grep example-storage-client | awk '{print $1}') 120 | STORAGE_CLIENT_RC=$(docker inspect --format '{{.State.ExitCode}}' "${STORAGE_CLIENT_NAME}") 121 | if [ "${STORAGE_CLIENT_RC}" != "0" ]; then 122 | echo "example-storage-client failed:" 123 | docker logs "${STORAGE_CLIENT_NAME}" 124 | exit 1 125 | fi 126 | docker-compose exec -T strongswan swanctl --stats 127 | docker-compose exec -T strongswan swanctl --list-sas 128 | 129 | 130 | 131 | # This should be last 132 | docker-compose ps 133 | # shellcheck disable=SC2046 134 | uniq -c <<< "$(sort <<< "$(docker inspect --format='{{json .State.Health.Status}}' $(docker-compose ps -q))")" 135 | } 136 | 137 | acquire_logs() { 138 | docker-compose ps -a 139 | docker-compose logs || true 140 | netstat -an || true 141 | ifconfig -a || true 142 | } 143 | 144 | stop_containers() { 145 | docker-compose down --volumes 146 | } 147 | 148 | if [ "$#" -lt 1 ] 149 | then 150 | usage 151 | exit 1 152 | fi 153 | 154 | if [ "$1" == "${INT_BUILD}" ] 155 | then 156 | build_containers 157 | elif [ "$1" == "${INT_START}" ] 158 | then 159 | start_containers 160 | elif [ "$1" == "${INT_TESTS}" ] 161 | then 162 | run_integration_tests 163 | elif [ "$1" == "${INT_LOGS}" ] 164 | then 165 | acquire_logs 166 | elif [ "$1" == "${INT_STOP}" ] 167 | then 168 | stop_containers 169 | else 170 | echo "Invalid argument: $1" 171 | usage 172 | exit 1 173 | fi 174 | -------------------------------------------------------------------------------- /integration/strongswan/server/eap.conf: -------------------------------------------------------------------------------- 1 | eap { 2 | pools = rw_pool 3 | 4 | local { 5 | auth = pubkey 6 | certs = serverCert.pem 7 | id = server.strongswan.org 8 | } 9 | remote { 10 | auth = eap-dynamic 11 | eap_id = %any 12 | } 13 | children { 14 | eap { 15 | local_ts = 10.1.0.0/24,192.168.0.2 16 | 17 | esp_proposals = aes256gcm128-chacha20poly1305-x25519 18 | dpd_action = clear 19 | } 20 | } 21 | version = 2 22 | proposals = aes256-sha256-x25519 23 | dpd_delay = 60s 24 | } 25 | -------------------------------------------------------------------------------- /integration/strongswan/server/ecdsa/serverKey.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MIGkAgEBBDDadusbSy1INVfd3rvhjJuw1MuYh6FiYnlxDtLoHtbQMMtxDAjwSl6V 3 | cGj/dBYoJgSgBwYFK4EEACKhZANiAASvt8McyFuyGVggth+Izf/qB+SQHgKxHEgv 4 | AB+6Gj52xrcdxZl0/cXwL5NG3rxur3dzBzEuJRb/oYxqgNcZrT/28239tAN8PHkS 5 | T0u+kFwZF3PTDXdyrkGT3PKv7Kb6dWU= 6 | -----END EC PRIVATE KEY----- 7 | -------------------------------------------------------------------------------- /integration/strongswan/server/psk.conf: -------------------------------------------------------------------------------- 1 | psk { 2 | pools = rw_pool 3 | 4 | local { 5 | auth = psk 6 | id = server.strongswan.org 7 | } 8 | remote { 9 | auth = psk 10 | } 11 | children { 12 | psk { 13 | local_ts = 10.1.0.0/24 14 | 15 | esp_proposals = aes256gcm128-chacha20poly1305-x25519 16 | dpd_action = clear 17 | } 18 | } 19 | version = 2 20 | proposals = aes256-sha256-x25519 21 | dpd_delay = 60s 22 | } 23 | -------------------------------------------------------------------------------- /integration/strongswan/server/rw.conf: -------------------------------------------------------------------------------- 1 | rw { 2 | pools = rw_pool 3 | 4 | local { 5 | auth = pubkey 6 | certs = serverCert.pem 7 | id = server.strongswan.org 8 | } 9 | remote { 10 | auth = pubkey 11 | cacerts = caCert.pem 12 | } 13 | children { 14 | net { 15 | local_ts = 10.1.0.0/24 16 | 17 | esp_proposals = aes256gcm128-chacha20poly1305-x25519 18 | dpd_action = clear 19 | } 20 | host { 21 | esp_proposals = aes256gcm128-chacha20poly1305-x25519 22 | dpd_action = clear 23 | } 24 | } 25 | version = 2 26 | proposals = aes256-sha256-x25519 27 | dpd_delay = 60s 28 | } 29 | -------------------------------------------------------------------------------- /integration/strongswan/server/swanctl.conf: -------------------------------------------------------------------------------- 1 | connections { 2 | 3 | include rw.conf 4 | include psk.conf 5 | include eap.conf 6 | } 7 | 8 | pools { 9 | 10 | rw_pool { 11 | addrs = 10.227.227.0/24 12 | } 13 | } 14 | 15 | secrets { 16 | 17 | ike-jane { 18 | id = jane@strongswan.org 19 | secret = 0sTtd7IOin6FuyjLsWtiM9o/1c 20 | } 21 | ike-hacker { 22 | id = hacker@strongswan.org 23 | secret = 0sH7+o6wysMoaELv5OBihKVa3F 24 | } 25 | eap-jane { 26 | id = jane 27 | secret = 3s9RFGdWE5EW 28 | } 29 | eap-hacker { 30 | id = hacker 31 | secret = K8FW9/N0VIAJ 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /integration/strongswan/server/x509/serverCert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICADCCAYWgAwIBAgIIWpEKfsh+QqYwCgYIKoZIzj0EAwMwNTELMAkGA1UEBhMC 3 | Q0gxDjAMBgNVBAoTBUN5YmVyMRYwFAYDVQQDEw1DeWJlciBSb290IENBMB4XDTIw 4 | MDMwOTEyMDMzOFoXDTI0MDMwOTEyMDMzOFowPTELMAkGA1UEBhMCQ0gxDjAMBgNV 5 | BAoTBUN5YmVyMR4wHAYDVQQDExVzZXJ2ZXIuc3Ryb25nc3dhbi5vcmcwdjAQBgcq 6 | hkjOPQIBBgUrgQQAIgNiAASvt8McyFuyGVggth+Izf/qB+SQHgKxHEgvAB+6Gj52 7 | xrcdxZl0/cXwL5NG3rxur3dzBzEuJRb/oYxqgNcZrT/28239tAN8PHkST0u+kFwZ 8 | F3PTDXdyrkGT3PKv7Kb6dWWjWjBYMB8GA1UdIwQYMBaAFLjSYIqHz0jucV3YUSAj 9 | WsGq5feyMCAGA1UdEQQZMBeCFXNlcnZlci5zdHJvbmdzd2FuLm9yZzATBgNVHSUE 10 | DDAKBggrBgEFBQcDATAKBggqhkjOPQQDAwNpADBmAjEA9TJ6mQykJfTD/MK4FqxA 11 | X4bTowM3LvMzniEBbDVCct3oLW2VqU8A12MEwYlj4FulAjEAlYCiquZUoPrV19gd 12 | N3MaXJe/7dwEjXczD1/5T4UjrWUbS+D7J5/DsEMYM10LNXgB 13 | -----END CERTIFICATE----- 14 | -------------------------------------------------------------------------------- /integration/strongswan/server/x509ca/caCert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIB3zCCAWWgAwIBAgIIWWpjqeLZ9K8wCgYIKoZIzj0EAwMwNTELMAkGA1UEBhMC 3 | Q0gxDjAMBgNVBAoTBUN5YmVyMRYwFAYDVQQDEw1DeWJlciBSb290IENBMB4XDTIw 4 | MDMwOTEyMDIwOVoXDTMwMDMwOTEyMDIwOVowNTELMAkGA1UEBhMCQ0gxDjAMBgNV 5 | BAoTBUN5YmVyMRYwFAYDVQQDEw1DeWJlciBSb290IENBMHYwEAYHKoZIzj0CAQYF 6 | K4EEACIDYgAEaRTgNHa/qa3D/mjvHr/Cr4f20KHiGzlarAXnnDkDUnT7/YEoWBAo 7 | hYqJFOOeuUc1dsIhXQFN/VVLP+Gyf0pSu272Nx7jA3IwERV12W7cV23YNUJVpNWp 8 | Y2t61UWiHeh4o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAd 9 | BgNVHQ4EFgQUuNJgiofPSO5xXdhRICNawarl97IwCgYIKoZIzj0EAwMDaAAwZQIw 10 | PR1T8MHS+aV9qSueIE9QfPRgEVyvuaz2g4q7DN51SUfypjYoAX+B6BqiR7vfgY2Y 11 | AjEA65R8XZy0N6LEYgAEPPbQSyCdJudoa4MwidaomSwwgiVDePN356onk/lhURmE 12 | QBaZ 13 | -----END CERTIFICATE----- 14 | -------------------------------------------------------------------------------- /integration/strongswan/strongswan.conf: -------------------------------------------------------------------------------- 1 | # strongSwan configuration file 2 | 3 | charon { 4 | start-scripts { 5 | creds = swanctl --load-creds 6 | conns = swanctl --load-conns 7 | pools = swanctl --load-pools 8 | } 9 | filelog { 10 | stderr { 11 | default = 1 12 | } 13 | } 14 | eap-dynamic { 15 | prefer_user = yes 16 | preferred = md5, tls 17 | } 18 | } 19 | 20 | libtls { 21 | suites = TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 22 | } 23 | -------------------------------------------------------------------------------- /integration/sztp/dhclient.conf: -------------------------------------------------------------------------------- 1 | option sztp-redirect-urls code 143 = text; 2 | request subnet-mask, 3 | broadcast-address, 4 | routers, 5 | domain-name, 6 | domain-name-servers, 7 | bootfile-name, 8 | tftp-server-name, 9 | sztp-redirect-urls, 10 | host-name; 11 | -------------------------------------------------------------------------------- /integration/sztp/dhcpd.conf.template: -------------------------------------------------------------------------------- 1 | # DHCP Server Configuration file. 2 | 3 | ddns-update-style interim; 4 | ignore client-updates; 5 | authoritative; 6 | allow booting; 7 | allow bootp; 8 | allow unknown-clients; 9 | option sztp-redirect-urls code 143 = text; 10 | 11 | 12 | # internal subnet for my DHCP Server 13 | subnet ${NODE_IP_SUBNET} netmask ${NODE_IP_NETMASK} { 14 | range ${NODE_IP_RANGE_MIN} ${NODE_IP_RANGE_MAX}; 15 | default-lease-time 600; 16 | max-lease-time 7200; 17 | 18 | #option 66 19 | option tftp-server-name \"w.x.y.z\"; 20 | 21 | #option 67 22 | option bootfile-name \"test.cfg\"; 23 | 24 | # option 143 (sZTP RFC-8572) or 136 for ipv6 25 | option sztp-redirect-urls \"https://redirecter:8080/restconf/operations/ietf-sztp-bootstrap-server:get-bootstrapping-data\"; 26 | 27 | # IP of PXE Server 28 | next-server ${NODE_IP_ADDRESS}; 29 | filename \"grubx64.efi\"; 30 | } 31 | -------------------------------------------------------------------------------- /integration/sztp/my-boot-image.img: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opiproject/opi-poc/afd4bca7aa9c093edeb538e4119e6b7a11c0d899/integration/sztp/my-boot-image.img -------------------------------------------------------------------------------- /integration/xPU-Integration-Blocks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opiproject/opi-poc/afd4bca7aa9c093edeb538e4119e6b7a11c0d899/integration/xPU-Integration-Blocks.png -------------------------------------------------------------------------------- /integration/xpu-cpu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/fedora:37 2 | 3 | # not sure what we run in xPU yet... 4 | -------------------------------------------------------------------------------- /integration/xpu-cpu/Dockerfile.telegraf: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/telegraf:1.29.2 2 | COPY telegraf-redfish.conf /etc/telegraf/telegraf.conf -------------------------------------------------------------------------------- /integration/xpu-cpu/README.md: -------------------------------------------------------------------------------- 1 | # xPU 2 | 3 | ## xPU's Main CPU 4 | 5 | ssh example: 6 | 7 | ```text 8 | $ ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2207 xpu@127.0.0.1 9 | Warning: Permanently added '[127.0.0.1]:2207' (ECDSA) to the list of known hosts. 10 | xpu@127.0.0.1's password: 11 | Welcome to OpenSSH Server 12 | 13 | xpu-cpu:~$ exit 14 | logout 15 | Connection to 127.0.0.1 closed. 16 | ``` 17 | 18 | telegraf example: 19 | 20 | ```text 21 | $ docker run --rm --net=host -v $(pwd)/telegraf-spdk.conf:/etc/telegraf/telegraf.conf:ro telegraf:1.22 22 | 2022-03-29T18:47:11Z I! Using config file: /etc/telegraf/telegraf.conf 23 | 2022-03-29T18:47:11Z I! Starting Telegraf 1.22.0 24 | 2022-03-29T18:47:11Z I! Loaded inputs: http 25 | 2022-03-29T18:47:11Z I! Loaded aggregators: 26 | 2022-03-29T18:47:11Z I! Loaded processors: 27 | 2022-03-29T18:47:11Z W! Outputs are not used in testing mode! 28 | 2022-03-29T18:47:11Z I! Tags enabled: host=localhost 29 | 30 | mem,host=52ee5c75df01 commit_limit=69312983040i,committed_as=13494636544i,huge_page_size=2097152i,used_percent=10.100053796757296,high_free=0i,inactive=13541511168i,low_free=0i,shared=3904901120i,sreclaimable=812650496i,swap_cached=0i,free=100370612224i,huge_pages_total=2048i,low_total=0i,page_tables=49500160i,used=13567504384i,huge_pages_free=1357i,mapped=901996544i,slab=2243977216i,swap_total=4294963200i,cached=20385955840i,vmalloc_chunk=0i,vmalloc_used=0i,write_back=0i,swap_free=4294963200i,high_total=0i,available_percent=86.20598148102354,available=115801366528i,sunreclaim=1431326720i,total=134331011072i,buffered=6938624i,dirty=856064i,vmalloc_total=14073748835531776i,write_back_tmp=0i,active=8859537408i 1650954170000000000 31 | 32 | net,host=52ee5c75df01,interface=eth0 drop_in=0i,drop_out=0i,bytes_sent=16589i,bytes_recv=13986i,packets_sent=89i,packets_recv=110i,err_in=0i,err_out=0i 1650954170000000000 33 | 34 | cpu,cpu=cpu0,host=52ee5c75df01 usage_user=99.6999999973923,usage_system=0.09999999999763531,usage_idle=0,usage_iowait=0,usage_softirq=0,usage_steal=0,usage_nice=0,usage_irq=0.19999999999527063,usage_guest=0,usage_guest_nice=0 1650954170000000000 35 | cpu,cpu=cpu1,host=52ee5c75df01 usage_user=99.70000000204891,usage_system=0,usage_irq=0.2999999999974534,usage_steal=0,usage_idle=0,usage_nice=0,usage_iowait=0,usage_softirq=0,usage_guest=0,usage_guest_nice=0 1650954170000000000 36 | cpu,cpu=cpu2,host=52ee5c75df01 usage_system=0,usage_idle=0,usage_iowait=0,usage_guest_nice=0,usage_user=99.79999999981374,usage_nice=0,usage_irq=0.20000000000436557,usage_softirq=0,usage_steal=0,usage_guest=0 1650954170000000000 37 | cpu,cpu=cpu3,host=52ee5c75df01 usage_guest_nice=0,usage_user=99.79999999981374,usage_idle=0,usage_nice=0,usage_iowait=0,usage_guest=0,usage_system=0,usage_irq=0.20000000000436557,usage_softirq=0,usage_steal=0 1650954170000000000 38 | cpu,cpu=cpu4,host=52ee5c75df01 usage_user=99.70029970233988,usage_guest=0,usage_steal=0,usage_guest_nice=0,usage_system=0.09990009990223975,usage_idle=0,usage_nice=0,usage_iowait=0,usage_irq=0.19980019979993657,usage_softirq=0 1650954170000000000 39 | cpu,cpu=cpu5,host=52ee5c75df01 usage_nice=0,usage_iowait=0,usage_irq=0.2997002997044478,usage_softirq=0,usage_steal=0,usage_guest_nice=0,usage_user=99.70029970233988,usage_idle=0,usage_guest=0,usage_system=0 1650954170000000000 40 | 41 | telegraf_1 | redfish_thermal_temperatures,address=xpu-bmc,health=OK,host=220e68143c3d,member_id=0,name=CPU1\ Temp,rack=WEB43,row=North,source=web483,state=Enabled reading_celsius=41,upper_threshold_critical=45,upper_threshold_fatal=48 1656718190000000000 42 | telegraf_1 | redfish_thermal_temperatures,address=xpu-bmc,host=220e68143c3d,member_id=1,name=CPU2\ Temp,rack=WEB43,row=North,source=web483,state=Disabled upper_threshold_critical=45,upper_threshold_fatal=48 1656718190000000000 43 | telegraf_1 | redfish_thermal_temperatures,address=xpu-bmc,health=OK,host=220e68143c3d,member_id=2,name=Chassis\ Intake\ Temp,rack=WEB43,row=North,source=web483,state=Enabled upper_threshold_critical=40,upper_threshold_fatal=50,lower_threshold_critical=5,lower_threshold_fatal=0,reading_celsius=25 1656718190000000000 44 | telegraf_1 | redfish_thermal_fans,address=xpu-bmc,health=OK,host=220e68143c3d,member_id=0,name=BaseBoard\ System\ Fan,rack=WEB43,row=North,source=web483,state=Enabled lower_threshold_fatal=0i,reading_rpm=2100i 1656718190000000000 45 | telegraf_1 | redfish_thermal_fans,address=xpu-bmc,health=OK,host=220e68143c3d,member_id=1,name=BaseBoard\ System\ Fan\ Backup,rack=WEB43,row=North,source=web483,state=Enabled lower_threshold_fatal=0i,reading_rpm=2050i 1656718190000000000 46 | telegraf_1 | redfish_power_powersupplies,address=xpu-bmc,health=Warning,host=220e68143c3d,member_id=0,name=Power\ Supply\ Bay,rack=WEB43,row=North,source=web483,state=Enabled line_input_voltage=120,last_power_output_watts=325,power_capacity_watts=800 1656718190000000000 47 | telegraf_1 | redfish_power_voltages,address=xpu-bmc,health=OK,host=220e68143c3d,member_id=0,name=VRM1\ Voltage,rack=WEB43,row=North,source=web483,state=Enabled reading_volts=12,upper_threshold_critical=13,upper_threshold_fatal=15,lower_threshold_critical=11,lower_threshold_fatal=10 1656718190000000000 48 | telegraf_1 | redfish_power_voltages,address=xpu-bmc,health=OK,host=220e68143c3d,member_id=1,name=VRM2\ Voltage,rack=WEB43,row=North,source=web483,state=Enabled reading_volts=5,upper_threshold_critical=7,lower_threshold_critical=4.5 1656718190000000000 49 | 50 | ... 51 | ``` 52 | 53 | ## xPU BMC 54 | 55 | Runs redfish server, example: 56 | 57 | ```text 58 | $ curl --fail http://127.0.0.1:8001/redfish/v1 { 59 | "@odata.id": "/redfish/v1/", 60 | "@odata.type": "#ServiceRoot.v1_6_0.ServiceRoot", 61 | "AccountService": { 62 | "@odata.id": "/redfish/v1/AccountService" 63 | }, 64 | "CertificateService": { 65 | "@odata.id": "/redfish/v1/CertificateService" 66 | }, 67 | "Chassis": { 68 | "@odata.id": "/redfish/v1/Chassis" 69 | }, 70 | "EventService": { 71 | "@odata.id": "/redfish/v1/EventService" 72 | }, 73 | "Id": "RootService", 74 | "Links": { 75 | "Sessions": { 76 | "@odata.id": "/redfish/v1/SessionService/Sessions" 77 | } 78 | }, 79 | "Managers": { 80 | "@odata.id": "/redfish/v1/Managers" 81 | }, 82 | "Name": "Root Service", 83 | "Oem": {}, 84 | "RedfishVersion": "1.6.0", 85 | "Registries": { 86 | "@odata.id": "/redfish/v1/Registries" 87 | }, 88 | "SessionService": { 89 | "@odata.id": "/redfish/v1/SessionService" 90 | }, 91 | "Systems": { 92 | "@odata.id": "/redfish/v1/Systems" 93 | }, 94 | "Tasks": { 95 | "@odata.id": "/redfish/v1/TaskService" 96 | }, 97 | "UUID": "92384634-2938-2342-8820-489239905423", 98 | "UpdateService": { 99 | "@odata.id": "/redfish/v1/UpdateService" 100 | } 101 | } 102 | ``` 103 | 104 | ssh example: 105 | 106 | ```text 107 | $ ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2209 bmc@127.0.0.1 108 | Warning: Permanently added '[127.0.0.1]:2209' (ECDSA) to the list of known hosts. 109 | bmc@127.0.0.1's password: 110 | Welcome to OpenSSH Server 111 | 112 | xpu-bmc:~$ exit 113 | logout 114 | Connection to 127.0.0.1 closed. 115 | ``` 116 | -------------------------------------------------------------------------------- /integration/xpu-cpu/telegraf-redfish.conf: -------------------------------------------------------------------------------- 1 | [[inputs.redfish]] 2 | address = "http://xpu-bmc-ssh:8000" 3 | username = "root" 4 | password = "password123456" 5 | computer_system_id="437XR1138R2" 6 | 7 | [[inputs.http]] 8 | urls = ["http://xpu-cpu-ssh:9009"] 9 | headers = {"Content-Type" = "application/json"} 10 | method = "POST" 11 | username = "spdkuser" 12 | password = "spdkpass" 13 | body = '{"id":1, "method": "bdev_get_bdevs"}' 14 | data_format = "json" 15 | name_override = "xpu" 16 | json_strict = true 17 | tag_keys = ["name"] 18 | json_query = "result" 19 | 20 | [[inputs.cpu]] 21 | percpu = true 22 | totalcpu = true 23 | collect_cpu_time = false 24 | report_active = false 25 | 26 | [[inputs.mem]] 27 | # no configuration 28 | 29 | [[inputs.net]] 30 | ignore_protocol_stats = false 31 | 32 | [[outputs.file]] 33 | files = ["stdout"] 34 | data_format = "influx" 35 | 36 | [[outputs.opentelemetry]] 37 | service_address = "otel-gw-collector:4317" 38 | -------------------------------------------------------------------------------- /lab/README.md: -------------------------------------------------------------------------------- 1 | # OPI Lab 2 | 3 | Moved to 4 | -------------------------------------------------------------------------------- /openoffload/README.md: -------------------------------------------------------------------------------- 1 | # openoffload PoC 2 | 3 | Based on openoffload opensource project, see [openoffload](https://github.com/opiproject/sessionOffload). 4 | Demo offload 5-Tuple UDP and TCP flows to the xPU. 5 | 6 | ## Description of Demo Steps 7 | 8 | 1. A packet arrives to xPU 9 | 2. The xPU does a lookup in the session table for the session. If the session is present and the action is offload the packet is sent out. 10 | 3. If the packet does not exist it is sent to the firewall. The firewall is responsible for managing session state. 11 | 4. Packet is sent to the firewall, if the packet is not suitable for offload it is processed by the firewall. 12 | 5. If the session is offloadable the session information is sent to the gRPC python client that calls the Vendor with the addSession call to add the session to the xPU session table. 13 | 6. The xPU daemon updates the session table and sets a timer for the session (in both directions for TCP flows). 14 | 7. Once the timer expires the xPU checks the flows in both directions to see if there have been any packets since the last check. If there have been packets the xPU continues to forward packets. If there have not been any packets the xPU sends an update to the firewall with the packet and byte count of the offloaded packets and removes the session from the session table. 15 | 8. The firewall receives the closed session message and updates the session statistics and goes about the session close process. 16 | 17 | ## GRPC client example 18 | 19 | Client code example is located at [client](https://github.com/opiproject/sessionOffload/blob/master/tests/basic/sessions_client.py) 20 | 21 | ## GRPC server example 22 | 23 | Server code example is located at [server](https://github.com/opiproject/sessionOffload/blob/master/tests/basic/sessions_server.py) 24 | 25 | ## protobuf API 26 | 27 | protobuf API is located at [protobuf](https://github.com/opiproject/sessionOffload/blob/master/protos/openoffload.proto) 28 | 29 | ## PoC execution 30 | 31 | For python PoC execution please follow [example](https://github.com/opiproject/sessionOffload/blob/master/tests/pythonSimulator/README.md) 32 | 33 | ## Generic SmartNIC Architecture 34 | 35 | [Architecture](https://github.com/opiproject/sessionOffload/blob/master/doc/SmartNIC_Arch.md) 36 | 37 | ## High level architecture with xPU 38 | 39 | ![xPU](host_xpu.png) 40 | 41 | ## Openoffload Demo with Marvell OCTEON xPU 42 | 43 | Demo presented during OPI API meeting(22/8/2022) 44 | -------------------------------------------------------------------------------- /openoffload/host_xpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opiproject/opi-poc/afd4bca7aa9c093edeb538e4119e6b7a11c0d899/openoffload/host_xpu.png -------------------------------------------------------------------------------- /security/README.md: -------------------------------------------------------------------------------- 1 | # Security 2 | 3 | Moved to 4 | -------------------------------------------------------------------------------- /setup/README.md: -------------------------------------------------------------------------------- 1 | # What is this? 2 | 3 | This is an ansible playbook for helping set up an environment to run the tests 4 | or run the developer platform. 5 | 6 | ## How do you use this? 7 | 8 | Create an inventory file like this, replacing `mytesthost.domain` with the host 9 | name or IP address to configure and `mytestuser` with the user to remotely log 10 | in as (if different from the local user): 11 | 12 | `mytesthost.domain ansible_user=mytestuser` 13 | 14 | The ansible_user should have sudo privileges. Preferably, set up password-less 15 | ssh and password-less sudo for that user, though you can instead instruct 16 | ansible to ask for passwords. 17 | 18 | Then run the playbook: 19 | 20 | ```bash 21 | ansible-playbook -i inventory setup.yml 22 | ``` 23 | 24 | ## Example inventory file 25 | 26 | The playbooks are set up to run on `hosts: all` so it will run on each host in 27 | your inventory file simultaneously. You could instead change the playbook to 28 | run it on a group of hosts (e.g., `opi-hosts`) or on individual hosts. See the 29 | ansible documentation for details. 30 | 31 | ```text 32 | [opi-hosts] 33 | myhost1.mydomain ansible_user=test 34 | 192.168.11.12 ansible_user=foo 35 | ``` 36 | -------------------------------------------------------------------------------- /setup/debian.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) 2022 Red Hat 3 | --- 4 | - name: Debian | Install package dependencies 5 | apt: 6 | name: docker.io,python3-pip,sshpass,git 7 | state: present 8 | 9 | - name: Debian | Install docker-compose 10 | pip: 11 | name: docker-compose>=1.29 12 | state: present 13 | -------------------------------------------------------------------------------- /setup/redhat.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) 2022 Red Hat 3 | --- 4 | - name: Red Hat | Install package dependencies 5 | dnf: 6 | name: podman,podman-docker,podman-plugins,python3-pip,sshpass,git 7 | state: present 8 | 9 | # docker-compose >=2 does not work with podman currently 10 | - name: Red Hat | Install docker-compose 11 | pip: 12 | name: docker-compose>=1.29,<2 13 | state: present 14 | 15 | - name: Red Hat | Enable podman socket for docker-compose 16 | systemd: 17 | name: podman.socket 18 | state: started 19 | -------------------------------------------------------------------------------- /setup/setup.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) 2022 Red Hat 3 | # Copyright (c) 2022 Dell Inc, or its subsidiaries. 4 | --- 5 | - name: Set up for OPI PoC development 6 | hosts: all 7 | become: yes 8 | tasks: 9 | - include: debian.yml 10 | when: ansible_os_family == 'Debian' 11 | 12 | - include: redhat.yml 13 | when: ansible_os_family == 'RedHat' 14 | 15 | - include: suse.yml 16 | when: ansible_os_family == 'Suse' 17 | 18 | # This is only needed for the first time to dynamically allocate huge pages 19 | # Subsequent boots do not need this configuration, so do not use sysctl here 20 | - name: Help ensure huge pages can be demand allocated 21 | shell: echo 1 > /proc/sys/vm/drop_caches 22 | 23 | - name: Check if hugetlbfs is mounted 24 | shell: grep hugetlbfs /proc/mounts 25 | register: result 26 | ignore_errors: yes 27 | 28 | - name: Is hugetlbfs mounted? 29 | set_fact: 30 | hugetlbfs_mounted: "{{ true if result.rc == 0 else false}}" 31 | 32 | - name: Mount hugetlbfs 33 | mount: 34 | path: /mnt/huge 35 | src: nodev 36 | fstype: hugetlbfs 37 | state: mounted 38 | when: hugetlbfs_mounted == false 39 | 40 | - name: Set huge pages 41 | sysctl: 42 | name: vm.nr_hugepages 43 | value: '1024' 44 | sysctl_set: yes 45 | state: present 46 | reload: yes 47 | 48 | # This will fail if we don't have exactly 1024 nr_hugepages 49 | - name: Verify huge pages set 50 | shell: grep 1024 /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages 51 | -------------------------------------------------------------------------------- /setup/suse.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) 2022 Red Hat 3 | # Copyright (c) 2022 Dell Inc, or its subsidiaries. 4 | --- 5 | - name: Suse | Install package dependencies 6 | zypper: 7 | name: docker.io,python3-pip,sshpass,git 8 | state: present 9 | 10 | - name: Suse | Install docker-compose 11 | pip: 12 | name: docker-compose>=1.29 13 | state: present 14 | -------------------------------------------------------------------------------- /storage/README.md: -------------------------------------------------------------------------------- 1 | # Storage 2 | 3 | Moved to 4 | --------------------------------------------------------------------------------