├── .github ├── dependabot.yml └── workflows │ ├── go-lint.yaml │ ├── go-unit-tests.yaml │ └── release.yaml ├── .gitignore ├── .ko.yaml ├── LICENSE ├── README.md ├── cmd ├── client │ └── main.go └── server │ ├── main.go │ ├── main_test.go │ ├── server.go │ └── server_test.go ├── config └── server.yaml ├── go.mod └── go.sum /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 5 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /.github/workflows/go-lint.yaml: -------------------------------------------------------------------------------- 1 | name: Code Linting 2 | 3 | on: 4 | push: 5 | branches: ["main", "master"] 6 | 7 | pull_request: 8 | branches: ["main", "master", "release-*"] 9 | 10 | jobs: 11 | lint: 12 | name: Code Linting 13 | strategy: 14 | matrix: 15 | go-version: ["1.19"] # >=1.16 required due to io.ReadAll() 16 | platform: ["ubuntu-latest"] 17 | 18 | runs-on: ${{ matrix.platform }} 19 | timeout-minutes: 10 20 | steps: 21 | - name: Checkout repository 22 | uses: actions/checkout@v3 23 | with: 24 | fetch-depth: 1 25 | 26 | - name: Run golangci-lint 27 | uses: golangci/golangci-lint-action@5f1fec7010f6ae3b84ea4f7b2129beb8639b564f -------------------------------------------------------------------------------- /.github/workflows/go-unit-tests.yaml: -------------------------------------------------------------------------------- 1 | name: Go Tests 2 | 3 | on: 4 | push: 5 | branches: ["main", "master"] 6 | 7 | pull_request: 8 | branches: ["main", "master", "release-*"] 9 | 10 | jobs: 11 | test: 12 | name: Go Tests 13 | strategy: 14 | matrix: 15 | go-version: ["1.19"] 16 | platform: ["ubuntu-latest"] 17 | 18 | runs-on: ${{ matrix.platform }} 19 | timeout-minutes: 10 20 | 21 | steps: 22 | - name: Set up Go ${{ matrix.go-version }} 23 | uses: actions/setup-go@v4 24 | with: 25 | go-version: ${{ matrix.go-version }} 26 | id: go 27 | 28 | - name: Install tparse 29 | run: go install github.com/mfridman/tparse@latest 30 | 31 | - name: Check out code 32 | uses: actions/checkout@v3 33 | 34 | - name: Check for .codecov.yaml 35 | id: codecov-enabled 36 | uses: andstor/file-existence-action@v2 37 | with: 38 | files: .codecov.yaml 39 | 40 | - if: steps.codecov-enabled.outputs.files_exists == 'true' 41 | name: Enable Go Coverage 42 | run: echo 'COVER_OPTS=-coverprofile=coverage.txt -covermode=atomic' >> $GITHUB_ENV 43 | 44 | - name: Test 45 | env: 46 | GOFLAGS: "-v -race -count=1 -json" 47 | run: go test $COVER_OPTS ./... | tparse -all -notests -format markdown >> $GITHUB_STEP_SUMMARY 48 | 49 | - name: Verify git clean 50 | shell: bash 51 | run: | 52 | if [[ -z "$(git status --porcelain)" ]]; then 53 | echo "${{ github.repository }} up to date." 54 | else 55 | echo "${{ github.repository }} is dirty." 56 | echo "::error:: $(git status)" 57 | exit 1 58 | fi 59 | 60 | - if: steps.codecov-enabled.outputs.files_exists == 'true' 61 | name: Produce Codecov Report 62 | uses: codecov/codecov-action@v3 -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Create Release 2 | 3 | on: 4 | # release will only be created when ref is a tag starting with "v" 5 | push: 6 | tags: 7 | - "v*" 8 | 9 | jobs: 10 | image: 11 | name: Create release and artifacts (release.yaml and GCR Container Image) 12 | runs-on: ubuntu-latest 13 | timeout-minutes: 10 14 | 15 | steps: 16 | - name: Setup ko 17 | # will install latest ko version and by default login/configure for ghcr.io 18 | uses: imjasonh/setup-ko@ace48d793556083a76f1e3e6068850c1f4a369aa 19 | 20 | - name: Set up Go 21 | uses: actions/setup-go@v4 22 | with: 23 | go-version: 1.17 24 | 25 | - name: Check out code 26 | uses: actions/checkout@v3 27 | 28 | - name: Get short COMMIT and TAG 29 | run: | 30 | echo "KO_COMMIT=$(echo -n $GITHUB_SHA | cut -c -8)" >> $GITHUB_ENV 31 | echo "KO_TAG=$(basename "${{ github.ref }}")" >> $GITHUB_ENV 32 | 33 | - name: Build and Publish Worker Image 34 | run: | 35 | # build, push and create release YAML 36 | ko resolve --tags ${KO_TAG},${KO_COMMIT},latest --bare -Rf config/ > release.yaml 37 | 38 | - name: Create Github Release and upload assets 39 | env: 40 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 41 | run: | 42 | gh release create ${KO_TAG} release.yaml README.md LICENSE 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # secrets 18 | username 19 | password 20 | 21 | # binaries 22 | cmd/client/client 23 | cmd/server/server -------------------------------------------------------------------------------- /.ko.yaml: -------------------------------------------------------------------------------- 1 | builds: 2 | - id: server 3 | dir: cmd/server 4 | # main: . 5 | env: 6 | - GOPRIVATE=*.vmware.com 7 | flags: 8 | - -tags 9 | - netgo 10 | ldflags: 11 | - -s -w 12 | - -extldflags "-static" 13 | - -X main.buildCommit={{.Env.KO_COMMIT}} 14 | - -X main.buildVersion={{.Env.KO_TAG}} -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vSphere Event Streaming 2 | 3 | [![Latest 4 | Release](https://img.shields.io/github/release/embano1/vsphere-event-streaming.svg?logo=github&style=flat-square)](https://github.com/embano1/vsphere-event-streaming/releases/latest) 5 | [![go.mod Go 6 | version](https://img.shields.io/github/go-mod/go-version/embano1/vsphere-event-streaming)](https://github.com/embano1/vsphere-event-streaming) 7 | 8 | Prototype to show how to transform an existing (SOAP) API into a modern 9 | HTTP/REST streaming API. 10 | 11 | ## Details 12 | 13 | The vSphere Event Streaming server connects to the vCenter event stream, 14 | transforms each event into a standarized [`CloudEvent`](https://cloudevents.io/) 15 | and exposes these as *JSON* objects via a (streaming) HTTP/REST API. 16 | 17 | The benefits of this approach are: 18 | 19 | - Simplified consumption via a modern HTTP/REST API instead of directly using 20 | vCenter SOAP APIs 21 | - Kubernetes inspired `watch` style to stream events from a specific position 22 | (or "now") 23 | - Decoupling from vCenter by proxying multiple clients onto the (cached) event 24 | stream of this streaming server 25 | - Apache [Kafka](https://kafka.apache.org/) inspired behavior, using `Offsets` 26 | to traverse and replay the event stream 27 | - Lightweight and stateless: events are stored (cached) in memory. If the server 28 | crashes it resumes from the configurable `VCENTER_STREAM_BEGIN` (default: last 29 | 10 minutes) 30 | 31 | The unique event `ID` of each vSphere event is mapped to the position (`Offset`) 32 | in the internal (immutable) event *Log* (journal). Clients use these event `IDs` 33 | (`Offsets`) to read/stream/replay events as JSON objects. 34 | 35 | ### Example 36 | 37 | This example uses `curl` and `jq` to query against a locally running vSphere 38 | Event Streaming server. 39 | 40 | 💡 If the server is running in Kubernetes (see below), use the `kubectl 41 | port-forward` command to forward CLI commands to streaming server running in a 42 | Kubernetes cluster. 43 | 44 | ```console 45 | # get current available event range 46 | $ curl -N -s localhost:8080/api/v1/range | jq . 47 | { 48 | "earliest": 44, 49 | "latest": 46 50 | } 51 | 52 | # read first available event 53 | $ curl -N -s localhost:8080/api/v1/events/44 | jq . 54 | { 55 | "specversion": "1.0", 56 | "id": "44", 57 | "source": "https://localhost:8989/sdk", 58 | "type": "vmware.vsphere.UserLoginSessionEvent.v0", 59 | "datacontenttype": "application/json", 60 | "time": "2022-01-14T13:26:22.0854137Z", 61 | "data": { 62 | "Key": 44, 63 | "ChainId": 44, 64 | "CreatedTime": "2022-01-14T13:26:22.0854137Z", 65 | "UserName": "user\n", 66 | "Datacenter": null, 67 | "ComputeResource": null, 68 | "Host": null, 69 | "Vm": null, 70 | "Ds": null, 71 | "Net": null, 72 | "Dvs": null, 73 | "FullFormattedMessage": "User user\n@172.17.0.1 logged in as Go-http-client/1.1", 74 | "ChangeTag": "", 75 | "IpAddress": "172.17.0.1", 76 | "UserAgent": "Go-http-client/1.1", 77 | "Locale": "en_US", 78 | "SessionId": "56c95aca-aed7-471d-b69f-be73468a89aa" 79 | }, 80 | "eventclass": "event" 81 | } 82 | 83 | # watch for new events and use a jq field selector 84 | $ curl -N -s localhost:8080/api/v1/events\?watch=true | jq '.eventclass+":"+.id+" "+.type' 85 | "event:47 vmware.vsphere.VmStartingEvent.v0" 86 | "event:48 vmware.vsphere.VmPoweredOnEvent.v0" 87 | 88 | # start watch from specific id (offset) 89 | curl -N -s localhost:8080/api/v1/events\?watch=true\&offset=44 | jq '.eventclass+":"+.id+" "+.type' 90 | "event:44 vmware.vsphere.UserLoginSessionEvent.v0" 91 | "event:45 vmware.vsphere.VmStoppingEvent.v0" 92 | "event:46 vmware.vsphere.VmPoweredOffEvent.v0" 93 | "event:47 vmware.vsphere.VmStartingEvent.v0" 94 | "event:48 vmware.vsphere.VmPoweredOnEvent.v0" 95 | ``` 96 | 97 | 💡 To retrieve the last 50 events use `curl -N -s localhost:8080/api/v1/events`. 98 | The current hardcoded page size is `50` and a pagination API is on my `TODO` 99 | list 🤓 100 | 101 | ## Deployment 102 | 103 | The vSphere Event Streaming server is packaged as a Kubernetes `Deployment` and 104 | configured via environment variables and a Kubernetes `Secret` holding the 105 | vCenter Server credentials. 106 | 107 | Requirements: 108 | 109 | - Service account/role to read vCenter events 110 | - Kubernetes cluster to deploy the vSphere Event Stream server 111 | - Kubernetes CLI `kubectl` installed 112 | - Network access to vCenter from within the Kubernetes cluster 113 | 114 | 💡 The vCenter Simulator 115 | [`vcsim`](https://github.com/vmware/govmomi/tree/master/vcsim) can be used to 116 | deploy a mock vCenter in Kubernetes for testing and experimenting. 117 | 118 | ### Create Credentials 119 | 120 | ```console 121 | # create namespace 122 | kubectl create namespace vcenter-stream 123 | 124 | # create Kubernetes secret holding the vSphere credentials 125 | # replace with your values 126 | kubectl --namespace vcenter-stream create secret generic vsphere-credentials --from-literal=username=user --from-literal=password=pass 127 | ``` 128 | 129 | **Optional** if you want to use `vcsim`: 130 | 131 | ```console 132 | # deploy container 133 | kubectl --namespace vcenter-stream create -f https://raw.githubusercontent.com/vmware-samples/vcenter-event-broker-appliance/development/vmware-event-router/deploy/vcsim.yaml 134 | 135 | # in a separate terminal start port-forwarding 136 | kubectl --namespace vcenter-stream port-forward deploy/vcsim 8989:8989 137 | ``` 138 | 139 | ### Change `Deployment` Manifest 140 | 141 | Download the latest deployment manifest (`release.yaml`) file from the Github 142 | release page and update the environment variables in `release.yaml` to match 143 | your setup. Then save the file under the same name (to follow along with the 144 | commands). 145 | 146 | 💡 The environment variables are explained below. 147 | 148 | Example Download with `curl`: 149 | 150 | ```console 151 | curl -L -O https://github.com/embano1/vsphere-event-streaming/releases/latest/download/release.yaml 152 | ``` 153 | 154 | #### vSphere Settings 155 | 156 | These settings are required to set up the connection between the event streaming 157 | server and VMware vCenter Server. 158 | 159 | 💡 If you are not making any modifications to the application leave the default 160 | value for `VCENTER_SECRET_PATH`. 161 | 162 | | Variable | Description | Required | Example | Default | 163 | |-----------------------|-------------------------------------------------------------------------------------|----------|-----------------------------------|---------------------------| 164 | | `VCENTER_URL` | vCenter Server URL | yes | `https://myvc-01.prod.corp.local` | (empty) | 165 | | `VCENTER_INSECURE` | Ignore vCenter Server certificate warnings | no | `"true"` | `"false"` | 166 | | `VCENTER_SECRET_PATH` | Directory where `username` and `password` files are located to retrieve credentials | yes | `"./"` | `"/var/bindings/vsphere"` | 167 | 168 | #### Streaming Settings 169 | 170 | These settings are used to customize the event streaming server. The event 171 | streaming server internally uses `memlog` as an append-only *Log*. See the 172 | [project](https://github.com/embano1/memlog) for details. 173 | 174 | If you want to account for longer downtime you might want to increase the 175 | default value of `5m` used to replay vCenter events after starting the server. 176 | 177 | The defaults for `LOG_MAX_RECORD_SIZE_BYTES` and `LOG_MAX_SEGMENT_SIZE` are 178 | usually fine. The total number of records in the internal event *Log* is twice 179 | the `LOG_MAX_SEGMENT_SIZE` (*active* and *history* segment). If the *active* 180 | segment is full and there is already a *history* segment, this *history segment* 181 | will be purged, i.e. events deleted from the internal *Log* (but not within 182 | vCenter Server!). 183 | 184 | Trying to read a purged event throws an `invalid offset` error. 185 | 186 | 💡 If you are seeing the server crashing with out of memory errors (`OOM`), try 187 | increasin the specified memory `limit` in the `release.yaml` manifest. 188 | 189 | | Variable | Description | Required | Example | Default | 190 | |-----------------------------|--------------------------------------------------------------------------------------------------------------------------------|----------|----------------|----------------------------------------------------------------| 191 | | `VCENTER_STREAM_BEGIN` | Stream vCenter events starting at "now" minus specified duration (requires suffix, e.g. `s`/`m`/`h` for seconds/minutes/hours) | yes | `"1h"` | `"5m"` (stream starts with events from last 5 minutes) | 192 | | `LOG_MAX_RECORD_SIZE_BYTES` | Maximum size of each record in the log | yes | `"1024"` (1Kb) | `"524288"` (512Kb) | 193 | | `LOG_MAX_SEGMENT_SIZE` | Maximum number of records per segment | yes | `"10000"` | `"1000"` (1000 entries in *active*, 1000 in *history* segment) | 194 | 195 | ### Deploy the Server 196 | 197 | ```console 198 | kubectl -n vcenter-stream create -f release.yaml 199 | ``` 200 | 201 | Verify that the server is correctly starting: 202 | 203 | ```console 204 | kubectl -n vcenter-stream logs deploy/vsphere-event-stream 205 | 2022-01-14T14:31:43.798Z INFO eventstream server/main.go:166 starting http listener {"port": 8080} 206 | 2022-01-14T14:31:43.874Z INFO eventstream server/main.go:104 starting vsphere event collector {"begin": "5m0s", "pollInterval": "1s"} 207 | 2022-01-14T14:31:44.877Z DEBUG eventstream server/main.go:122 initializing new log {"startOffset": 27, "maxSegmentSize": 1000, "maxRecordSize": 524288} 208 | 2022-01-14T14:31:44.878Z DEBUG eventstream server/main.go:155 wrote cloudevent to log {"offset": 27, "event": "Context Attributes,\n specversion: 1.0\n type: vmware.vsphere.UserLoginSessionEvent.v0\n source: https://vcsim.vcenter-stream.svc.cluster.local/sdk\n id: 27\n time: 2022-01-14T14:31:43.7884757Z\n datacontenttype: application/json\nExtensions,\n eventclass: event\nData,\n {\n \"Key\": 27,\n \"ChainId\": 27,\n \"CreatedTime\": \"2022-01-14T14:31:43.7884757Z\",\n \"UserName\": \"user\",\n \"Datacenter\": null,\n \"ComputeResource\": null,\n \"Host\": null,\n \"Vm\": null,\n \"Ds\": null,\n \"Net\": null,\n \"Dvs\": null,\n \"FullFormattedMessage\": \"User user@10.244.0.6 logged in as Go-http-client/1.1\",\n \"ChangeTag\": \"\",\n \"IpAddress\": \"10.244.0.6\",\n \"UserAgent\": \"Go-http-client/1.1\",\n \"Locale\": \"en_US\",\n \"SessionId\": \"72096903-7acb-476f-bb76-29941a91fa1b\"\n }\n", "bytes": 646} 209 | ``` 210 | 211 | 💡 If you delete (kill) the Kubernetes `Pod` of the vSphere Event Stream server 212 | to simulate an outage, Kubernetes will automatically restart the server. You 213 | will then be able to query the events starting off of the specified interval 214 | defined via `VCENTER_STREAM_BEGIN`. Events within this timeframe will not be 215 | lost and clients can restart their `watch` from the `earliest` event retrieved 216 | via the `/api/v1/range` endpoint. 217 | 218 | ### Set up Port-Forwarding 219 | 220 | Inside Kubernetes the server is configured with a Kubernetes `Service` and 221 | accessible over the `Service` port `80` within the cluster. 222 | 223 | To query the server from a local (remote) machine it is the easiest to create a 224 | port-forwarding. 225 | 226 | ```console 227 | # forward local port 8080 to service port 80 228 | kubectl -n vcenter-stream port-forward service/vsphere-event-stream 8080:80 229 | ``` 230 | 231 | Then in a separate terminal run `curl` as usual. 232 | 233 | ```console 234 | curl -s -N localhost:8080/api/v1/range 235 | {"earliest":27,"latest":27} 236 | ``` 237 | 238 | ## Uninstall 239 | 240 | To uninstall the vSphere Event Stream server and all its dependencies, run: 241 | 242 | ```console 243 | kubectl delete namespace vcenter-stream 244 | ``` 245 | 246 | ## Build Custom Image 247 | 248 | **Note:** This step is only required if you made code changes to the Go code. 249 | 250 | This example uses [`ko`](https://github.com/google/ko) to build and push 251 | container artifacts. 252 | 253 | ```console 254 | # only when using kind: 255 | # export KIND_CLUSTER_NAME=kind 256 | # export KO_DOCKER_REPO=kind.local 257 | 258 | export KO_DOCKER_REPO=my-docker-username 259 | export KO_COMMIT=$(git rev-parse --short=8 HEAD) 260 | export KO_TAG=$(git describe --abbrev=0 --tags) 261 | 262 | # build, push and run the worker in the configured Kubernetes context 263 | # and vmware-preemption Kubernetes namespace 264 | ko resolve -BRf config | kubectl -n vcenter-stream apply -f - 265 | ``` 266 | 267 | To delete the deployment: 268 | 269 | ```console 270 | ko -n vcenter-stream delete -f config 271 | ``` 272 | -------------------------------------------------------------------------------- /cmd/client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "flag" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "strconv" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/embano1/vsphere/logger" 15 | "go.uber.org/zap" 16 | ) 17 | 18 | var ( 19 | address string 20 | watch bool 21 | start int 22 | ) 23 | 24 | func main() { 25 | flag.StringVar(&address, "server", "http://localhost:8080/api/v1/events", "full stream server URL") 26 | flag.IntVar(&start, "start", 0, "start offset (0 for latest)") 27 | flag.BoolVar(&watch, "watch", false, "watch the event stream") 28 | flag.Parse() 29 | 30 | l, err := zap.NewDevelopment() 31 | if err != nil { 32 | panic("create logger: " + err.Error()) 33 | } 34 | 35 | ctx := logger.Set(context.Background(), l) 36 | ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) 37 | defer cancel() 38 | 39 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) 40 | if err != nil { 41 | l.Fatal("could not create request", zap.Error(err)) 42 | } 43 | 44 | req.Header.Add("Transfer-Encoding", "chunked") 45 | 46 | q := req.URL.Query() 47 | 48 | var startOffset string 49 | if start != 0 { 50 | startOffset = strconv.Itoa(start) 51 | } 52 | if startOffset != "" { 53 | q.Add("offset", startOffset) 54 | } 55 | 56 | if watch { 57 | q.Add("watch", strconv.FormatBool(watch)) 58 | } 59 | 60 | req.URL.RawQuery = q.Encode() 61 | 62 | c := http.Client{ 63 | Timeout: time.Second * 10, // will also terminate watch after this time 64 | } 65 | res, err := c.Do(req) 66 | if err != nil { 67 | l.Fatal("could not send request", zap.Error(err)) 68 | } 69 | defer func() { 70 | if err = res.Body.Close(); err != nil { 71 | l.Error("could not close response body", zap.Error(err)) 72 | } 73 | }() 74 | 75 | if res.StatusCode > 299 { 76 | l.Fatal("could not read event stream", zap.Int("statusCode", res.StatusCode)) 77 | } 78 | 79 | scanner := bufio.NewScanner(res.Body) 80 | for scanner.Scan() { 81 | l.Debug("received new event", zap.String("event", scanner.Text())) 82 | } 83 | 84 | if scanner.Err() != nil { 85 | l.Fatal("could not read response body", zap.Error(scanner.Err())) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /cmd/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "sync" 12 | "syscall" 13 | "time" 14 | 15 | "github.com/embano1/memlog" 16 | "github.com/embano1/vsphere/event" 17 | "github.com/embano1/vsphere/logger" 18 | "github.com/kelseyhightower/envconfig" 19 | "github.com/vmware/govmomi/vim25/types" 20 | "go.uber.org/zap" 21 | "golang.org/x/sync/errgroup" 22 | ) 23 | 24 | var pollInterval = time.Second // poll vcenter events 25 | 26 | func main() { 27 | var env envConfig 28 | if err := envconfig.Process("", &env); err != nil { 29 | panic("could not process environment variables: " + err.Error()) 30 | } 31 | 32 | var l *zap.Logger 33 | if env.Debug { 34 | zapLogger, err := zap.NewDevelopment() 35 | if err != nil { 36 | panic("could not create logger: " + err.Error()) 37 | } 38 | l = zapLogger 39 | 40 | } else { 41 | zapLogger, err := zap.NewProduction() 42 | if err != nil { 43 | panic("could not create logger: " + err.Error()) 44 | } 45 | l = zapLogger 46 | } 47 | 48 | l = l.Named("eventstream") 49 | ctx := logger.Set(context.Background(), l) 50 | ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) 51 | defer cancel() 52 | 53 | srv, err := newServer(ctx, fmt.Sprintf("0.0.0.0:%d", env.Port)) 54 | if err != nil { 55 | l.Fatal("could not create server", zap.Error(err)) 56 | } 57 | 58 | if err = run(ctx, srv); err != nil && !errors.Is(err, context.Canceled) { 59 | l.Fatal("could not run server", zap.Error(err)) 60 | } 61 | } 62 | 63 | func run(ctx context.Context, srv *server) error { 64 | var env envConfig 65 | if err := envconfig.Process("", &env); err != nil { 66 | return fmt.Errorf("process environment variables: %w", err) 67 | } 68 | 69 | l := logger.Get(ctx) 70 | eg, egCtx := errgroup.WithContext(ctx) 71 | 72 | eg.Go(func() error { 73 | <-egCtx.Done() 74 | 75 | // use fresh ctx to give clients time to disconnect 76 | shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) 77 | l.Debug("stopping server", zap.Duration("timeout", shutdownTimeout)) 78 | defer cancel() 79 | if err := srv.stop(shutdownCtx); err != nil { 80 | l.Error("could not gracefully stop server", zap.Error(err)) 81 | } 82 | return egCtx.Err() 83 | }) 84 | 85 | var once sync.Once 86 | eg.Go(func() error { 87 | root := srv.vc.SOAP.ServiceContent.RootFolder 88 | mgr := srv.vc.Events 89 | source := srv.vc.SOAP.URL().String() 90 | start := types.EventFilterSpecByTime{ 91 | BeginTime: types.NewTime(time.Now().UTC().Add(-5 * time.Minute)), 92 | } 93 | 94 | collector, err := event.NewHistoryCollector(egCtx, mgr, root, event.WithTime(&start)) 95 | if err != nil { 96 | return fmt.Errorf("create event collector: %w", err) 97 | } 98 | 99 | l.Info("starting vsphere event collector", zap.Duration("begin", env.StreamBegin), zap.Duration("pollInterval", pollInterval)) 100 | ticker := time.NewTicker(pollInterval) 101 | defer ticker.Stop() 102 | for { 103 | select { 104 | case <-egCtx.Done(): 105 | return egCtx.Err() 106 | case <-ticker.C: 107 | events, err := collector.ReadNextEvents(egCtx, 50) 108 | if err != nil { 109 | return fmt.Errorf("read events: %w", err) 110 | } 111 | 112 | for _, e := range events { 113 | id := e.GetEvent().Key 114 | 115 | // set event ID as start offset 116 | once.Do(func() { 117 | l.Debug("initializing new log", 118 | zap.Int32("startOffset", id), 119 | zap.Int("maxSegmentSize", env.SegmentSize), 120 | zap.Int("maxRecordSize", env.RecordSize), 121 | ) 122 | if err := srv.initializeLog(egCtx, memlog.Offset(id), env.SegmentSize, env.RecordSize); err != nil { 123 | l.Fatal("initialize log", zap.Error(err)) 124 | } 125 | }) 126 | 127 | details := event.GetDetails(e) 128 | cevent, err := event.ToCloudEvent(source, e, map[string]string{"eventclass": details.Class}) 129 | if err != nil { 130 | l.Error("convert vsphere event to cloudevent", zap.Error(err), zap.Any("event", e)) 131 | return fmt.Errorf("convert vsphere event to cloudevent: %w", err) 132 | } 133 | 134 | b, err := json.Marshal(cevent) 135 | if err != nil { 136 | l.Error("marshal cloudevent to JSON", zap.Error(err), zap.String("event", cevent.String())) 137 | return fmt.Errorf("marshal cloudevent to JSON: %w", err) 138 | } 139 | 140 | offset, err := srv.log.Write(egCtx, b) 141 | if err != nil { 142 | return fmt.Errorf("write to log: %w", err) 143 | } 144 | l.Debug("wrote cloudevent to log", 145 | zap.Any("offset", offset), 146 | zap.String("event", cevent.String()), 147 | zap.Int("bytes", len(b)), 148 | ) 149 | } 150 | } 151 | } 152 | }) 153 | 154 | eg.Go(func() error { 155 | l.Info("starting http listener", zap.String("address", srv.http.Addr)) 156 | if err := srv.http.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { 157 | return fmt.Errorf("serve http: %w", err) 158 | } 159 | return nil 160 | }) 161 | 162 | return eg.Wait() 163 | } 164 | -------------------------------------------------------------------------------- /cmd/server/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "os" 10 | "path/filepath" 11 | "testing" 12 | "time" 13 | 14 | ce "github.com/cloudevents/sdk-go/v2" 15 | "github.com/embano1/vsphere/client" 16 | "github.com/embano1/vsphere/logger" 17 | "github.com/julienschmidt/httprouter" 18 | "github.com/vmware/govmomi/simulator" 19 | _ "github.com/vmware/govmomi/vapi/simulator" 20 | "github.com/vmware/govmomi/vim25" 21 | "go.uber.org/zap/zaptest" 22 | "gotest.tools/v3/assert" 23 | ) 24 | 25 | const ( 26 | userFileKey = "username" 27 | passwordFileKey = "password" 28 | ) 29 | 30 | func Test_run(t *testing.T) { 31 | t.Run("successfully shuts down server", func(t *testing.T) { 32 | dir := tempDir(t) 33 | 34 | t.Cleanup(func() { 35 | err := os.RemoveAll(dir) 36 | assert.NilError(t, err) 37 | }) 38 | 39 | simulator.Run(func(ctx context.Context, vimclient *vim25.Client) error { 40 | ctx = logger.Set(ctx, zaptest.NewLogger(t)) 41 | 42 | pollInterval = time.Millisecond * 10 43 | ctx, cancel := context.WithCancel(ctx) 44 | defer cancel() 45 | 46 | t.Setenv("VCENTER_URL", vimclient.URL().String()) 47 | t.Setenv("VCENTER_INSECURE", "true") 48 | t.Setenv("VCENTER_SECRET_PATH", dir) 49 | 50 | vc, err := client.New(ctx) 51 | assert.NilError(t, err) 52 | 53 | const address = "127.0.0.1:8080" 54 | srv, err := newServer(ctx, address) 55 | assert.NilError(t, err) 56 | 57 | srv.vc = vc 58 | 59 | runErrCh := make(chan error) 60 | go func() { 61 | runErrCh <- run(ctx, srv) 62 | }() 63 | 64 | // give server time to initialize event stream log 65 | time.Sleep(time.Second) 66 | 67 | wanteventID := "20" 68 | rec := httptest.NewRecorder() 69 | req := httptest.NewRequest( 70 | http.MethodGet, 71 | fmt.Sprintf("http://%s/api/v1/events/%s", address, wanteventID), 72 | nil, 73 | ) 74 | 75 | h := srv.getEvent(ctx) 76 | h(rec, req, httprouter.Params{{ 77 | Key: "id", 78 | Value: wanteventID, 79 | }}) 80 | 81 | assert.Equal(t, rec.Code, http.StatusOK) 82 | 83 | var gotevent ce.Event 84 | err = json.NewDecoder(rec.Body).Decode(&gotevent) 85 | assert.NilError(t, err) 86 | assert.NilError(t, gotevent.Validate()) 87 | assert.Equal(t, gotevent.ID(), wanteventID) 88 | 89 | // stop server 90 | cancel() 91 | err = <-runErrCh 92 | assert.ErrorContains(t, err, "context canceled") 93 | 94 | return nil 95 | }) 96 | }) 97 | } 98 | 99 | func tempDir(t *testing.T) string { 100 | t.Helper() 101 | 102 | dir, err := os.MkdirTemp("", "") 103 | assert.NilError(t, err) 104 | 105 | f, err := os.Create(filepath.Join(dir, userFileKey)) 106 | assert.NilError(t, err) 107 | 108 | _, err = f.Write([]byte("user")) 109 | assert.NilError(t, err) 110 | assert.NilError(t, f.Close()) 111 | 112 | f, err = os.Create(filepath.Join(dir, passwordFileKey)) 113 | assert.NilError(t, err) 114 | 115 | _, err = f.Write([]byte("pass")) 116 | assert.NilError(t, err) 117 | assert.NilError(t, f.Close()) 118 | 119 | return dir 120 | } 121 | -------------------------------------------------------------------------------- /cmd/server/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "html" 9 | "io" 10 | "net/http" 11 | "strconv" 12 | "time" 13 | 14 | ce "github.com/cloudevents/sdk-go/v2" 15 | "github.com/embano1/memlog" 16 | "github.com/embano1/vsphere/client" 17 | "github.com/embano1/vsphere/logger" 18 | "github.com/google/uuid" 19 | "github.com/julienschmidt/httprouter" 20 | "go.uber.org/zap" 21 | ) 22 | 23 | const ( 24 | // http defaults 25 | apiPath = "/api/v1" 26 | readTimeout = 3 * time.Second 27 | streamTimeout = 5 * time.Minute // forces stream disconnect after this time 28 | shutdownTimeout = 5 * time.Second 29 | pageSize = 50 30 | offsetKey = "offset" 31 | watchKey = "watch" 32 | ) 33 | 34 | type server struct { 35 | http *http.Server 36 | vc *client.Client // vsphere 37 | log *memlog.Log 38 | } 39 | 40 | type logRange struct { 41 | Earliest memlog.Offset `json:"earliest"` 42 | Latest memlog.Offset `json:"latest"` 43 | } 44 | 45 | type envConfig struct { 46 | RecordSize int `envconfig:"LOG_MAX_RECORD_SIZE_BYTES" required:"true" default:"524288"` 47 | SegmentSize int `envconfig:"LOG_MAX_SEGMENT_SIZE" required:"true" default:"1000"` 48 | StreamBegin time.Duration `envconfig:"VCENTER_STREAM_BEGIN" required:"true" default:"10m"` 49 | Port int `envconfig:"PORT" required:"true" default:"8080"` 50 | Debug bool `envconfig:"DEBUG" default:"false"` 51 | } 52 | 53 | func newServer(ctx context.Context, address string) (*server, error) { 54 | var srv server 55 | vc, err := client.New(ctx) 56 | if err != nil { 57 | return nil, fmt.Errorf("create vsphere client: %w", err) 58 | } 59 | srv.vc = vc 60 | 61 | router := httprouter.New() 62 | router.GET(apiPath+"/events", srv.getEvents(ctx)) 63 | router.GET(apiPath+"/events/:id", srv.getEvent(ctx)) 64 | router.GET(apiPath+"/range", srv.getRange(ctx)) 65 | 66 | h := http.Server{ 67 | Addr: address, 68 | Handler: router, 69 | ReadTimeout: readTimeout, 70 | WriteTimeout: streamTimeout, 71 | } 72 | srv.http = &h 73 | 74 | return &srv, nil 75 | } 76 | 77 | func (s *server) initializeLog(ctx context.Context, start memlog.Offset, segmentSize, recordSize int) error { 78 | if s.log != nil { 79 | return nil 80 | } 81 | 82 | opts := []memlog.Option{ 83 | memlog.WithStartOffset(start), 84 | memlog.WithMaxSegmentSize(segmentSize), 85 | memlog.WithMaxRecordDataSize(recordSize), 86 | } 87 | ml, err := memlog.New(ctx, opts...) 88 | if err != nil { 89 | return fmt.Errorf("create log: %w", err) 90 | } 91 | s.log = ml 92 | 93 | return nil 94 | } 95 | 96 | func (s *server) stop(ctx context.Context) error { 97 | if err := s.http.Shutdown(ctx); err != nil { 98 | return err 99 | } 100 | return nil 101 | } 102 | 103 | // returns the last page 104 | // if "watch=true" starts streaming from next (latest+1) 105 | // if "watch=true" and a valid "offset" is specified starts streaming from offset 106 | func (s *server) getEvents(ctx context.Context) httprouter.Handle { 107 | return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 108 | var watch bool 109 | 110 | if val := r.FormValue(watchKey); val != "" { 111 | val = html.EscapeString(val) 112 | switch val { 113 | case "true": 114 | watch = true 115 | default: 116 | http.Error(w, "invalid watch parameter", http.StatusBadRequest) 117 | return 118 | } 119 | } 120 | 121 | if !watch { 122 | s.readEvents(ctx, w, r) 123 | return 124 | } 125 | 126 | s.streamEvents(ctx, w, r) 127 | } 128 | } 129 | 130 | func (s *server) streamEvents(ctx context.Context, w http.ResponseWriter, r *http.Request) { 131 | log := logger.Get(ctx).With(zap.String("streamID", uuid.New().String())) 132 | log.Debug("new stream request") 133 | defer func() { 134 | log.Debug("stream stopped") 135 | }() 136 | 137 | flusher, ok := w.(http.Flusher) 138 | if !ok { 139 | log.Error("writer does not implement flusher") 140 | w.WriteHeader(http.StatusInternalServerError) 141 | return 142 | } 143 | 144 | w.Header().Set("Connection", "Keep-Alive") 145 | w.Header().Set("X-Content-Type-Options", "nosniff") 146 | w.Header().Set("Content-Type", "application/json") 147 | 148 | rctx := r.Context() 149 | start := memlog.Offset(-1) 150 | 151 | if o := r.FormValue(offsetKey); o != "" { 152 | o = html.EscapeString(o) 153 | offset, err := strconv.Atoi(o) 154 | if err != nil { 155 | http.Error(w, "invalid offset", http.StatusBadRequest) 156 | return 157 | } 158 | start = memlog.Offset(offset) 159 | } 160 | 161 | if start == -1 { 162 | log.Debug("no start offset specified") 163 | earliest, latest := s.log.Range(rctx) 164 | log.Debug("current log range", zap.Any("earliest", earliest), zap.Any("latest", latest)) 165 | start = latest + 1 166 | } 167 | 168 | log.Debug("starting stream", zap.Any("start", start)) 169 | stream := s.log.Stream(rctx, start) 170 | 171 | for { 172 | // give a chance for server shutdown (not guaranteed) 173 | if ctx.Err() != nil { 174 | return 175 | } 176 | 177 | if rec, ok := stream.Next(); ok { 178 | b := rec.Data 179 | b = append(b, byte('\n')) 180 | data := string(b) 181 | _, err := io.WriteString(w, data) 182 | if err != nil { 183 | log.Error("write event", zap.Error(err)) 184 | w.WriteHeader(http.StatusInternalServerError) 185 | return 186 | } 187 | 188 | log.Debug("sending event", zap.String("event", data)) 189 | flusher.Flush() 190 | continue 191 | } 192 | break 193 | } 194 | 195 | if err := stream.Err(); err != nil { 196 | if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 197 | return 198 | } 199 | 200 | if errors.Is(err, memlog.ErrOutOfRange) { 201 | http.Error(w, "invalid offset: "+err.Error(), http.StatusBadRequest) 202 | return 203 | } 204 | 205 | log.Error("failed to stream", zap.Error(err)) 206 | w.WriteHeader(http.StatusInternalServerError) 207 | return 208 | } 209 | } 210 | 211 | func (s *server) readEvents(ctx context.Context, w http.ResponseWriter, r *http.Request) { 212 | log := logger.Get(ctx) 213 | 214 | rctx := r.Context() 215 | earliest, latest := s.log.Range(rctx) 216 | 217 | // empty log 218 | if latest == -1 { 219 | w.WriteHeader(http.StatusNoContent) 220 | return 221 | } 222 | 223 | start := getStart(earliest, latest, pageSize) 224 | 225 | var events []ce.Event 226 | for i := start; i <= latest; i++ { 227 | record, err := s.log.Read(rctx, i) 228 | if err != nil { 229 | // client gone 230 | if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 231 | return 232 | } 233 | 234 | // purged record, continue 235 | if errors.Is(err, memlog.ErrOutOfRange) { 236 | continue 237 | } 238 | 239 | log.Error("read record", zap.Error(err)) 240 | w.WriteHeader(http.StatusInternalServerError) 241 | return 242 | } 243 | 244 | var e ce.Event 245 | if err = json.Unmarshal(record.Data, &e); err != nil { 246 | log.Error("unmarshal event", zap.Error(err)) 247 | w.WriteHeader(http.StatusInternalServerError) 248 | return 249 | } 250 | 251 | events = append(events, e) 252 | } 253 | 254 | b, err := json.Marshal(events) 255 | if err != nil { 256 | log.Error("marshal events response", zap.Error(err)) 257 | w.WriteHeader(http.StatusInternalServerError) 258 | return 259 | } 260 | 261 | w.Header().Set("Content-Type", "application/json") 262 | w.Header().Set("Content-Length", strconv.Itoa(len(b))) 263 | _, err = w.Write(b) 264 | if err != nil { 265 | log.Error("write response", zap.Error(err)) 266 | } 267 | } 268 | 269 | func (s *server) getEvent(ctx context.Context) httprouter.Handle { 270 | return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { 271 | id := ps.ByName("id") 272 | offset, err := strconv.Atoi(id) 273 | if err != nil { 274 | http.Error(w, "invalid offset", http.StatusBadRequest) 275 | return 276 | } 277 | 278 | rctx := r.Context() 279 | rec, err := s.log.Read(rctx, memlog.Offset(offset)) 280 | if err != nil { 281 | if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 282 | return 283 | } 284 | 285 | if errors.Is(err, memlog.ErrOutOfRange) || errors.Is(err, memlog.ErrFutureOffset) { 286 | http.Error(w, "invalid offset: "+err.Error(), http.StatusBadRequest) 287 | return 288 | } 289 | 290 | logger.Get(ctx).Error("read record", zap.Error(err)) 291 | w.WriteHeader(http.StatusInternalServerError) 292 | return 293 | } 294 | w.Header().Set("Content-Type", "application/json") 295 | _, err = io.WriteString(w, string(rec.Data)) 296 | if err != nil { 297 | logger.Get(ctx).Error("write event", zap.Error(err)) 298 | w.WriteHeader(http.StatusInternalServerError) 299 | return 300 | } 301 | } 302 | } 303 | 304 | // 204 on empty log 305 | func (s *server) getRange(ctx context.Context) httprouter.Handle { 306 | return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 307 | rctx := r.Context() 308 | earliest, latest := s.log.Range(rctx) 309 | 310 | if latest == -1 { 311 | w.WriteHeader(http.StatusNoContent) 312 | return 313 | } 314 | 315 | w.Header().Set("Content-Type", "application/json") 316 | lr := logRange{ 317 | Earliest: earliest, 318 | Latest: latest, 319 | } 320 | 321 | if err := json.NewEncoder(w).Encode(lr); err != nil { 322 | logger.Get(ctx).Error("marshal range response", zap.Error(err)) 323 | w.WriteHeader(http.StatusInternalServerError) 324 | return 325 | } 326 | } 327 | } 328 | 329 | func getStart(earliest, latest memlog.Offset, pageSize int) memlog.Offset { 330 | start := earliest 331 | if int(latest-earliest+1) > pageSize { 332 | start = latest - memlog.Offset(pageSize) + 1 // include latest in range 333 | } 334 | 335 | return start 336 | } 337 | -------------------------------------------------------------------------------- /cmd/server/server_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "net/http" 8 | "net/http/httptest" 9 | "strconv" 10 | "strings" 11 | "testing" 12 | "time" 13 | 14 | ce "github.com/cloudevents/sdk-go/v2" 15 | "github.com/embano1/memlog" 16 | "github.com/embano1/vsphere/logger" 17 | "github.com/google/go-cmp/cmp" 18 | "github.com/julienschmidt/httprouter" 19 | "go.uber.org/zap/zaptest" 20 | "gotest.tools/v3/assert" 21 | ) 22 | 23 | func Test_getRange(t *testing.T) { 24 | tests := []struct { 25 | name string 26 | start memlog.Offset 27 | size int 28 | data [][]byte 29 | wantCode int 30 | wantContentType string 31 | wantRange string // json 32 | }{ 33 | { 34 | name: "204 on empty log", 35 | start: 0, 36 | size: 10, 37 | data: nil, 38 | wantCode: http.StatusNoContent, 39 | wantContentType: "", 40 | wantRange: "", 41 | }, 42 | { 43 | name: "returns range for not truncated log", 44 | start: 0, 45 | size: 10, 46 | data: createData(5), 47 | wantCode: http.StatusOK, 48 | wantContentType: "application/json", 49 | wantRange: `{"earliest":0,"latest":4}`, 50 | }, 51 | { 52 | name: "returns range after truncated log", 53 | start: 0, 54 | size: 5, // current + history = 10 55 | data: createData(20), 56 | wantCode: http.StatusOK, 57 | wantContentType: "application/json", 58 | wantRange: `{"earliest":10,"latest":19}`, 59 | }, 60 | } 61 | 62 | for _, tc := range tests { 63 | t.Run(tc.name, func(t *testing.T) { 64 | ctx := context.Background() 65 | opts := []memlog.Option{ 66 | memlog.WithStartOffset(tc.start), 67 | memlog.WithMaxSegmentSize(tc.size), 68 | } 69 | log, err := memlog.New(ctx, opts...) 70 | assert.NilError(t, err) 71 | 72 | for _, v := range tc.data { 73 | _, err = log.Write(ctx, v) 74 | assert.NilError(t, err) 75 | } 76 | 77 | srv := server{ 78 | log: log, 79 | } 80 | 81 | rec := httptest.NewRecorder() 82 | req := httptest.NewRequest(http.MethodGet, "/range", nil) 83 | 84 | h := srv.getRange(ctx) 85 | h(rec, req, nil) 86 | 87 | assert.Equal(t, rec.Result().Header.Get("content-type"), tc.wantContentType) 88 | assert.Equal(t, rec.Result().StatusCode, tc.wantCode) 89 | assert.Equal(t, strings.TrimRight(rec.Body.String(), "\n"), tc.wantRange) 90 | }) 91 | } 92 | } 93 | 94 | func Test_getEvent(t *testing.T) { 95 | tests := []struct { 96 | name string 97 | start memlog.Offset 98 | size int 99 | data [][]byte 100 | eventID string 101 | wantCode int 102 | wantContentType string 103 | want string 104 | }{ 105 | { 106 | name: "400 future offset on empty log", 107 | start: 0, 108 | size: 10, 109 | data: nil, 110 | eventID: "3", 111 | wantCode: http.StatusBadRequest, 112 | wantContentType: "text/plain; charset=utf-8", 113 | want: "future offset", 114 | }, 115 | { 116 | name: "400 invalid offset on truncated log", 117 | start: 0, 118 | size: 5, 119 | data: createData(20), 120 | eventID: "3", 121 | wantCode: http.StatusBadRequest, 122 | wantContentType: "text/plain; charset=utf-8", 123 | want: "invalid offset", 124 | }, 125 | { 126 | name: "400 id is not a number", 127 | start: 0, 128 | size: 10, 129 | data: createData(10), 130 | eventID: "blabla", 131 | wantCode: http.StatusBadRequest, 132 | wantContentType: "text/plain; charset=utf-8", 133 | want: "invalid offset", 134 | }, 135 | { 136 | name: "200 returns event on not truncated log", 137 | start: 0, 138 | size: 10, 139 | data: createData(10), 140 | eventID: "3", 141 | wantCode: http.StatusOK, 142 | wantContentType: "application/json", 143 | want: "3", 144 | }, 145 | { 146 | name: "200 returns event on truncated log", 147 | start: 0, 148 | size: 5, 149 | data: createData(20), 150 | eventID: "11", 151 | wantCode: http.StatusOK, 152 | wantContentType: "application/json", 153 | want: "11", 154 | }, 155 | { 156 | name: "200 returns event on not truncated log with start offset 10", 157 | start: 10, 158 | size: 10, 159 | data: createData(10), 160 | eventID: "11", 161 | wantCode: http.StatusOK, 162 | wantContentType: "application/json", 163 | want: "1", 164 | }, 165 | { 166 | name: "200 returns event on truncated log with start offset 20", 167 | start: 20, 168 | size: 5, 169 | data: createData(20), 170 | eventID: "31", 171 | wantCode: http.StatusOK, 172 | wantContentType: "application/json", 173 | want: "11", 174 | }, 175 | } 176 | 177 | for _, tc := range tests { 178 | t.Run(tc.name, func(t *testing.T) { 179 | ctx := context.Background() 180 | opts := []memlog.Option{ 181 | memlog.WithStartOffset(tc.start), 182 | memlog.WithMaxSegmentSize(tc.size), 183 | } 184 | log, err := memlog.New(ctx, opts...) 185 | assert.NilError(t, err) 186 | 187 | for _, v := range tc.data { 188 | _, err = log.Write(ctx, v) 189 | assert.NilError(t, err) 190 | } 191 | 192 | srv := server{ 193 | log: log, 194 | } 195 | 196 | rec := httptest.NewRecorder() 197 | req := httptest.NewRequest(http.MethodGet, "/events/"+tc.eventID, nil) 198 | 199 | h := srv.getEvent(ctx) 200 | h(rec, req, []httprouter.Param{{Key: "id", Value: tc.eventID}}) 201 | 202 | assert.Equal(t, rec.Result().Header.Get("content-type"), tc.wantContentType) 203 | assert.Equal(t, rec.Result().StatusCode, tc.wantCode) 204 | if !assert.Check(t, strings.Contains(rec.Body.String(), tc.want)) { 205 | t.Logf("got vs want diff: %s", cmp.Diff(rec.Body.String(), tc.want)) 206 | } 207 | }) 208 | } 209 | } 210 | 211 | func Test_getEvents(t *testing.T) { 212 | tests := []struct { 213 | name string 214 | start memlog.Offset 215 | size int 216 | data [][]byte 217 | wantCode int 218 | wantContentType string 219 | }{ 220 | { 221 | name: "204 on empty log", 222 | start: 0, 223 | size: 10, 224 | data: nil, 225 | wantCode: 204, 226 | wantContentType: "", 227 | }, 228 | { 229 | name: "200, log with 3 entries, returns 3 results", 230 | start: 0, 231 | size: 10, 232 | data: createData(3), 233 | wantCode: 200, 234 | wantContentType: "application/json", 235 | }, 236 | } 237 | 238 | for _, tc := range tests { 239 | t.Run(tc.name, func(t *testing.T) { 240 | ctx := context.Background() 241 | opts := []memlog.Option{ 242 | memlog.WithStartOffset(tc.start), 243 | memlog.WithMaxSegmentSize(tc.size), 244 | } 245 | log, err := memlog.New(ctx, opts...) 246 | assert.NilError(t, err) 247 | 248 | var ( 249 | want []ce.Event 250 | now = time.Now().UTC() 251 | ) 252 | 253 | for _, v := range tc.data { 254 | // getevents expects CloudEvents 255 | e := ce.NewEvent() 256 | e.SetID(string(v)) 257 | e.SetType("test.event.v0") 258 | e.SetTime(now) 259 | e.SetSource("/test/source") 260 | err = e.SetData(ce.ApplicationJSON, string(v)) 261 | assert.NilError(t, err) 262 | 263 | b, err := json.Marshal(e) 264 | assert.NilError(t, err) 265 | 266 | _, err = log.Write(ctx, b) 267 | assert.NilError(t, err) 268 | 269 | want = append(want, e) 270 | } 271 | 272 | srv := server{ 273 | log: log, 274 | } 275 | 276 | rec := httptest.NewRecorder() 277 | req := httptest.NewRequest(http.MethodGet, "/events", nil) 278 | 279 | h := srv.getEvents(ctx) 280 | h(rec, req, nil) 281 | 282 | assert.Equal(t, rec.Result().StatusCode, tc.wantCode) 283 | 284 | var got []ce.Event 285 | err = json.NewDecoder(rec.Body).Decode(&got) 286 | assert.Equal(t, rec.Result().Header.Get("content-type"), tc.wantContentType) 287 | assert.Assert(t, err == nil || err == io.EOF) // empty response throws EOF 288 | assert.DeepEqual(t, want, got) 289 | }) 290 | } 291 | } 292 | 293 | func Test_streamEvents(t *testing.T) { 294 | tests := []struct { 295 | name string 296 | start memlog.Offset 297 | size int 298 | data [][]byte 299 | watchParam string 300 | offsetParam string 301 | wantCode int 302 | wantContentType string 303 | wantResult []byte 304 | }{ 305 | { 306 | name: "200, no data on empty log", 307 | start: 0, 308 | size: 10, 309 | data: nil, 310 | watchParam: "true", 311 | offsetParam: "", 312 | wantCode: 200, 313 | wantContentType: "application/json", 314 | wantResult: []byte{}, 315 | }, 316 | { 317 | name: "400, invalid watch param specified", 318 | start: 0, 319 | size: 10, 320 | data: nil, 321 | watchParam: "invalid", 322 | offsetParam: "", 323 | wantCode: 400, 324 | wantContentType: "text/plain; charset=utf-8", 325 | wantResult: []byte("invalid watch parameter\n"), 326 | }, 327 | { 328 | name: "200, write 3 records to log, no offset specified, no data returned", 329 | start: 0, 330 | size: 10, 331 | data: createData(3), 332 | watchParam: "true", 333 | offsetParam: "", 334 | wantCode: 200, 335 | wantContentType: "application/json", 336 | wantResult: []byte{}, 337 | }, 338 | { 339 | name: "200, write 3 records to log, offset 0, 3 records returned", 340 | start: 0, 341 | size: 10, 342 | data: createData(3), 343 | watchParam: "true", 344 | offsetParam: "0", 345 | wantCode: 200, 346 | wantContentType: "application/json", 347 | wantResult: []byte("0\n1\n2\n"), 348 | }, 349 | { 350 | name: "400, write 20 records to log with size 5, offset 0, out of range", 351 | start: 0, 352 | size: 5, 353 | data: createData(20), 354 | watchParam: "true", 355 | offsetParam: "0", 356 | wantCode: 400, 357 | wantContentType: "text/plain; charset=utf-8", 358 | wantResult: []byte("invalid offset: offset out of range\n"), 359 | }, 360 | { 361 | name: "400, write 15 records to log with size 5, offset 10, 5 records returned", 362 | start: 0, 363 | size: 5, 364 | data: createData(15), 365 | watchParam: "true", 366 | offsetParam: "10", 367 | wantCode: 200, 368 | wantContentType: "application/json", 369 | wantResult: []byte("10\n11\n12\n13\n14\n"), 370 | }, 371 | } 372 | 373 | for _, tc := range tests { 374 | t.Run(tc.name, func(t *testing.T) { 375 | ctx := logger.Set(context.Background(), zaptest.NewLogger(t)) 376 | 377 | opts := []memlog.Option{ 378 | memlog.WithStartOffset(tc.start), 379 | memlog.WithMaxSegmentSize(tc.size), 380 | } 381 | log, err := memlog.New(ctx, opts...) 382 | assert.NilError(t, err) 383 | 384 | for _, v := range tc.data { 385 | _, err = log.Write(ctx, v) 386 | assert.NilError(t, err) 387 | } 388 | 389 | srv := server{ 390 | log: log, 391 | } 392 | 393 | rec := httptest.NewRecorder() 394 | req := httptest.NewRequest(http.MethodGet, "/events", nil) 395 | 396 | ctx, cancel := context.WithTimeout(ctx, time.Millisecond*100) 397 | defer cancel() 398 | 399 | req = req.WithContext(ctx) 400 | q := req.URL.Query() 401 | q.Add(watchKey, tc.watchParam) 402 | 403 | if tc.offsetParam != "" { 404 | q.Add(offsetKey, tc.offsetParam) 405 | } 406 | req.URL.RawQuery = q.Encode() 407 | 408 | h := srv.getEvents(ctx) 409 | h(rec, req, nil) 410 | 411 | assert.Equal(t, rec.Result().StatusCode, tc.wantCode) 412 | assert.Equal(t, rec.Result().Header.Get("content-type"), tc.wantContentType) 413 | assert.Equal(t, rec.Body.String(), string(tc.wantResult)) 414 | }) 415 | } 416 | } 417 | 418 | func Test_getStart(t *testing.T) { 419 | type args struct { 420 | earliest memlog.Offset 421 | latest memlog.Offset 422 | pageSize int 423 | } 424 | tests := []struct { 425 | name string 426 | args args 427 | want memlog.Offset 428 | }{ 429 | { 430 | name: "empty log", 431 | args: args{ 432 | earliest: -1, 433 | latest: -1, 434 | pageSize: 50, 435 | }, 436 | want: -1, 437 | }, 438 | { 439 | name: "earliest 0, latest 10, page 50", 440 | args: args{ 441 | earliest: 0, 442 | latest: 10, 443 | pageSize: 50, 444 | }, 445 | want: 0, 446 | }, 447 | { 448 | name: "earliest 0, latest 100, page 50", 449 | args: args{ 450 | earliest: 0, 451 | latest: 100, 452 | pageSize: 50, 453 | }, 454 | want: 51, 455 | }, 456 | { 457 | name: "earliest 99, latest 100, page 50", 458 | args: args{ 459 | earliest: 99, 460 | latest: 100, 461 | pageSize: 50, 462 | }, 463 | want: 99, 464 | }, 465 | { 466 | name: "earliest 99, latest 100, page 50", 467 | args: args{ 468 | earliest: 99, 469 | latest: 100, 470 | pageSize: 50, 471 | }, 472 | want: 99, 473 | }, 474 | { 475 | name: "earliest 51, latest 89, page 50", 476 | args: args{ 477 | earliest: 51, 478 | latest: 89, 479 | pageSize: 50, 480 | }, 481 | want: 51, 482 | }, 483 | { 484 | name: "earliest 151, latest 304, page 50", 485 | args: args{ 486 | earliest: 151, 487 | latest: 304, 488 | pageSize: 50, 489 | }, 490 | want: 255, 491 | }, 492 | { 493 | name: "earliest 151, latest 304, page 10", 494 | args: args{ 495 | earliest: 151, 496 | latest: 304, 497 | pageSize: 10, 498 | }, 499 | want: 295, 500 | }, 501 | } 502 | 503 | for _, tt := range tests { 504 | t.Run(tt.name, func(t *testing.T) { 505 | if got := getStart(tt.args.earliest, tt.args.latest, tt.args.pageSize); got != tt.want { 506 | t.Errorf("getStart() = %v, want %v", got, tt.want) 507 | } 508 | }) 509 | } 510 | } 511 | 512 | // createData returns a slice of []byte with the number of elements specified by 513 | // vals. The value of each element is the current index converted to a string 514 | // starting at 0. 515 | func createData(vals int) [][]byte { 516 | data := make([][]byte, vals) 517 | 518 | for i := 0; i < vals; i++ { 519 | data[i] = []byte(strconv.Itoa(i)) 520 | } 521 | 522 | return data 523 | } 524 | -------------------------------------------------------------------------------- /config/server.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: &applabels 5 | app: vsphere-event-stream-prototype 6 | name: vsphere-event-stream 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: *applabels 11 | template: 12 | metadata: 13 | labels: *applabels 14 | spec: 15 | containers: 16 | - image: ko://github.com/embano1/vsphere-event-streaming/cmd/server 17 | name: stream-server 18 | env: 19 | - name: VCENTER_INSECURE 20 | value: "true" 21 | - name: VCENTER_URL 22 | value: "https://vcsim.vcenter-stream.svc.cluster.local" 23 | - name: PORT 24 | value: "8080" #default 25 | - name: VCENTER_STREAM_BEGIN 26 | value: "5m" # default 27 | - name: LOG_MAX_RECORD_SIZE_BYTES 28 | value: "524288" # default (512Kb) 29 | - name: LOG_MAX_SEGMENT_SIZE 30 | value: "1000" # default 31 | - name: DEBUG 32 | value: "true" # print debug logs 33 | - name: VCENTER_SECRET_PATH 34 | value: "/var/bindings/vsphere" # this is the default path 35 | resources: 36 | requests: 37 | cpu: 200m 38 | memory: 512Mi 39 | limits: 40 | cpu: 500m 41 | memory: 512Mi 42 | volumeMounts: 43 | - name: credentials 44 | mountPath: /var/bindings/vsphere # this is the default path 45 | readOnly: true 46 | volumes: 47 | - name: credentials 48 | secret: 49 | secretName: vsphere-credentials 50 | --- 51 | apiVersion: v1 52 | kind: Service 53 | metadata: 54 | labels: &applabels 55 | app: vsphere-event-stream-prototype 56 | name: vsphere-event-stream 57 | spec: 58 | ports: 59 | - port: 80 60 | protocol: TCP 61 | targetPort: 8080 62 | selector: *applabels -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/embano1/vsphere-event-streaming 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/cloudevents/sdk-go/v2 v2.14.0 7 | github.com/embano1/memlog v0.4.4 8 | github.com/embano1/vsphere v0.2.5 9 | github.com/google/go-cmp v0.5.9 10 | github.com/google/uuid v1.3.0 11 | github.com/julienschmidt/httprouter v1.3.0 12 | github.com/kelseyhightower/envconfig v1.4.0 13 | github.com/vmware/govmomi v0.30.4 14 | go.uber.org/zap v1.24.0 15 | golang.org/x/sync v0.2.0 16 | gotest.tools/v3 v3.4.0 17 | ) 18 | 19 | require ( 20 | github.com/benbjohnson/clock v1.3.5 // indirect 21 | github.com/hashicorp/errwrap v1.1.0 // indirect 22 | github.com/hashicorp/go-multierror v1.1.1 // indirect 23 | github.com/json-iterator/go v1.1.12 // indirect 24 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 25 | github.com/modern-go/reflect2 v1.0.2 // indirect 26 | go.uber.org/atomic v1.11.0 // indirect 27 | go.uber.org/multierr v1.11.0 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= 2 | github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 3 | github.com/cloudevents/sdk-go/v2 v2.14.0 h1:Nrob4FwVgi5L4tV9lhjzZcjYqFVyJzsA56CwPaPfv6s= 4 | github.com/cloudevents/sdk-go/v2 v2.14.0/go.mod h1:xDmKfzNjM8gBvjaF8ijFjM1VYOVUEeUfapHMUX1T5To= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/embano1/memlog v0.4.4 h1:t12/1vR1RXYs/kRB0/Yl6d/CwwSmmJeEGF2sPcKUPR0= 9 | github.com/embano1/memlog v0.4.4/go.mod h1:KwZp72rqDg8jn7LgwwPST1VHuQbEACEpr23SaM0REbw= 10 | github.com/embano1/vsphere v0.2.5 h1:sQJ0neNVQ6nfqBZ6J/J2cmg+6TVFSBOFHL/kBY1leaw= 11 | github.com/embano1/vsphere v0.2.5/go.mod h1:eIUzez4XLPkzryqfVrOrQ/PlYkHslR7QfhQNRKlt0XA= 12 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 13 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 14 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 15 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 16 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 17 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 18 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 19 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 20 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 21 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 22 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 23 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 24 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 25 | github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= 26 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 27 | github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= 28 | github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= 29 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 30 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 31 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 32 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 33 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 34 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 35 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 36 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 37 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 38 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 39 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 40 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 41 | github.com/vmware/govmomi v0.30.4 h1:BCKLoTmiBYRuplv3GxKEMBLtBaJm8PA56vo9bddIpYQ= 42 | github.com/vmware/govmomi v0.30.4/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= 43 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 44 | go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 45 | go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 46 | go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= 47 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 48 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 49 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 50 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 51 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 52 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 53 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 54 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 55 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 56 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 57 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 58 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 59 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 60 | golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= 61 | golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 62 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 63 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 64 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 65 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 66 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 67 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 68 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 69 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 70 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 71 | golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= 72 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 73 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 74 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 75 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 76 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 77 | gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= 78 | gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= 79 | --------------------------------------------------------------------------------