├── .dockerignore ├── .github └── workflows │ └── go.yml ├── .gitignore ├── DESIGN.md ├── Dockerfile ├── LICENSE ├── README.md ├── codecov.yml ├── config ├── github_action_beam.json ├── prototype-db │ └── default.json ├── pulsar_beam.json ├── pulsar_beam.yml └── pulsar_beam_inmemory_db.yml ├── docker-compose.yml ├── go.mod ├── go.sum ├── scripts ├── ci.sh ├── secret-post-process.py ├── swagger.sh └── test_coverage.sh └── src ├── broker ├── sse-broker.go └── webhook.go ├── db ├── in-memory.go ├── interface.go ├── mongo.go └── pulsardb.go ├── docs ├── api.go └── docs.go ├── e2e ├── e2etest.go └── source_this_env.sh ├── icrypto ├── icrypto.go ├── pulsar-jwt.go └── util.go ├── main.go ├── middleware ├── middleware.go └── semaphore.go ├── model ├── message.go └── topic.go ├── pulsardriver ├── pulsar-client.go ├── pulsar-consumer.go └── pulsar-producer.go ├── route ├── error.go ├── handlers.go ├── logger.go ├── router.go ├── router_test.go └── routes.go ├── unit-test ├── crypto_test.go ├── db_test.go ├── example_private_key ├── example_public_key.pub ├── handlers_test.go ├── middleware_test.go ├── model_test.go ├── pk12-binary-private.key ├── pk12-binary-public.key ├── pulsar_test.go ├── test_util.go └── util_test.go └── util ├── cache-item.go ├── cert-loader.go ├── config.go ├── main_control.go ├── ttlcache.go └── util.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .git 3 | *.md 4 | LICENSE 5 | Makefile 6 | NOTICE 7 | arm/ 8 | powerpc/ 9 | mips/ -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | # Also trigger on page_build, as well as release created events 8 | page_build: 9 | release: 10 | types: # This configuration does not affect the page_build event above 11 | - created 12 | 13 | jobs: 14 | analysis: 15 | name: static analysis 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Set up Go 19 | uses: actions/setup-go@v1 20 | with: 21 | go-version: 1.17 22 | 23 | - name: Check out code 24 | uses: actions/checkout@v1 25 | with: 26 | fetch-depth: 1 27 | path: go/src/github.com/kafkaesque-io/pulsar-beam 28 | 29 | - name: Lint Go Code 30 | run: | 31 | export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14 32 | go get -u golang.org/x/lint/golint 33 | cd src 34 | golint ./... 35 | - name: Set up Python 3 36 | uses: actions/setup-python@v1 37 | with: 38 | python-version: '3.x' # Version range or exact version of a Python version to use, using SemVer's version range syntax 39 | architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified 40 | build_test: 41 | name: build and test 42 | runs-on: ubuntu-latest 43 | strategy: 44 | matrix: 45 | mongodb-version: [4.2] 46 | steps: 47 | - name: Set up Go 48 | uses: actions/setup-go@v1 49 | with: 50 | go-version: 1.17 51 | 52 | - name: Check out code 53 | uses: actions/checkout@v1 54 | with: 55 | fetch-depth: 1 56 | path: go/src/github.com/kafkaesque-io/pulsar-beam 57 | - name: Start MongoDB v${{ matrix.mongodb-version }} 58 | uses: supercharge/mongodb-github-action@1.2.0 59 | with: 60 | mongodb-version: ${{ matrix.mongodb-version }} 61 | - name: Verify MongoDB Installation and Status 62 | run: | 63 | sudo docker ps 64 | - name: Build Binary 65 | run: | 66 | go mod download 67 | cd src 68 | go build ./... 69 | - name: Go Vet 70 | run: | 71 | cd src 72 | go vet ./... 73 | - name: Set up root certificate 74 | env: 75 | PULSAR_CLIENT_CERT: ${{ secrets.PULSAR_CLIENT_CERT }} 76 | run: | 77 | pwd 78 | sudo apt-get install ca-certificates -y 79 | sudo mkdir -p /usr/local/share/ca-certificate 80 | echo $PULSAR_CLIENT_CERT | sed 's/\\n/\n/g' > ./pulsar-ca.crt 81 | sudo cp ./pulsar-ca.crt /usr/local/share/ca-certificate/pulsar-ca.crt 82 | ls /usr/local/share/ca-certificate 83 | sudo update-ca-certificates 84 | ls /etc/ssl/certs 85 | - name: Unit test 86 | run: | 87 | cd src/unit-test/ 88 | go test ./... 89 | - name: Run Test and Code Coverage 90 | run: | 91 | echo $TrustStore 92 | pwd 93 | cd ./scripts 94 | ./test_coverage.sh 95 | env: 96 | GOPATH: /home/runner/work/pulsar-beam/go 97 | TrustStore: /etc/ssl/certs/ca-certificates.crt 98 | PULSAR_TOKEN: ${{ secrets.PULSAR_TOKEN }} 99 | PULSAR_URI: ${{ secrets.PULSAR_URI }} 100 | REST_DB_TABLE_TOPIC: ${{ secrets.REST_DB_TABLE_TOPIC }} 101 | - name: Upload Coverage 102 | if: github.repository == 'kafkaesque-io/pulsar-beam' 103 | uses: codecov/codecov-action@v1.0.0 104 | with : 105 | token: ${{ secrets.CODECOV_TOKEN }} 106 | file: ./coverage.txt 107 | yml: ./.codecov.yml 108 | fail_ci_fi_error: true 109 | path: go/src/github.com/kafkaesque-io/pulsar-beam 110 | 111 | e2e_test: 112 | name: e2e_test 113 | needs: [analysis, build_test] 114 | runs-on: ubuntu-latest 115 | strategy: 116 | matrix: 117 | mongodb-version: [4.2] 118 | steps: 119 | - name: Check out code 120 | uses: actions/checkout@v1 121 | with: 122 | fetch-depth: 1 123 | path: go/src/github.com/kafkaesque-io/pulsar-beam 124 | 125 | - name: Install dependencies 126 | run: | 127 | pwd 128 | go mod download 129 | - name: Start MongoDB v${{ matrix.mongodb-version }} 130 | uses: supercharge/mongodb-github-action@1.2.0 131 | with: 132 | mongodb-version: ${{ matrix.mongodb-version }} 133 | - name: Verify MongoDB Installation and Status 134 | run: | 135 | sudo docker ps 136 | - name: Set up root certificate 137 | env: 138 | PULSAR_CLIENT_CERT: ${{ secrets.PULSAR_CLIENT_CERT }} 139 | run: | 140 | pwd 141 | sudo apt-get install ca-certificates -y 142 | sudo mkdir -p /usr/local/share/ca-certificate 143 | echo $PULSAR_CLIENT_CERT | sed 's/\\n/\n/g' > ./pulsar-ca.crt 144 | sudo cp ./pulsar-ca.crt /usr/local/share/ca-certificate/pulsar-ca.crt 145 | ls /usr/local/share/ca-certificate 146 | sudo update-ca-certificates 147 | ls /etc/ssl/certs 148 | 149 | docker: 150 | name: docker 151 | runs-on: ubuntu-latest 152 | steps: 153 | - name: Check out code 154 | uses: actions/checkout@v1 155 | with: 156 | fetch-depth: 1 157 | path: go/src/github.com/kafkaesque-io/pulsar-beam 158 | 159 | - name: Build Docker Image 160 | run: | 161 | pwd 162 | sudo docker build -t pulsar-beam . -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # visual studio code 15 | .vscode 16 | 17 | .env 18 | .pconfig 19 | 20 | *.sve 21 | 22 | coverage.out 23 | cover.out 24 | coverage.txt 25 | 26 | # build artifacts 27 | main 28 | 29 | /bin 30 | 31 | /vendor 32 | 33 | # swagger 34 | swagger.yaml -------------------------------------------------------------------------------- /DESIGN.md: -------------------------------------------------------------------------------- 1 | # Design 2 | 3 | ## Components 4 | Separation of responsiblity - Micro-services 5 | 1. Beam receives all the events in a simple endpoint 6 | 2. Beam pushes data(single or batch) via webhook 7 | 2. REST API for user based (used by UI) 8 | 9 | 10 | TODO : -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile References: https://docs.docker.com/engine/reference/builder/ 2 | 3 | # Start from the latest golang base image 4 | FROM golang:alpine AS builder 5 | 6 | # Add Maintainer Info 7 | LABEL maintainer="kesque" 8 | 9 | RUN apk --no-cache add build-base git 10 | 11 | # Build Delve 12 | RUN go install github.com/google/gops@latest 13 | 14 | WORKDIR /root/ 15 | ADD . /root 16 | RUN cd /root/src && go build -o pulsar-beam 17 | 18 | ######## Start a new stage from scratch ####### 19 | FROM alpine 20 | 21 | # RUN apk update 22 | WORKDIR /root/bin 23 | RUN mkdir /root/config/ 24 | 25 | # Copy the Pre-built binary file and default configuraitons from the previous stage 26 | COPY --from=builder /root/src/pulsar-beam /root/bin 27 | COPY --from=builder /root/config/pulsar_beam_inmemory_db.yml /root/config/pulsar_beam.yml 28 | COPY --from=builder /root/src/unit-test/example_p* /root/config/ 29 | 30 | # Copy debug tools 31 | COPY --from=builder /go/bin/gops /root/bin 32 | 33 | # Command to run the executable 34 | ENTRYPOINT ["./pulsar-beam"] 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2019-2020] [Kafkaesque.io] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Join the chat at https://gitter.im/FaradayRF/Lobby](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/kafkaesque-io/community?utm_source=badge&utm_medium=badge&utm_content=badge) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/kafkaesque-io/pulsar-beam)](https://goreportcard.com/report/github.com/kafkaesque-io/pulsar-beam) 3 | [![CI Build](https://github.com/kafkaesque-io/pulsar-beam/workflows/ci/badge.svg 4 | )](https://github.com/kafkaesque-io/pulsar-beam/actions) 5 | [![Language](https://img.shields.io/badge/Language-Go-blue.svg)](https://golang.org/) 6 | [![codecov](https://codecov.io/gh/kafkaesque-io/pulsar-beam/branch/master/graph/badge.svg)](https://codecov.io/gh/kafkaesque-io/pulsar-beam) 7 | [![Docker image](https://shields.beevelop.com/docker/image/image-size/kafkaesqueio/pulsar-beam/0.22.svg?style=round-square)](https://hub.docker.com/r/kafkaesqueio/pulsar-beam/) 8 | [![LICENSE](https://img.shields.io/hexpm/l/pulsar.svg)](https://github.com/kafkaesque-io/pulsar-beam/blob/master/LICENSE) 9 | 10 | # Pulsar Beam 11 | 12 | Beam is an http based streaming and queueing system backed up by Apache Pulsar. 13 | 14 | - [x] A message can be sent to Pulsar via an HTTP POST method as a producer. 15 | - [x] A message can be pushed to a webhook or Cloud Function for consumption. 16 | - [x] A webhook or Cloud Function receives a message, process it and reply another message, in a response body, back to another Pulsar topic via Pulsar Beam. 17 | - [x] Messages can be streamed via HTTP Sever Sent Event, [SSE](https://www.html5rocks.com/en/tutorials/eventsource/basics/) 18 | - [x] Support HTTP polling of batch messages 19 | 20 | Opening an issue and PR are welcomed! Please email `contact@kafkaesque.io` for any inquiry or demo. 21 | 22 | ## Advantages 23 | 1. Since Beam speaks http, it is language and OS independent. You can take advantage of powerhouse of Apache Pulsar without limitation of client library and OS. 24 | 25 | Immediately, Pulsar can be supported on Windows and any languages with HTTP support. 26 | 27 | 2. It has a very small footprint with a 15MB docker image size. 28 | 29 | 3. Supports HTTP SSE streaming 30 | 31 | ## Interface 32 | 33 | REST API and endpoint swagger document is published at [this link](https://kafkaesque-io.github.io/pulsar-beam-swagger/) 34 | 35 | ### Endpoint to send messages 36 | This is the endpoint to `POST` a message to Pulsar. 37 | 38 | ``` 39 | /v2/firehose/{persistent}/{tenant}/{namespace}/{topic} 40 | ``` 41 | Valid values of {persistent} are `p`, `persistent`, `np`, `nonpersistent` 42 | 43 | These HTTP headers may be required to map to Pulsar topic. 44 | 1. Authorization -> Bearer token as Pulsar token 45 | 2. PulsarUrl -> *optional* a fully qualified pulsar or pulsar+ssl URL where the message should be sent to. It is optional. The message will be sent to Pulsar URL specified under `PulsarBrokerURL` in the pulsar-beam.yml file if it is absent. 46 | 47 | ### Endpoint to stream HTTP Server Sent Event 48 | This is the endpoint to `GET` messages from Pulsar as a consumer subscription 49 | ``` 50 | /v2/sse/{persistent}/{tenant}/{namespace}/{topic} 51 | ``` 52 | Valid values of {persistent} are `p`, `persistent`, `np`, `nonpersistent` 53 | 54 | These HTTP headers may be required to map to Pulsar topic. 55 | 1. Authorization -> Bearer token as Pulsar token 56 | 2. PulsarUrl -> *optional* a fully qualified pulsar or pulsar+ssl URL where the message should be sent to. It is optional. The message will be sent to Pulsar URL specified under `PulsarBrokerURL` in the pulsar-beam.yml file if it is absent. 57 | 58 | Query parameters 59 | 1. SubscriptionType -> Supported type strings are `exclusive` as default, `shared`, and `failover` 60 | 2. SubscriptionInitialPosition -> supported type are `latest` as default and `earliest` 61 | 3. SubscriptionName -> the length must be 5 characters or longer. An auto-generated name will be provided in absence. Only the auto-generated subscription will be unsubscribed. 62 | 63 | ### Endpoint to poll batch messages 64 | Polls a batch of messages always from the earliest subscription position from a topic. 65 | ``` 66 | /v2/poll/{persistent}/{tenant}/{namespace}/{topic} 67 | ``` 68 | These HTTP headers may be required to map to Pulsar topic. 69 | 1. Authorization -> Bearer token as Pulsar token 70 | 2. PulsarUrl -> *optional* a fully qualified pulsar or pulsar+ssl URL where the message should be sent to. It is optional. The message will be sent to Pulsar URL specified under `PulsarBrokerURL` in the pulsar-beam.yml file if it is absent. 71 | 72 | Query parameters 73 | 1. SubscriptionType -> Supported type strings are `exclusive` as default, `shared`, and `failover` 74 | 2. SubscriptionName -> the length must be 5 characters or longer. An auto-generated name will be provided in absence. Only the auto-generated subscription will be unsubscribed. 75 | 3. batchSize -> Replies to a client when the batch size limit is reached. The default is 10 messages per batch. 76 | 4. perMessageTimeoutMs -> is a time out to wait for the next message's arrival from a Pulsar topic. It is in milliseconds per message. The default is 300ms. 77 | 78 | ### Webhook registration 79 | Webhook registration is done via REST API backed by a database of your choice, such as MongoDB, in momery cache, and Pulsar itself. Yes, you can use a compacted Pulsar topic as a database table to perform CRUD. The configuration parameter is `"PbDbType": "inmemory",` in the `pulsar_beam.yml` file or the env variable `PbDbType`. 80 | 81 | #### Webhook or Cloud function management API 82 | The management REAT API has this endpoint. Here is [the swagger document](https://kafkaesque-io.github.io/pulsar-beam-swagger/#/Create-or-Update-Topic) 83 | ``` 84 | /v2/topic 85 | ``` 86 | 87 | #### Bearer Token Authentication 88 | Pulsar Beam can decode and authenticate JWT generated by Pulsar. Webhook management requires a subject in JWT that matches the tenant name in the topic full name. `pulsar-admin token` can be used to generate such token. 89 | 90 | Pulsar Beam requires the same public and private keys to generate and verify JWT. These public and private key should be specified in the config to be loaded. 91 | 92 | To disable JWT authentication, set the paramater `HTTPAuthImpl` in the config file or env variable to `noauth`. 93 | 94 | ### Sink source 95 | 96 | If a webhook's response contains a body and three headers including `Authorization` for Pulsar JWT, `TopicFn` for a topic fully qualified name, and `PulsarUrl`, the beam server will send the body as a new message to the Pulsar's topic specified as in TopicFn and PulsarUrl. 97 | 98 | ### Server configuration 99 | 100 | Both [json](./config/pulsar_beam.json) and [yml format](./config/pulsar_beam.yml) are supported as configuration file. The configuration paramters are specified by [config.go](https://github.com/kafkaesque-io/pulsar-beam/blob/master/src/util/config.go#L25). Every parameter can be overridden by an environment variable with the same name. 101 | 102 | #### Server Mode 103 | In order to offer high performance and division of responsiblity, webhook and receiver endpoint can run independently `-mode broker` or `-mode receiver`. By default, the server runs in a hybrid mode with all features running in the same process. 104 | 105 | 106 | ### Docker image and Docker builds 107 | The docker image can be pulled from dockerhub.io. 108 | ``` 109 | $ sudo docker pull kafkaesqueio/pulsar-beam 110 | ``` 111 | 112 | Here are steps to build docker image and run docker container in a file based configuration. 113 | 114 | 1. Build docker image 115 | ``` 116 | $ sudo docker build -t pulsar-beam . 117 | ``` 118 | 119 | 2. Run docker 120 | This is an example of a default configurations using in-memory database. Customized `pulsar_beam.yml` and private and public key files can be mounted and passed in as an env variable `PULSAR_BEAM_CONFIG`. The certificate is required to connect to Pulsar with TLS enabled. 121 | 122 | ``` 123 | $ sudo docker run -d -it -v /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:/etc/ssl/certs/ca-bundle.crt -p 8085:8085 --name=pbeam-server pulsar-beam 124 | ``` 125 | 126 | `gops` is built in the docker image for troubleshooting purpose. 127 | 128 | ### Pulsar Kubernetes cluster deployment 129 | 130 | Pulsar Beam can be deployed within the same cluster as Pulsar. This [helm chart](https://github.com/kafkaesque-io/pulsar-helm-chart/blob/master/helm-chart-sources/pulsar/templates/beamwh-deployment.yaml) deploys a webhook broker in its own pod. The rest of HTTP receiver endpoint and REST API are deployed as a container within the [Pulsar proxy pod](https://github.com/kafkaesque-io/pulsar-helm-chart), that offers scalability with multiple replicas. 131 | 132 | 133 | ## Dev set up 134 | Clone the repo at your gopath src/github.com/kafkaesque-io/pulsar-beam folder. 135 | 136 | ### Linting 137 | Install golint. 138 | ```bash 139 | $ go install github.com/golang/lint 140 | ``` 141 | 142 | ```bash 143 | $ cd src 144 | $ golint ./... 145 | ``` 146 | 147 | There are two scripts used for CI. You might want to run them in the local environment before submitting a PR. 148 | This [CI script](./scripts/ci.sh) does linting, go vet and go build. 149 | The [code coverage script](./scripts/test_coverage.sh) runs unit test and tallies up the code coverage. 150 | 151 | ### How to run 152 | The steps how to start the web server. 153 | ```bash 154 | $ cd src 155 | $ go run main.go 156 | ``` 157 | 158 | ### Local CI, unit test and end to end test 159 | There are scripts under `./scripts` folder to run code analysis, vetting, compilation, unit test, and code coverage manually as all of these are part of CI checks by Github Actions. 160 | 161 | One end to end test is under `./src/e2e/e2etest.go`, that performs the following steps in order: 162 | 1. Create a topic and its webhook via RESTful API. The webhook URL can be an HTTP triggered Cloud Function. CI process uses a GCP 163 | 2. Send a message to Pulsar Beam's v1 injestion endpoint 164 | 3. Waiting on the sink topic where the first message will be sent to a GCP Cloud Function (in CI) and in turn reply to Pulsar Beam to forward to the second sink topic 165 | 4. Verify the replied message on the sink topic 166 | 5. Delete the topic and its webhook document via RESTful API 167 | 168 | Since the set up is non-trivial involving Pulsar Beam, a Cloud function or webhook, the test tool, and Pulsar itself with SSL, we recommend to take advantage of [the free plan at kesque.com](https://kesque.com) as the Pulsar server and a Cloud Function that we have verified GCP Fcuntion, Azure Function or AWS Lambda will suffice in the e2e flow. 169 | 170 | Step to perform unit test 171 | ```bash 172 | $ cd src/unit-test 173 | $ go test -v . 174 | ``` 175 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | # For more configuration details: 2 | # https://docs.codecov.io/docs/codecov-yaml 3 | 4 | # Check if this file is valid by running in bash: 5 | # curl -X POST --data-binary @.codecov.yml https://codecov.io/validate 6 | 7 | comment: no # do not comment PR with the result 8 | # Coverage configuration 9 | # ---------------------- 10 | coverage: 11 | range: 40..55 12 | precision: 2 13 | round: up 14 | 15 | status: 16 | project: 17 | default: 18 | target: auto 19 | threshold: 5% 20 | 21 | patch: false -------------------------------------------------------------------------------- /config/github_action_beam.json: -------------------------------------------------------------------------------- 1 | { 2 | "PORT": "8085", 3 | "CLUSTER": "localhost", 4 | "PbDbType": "mongo", 5 | "PulsarPublicKey": "/home/runner/work/pulsar-beam/go/src/github.com/kafkaesque-io/pulsar-beam/src/unit-test/example_public_key.pub", 6 | "PulsarPrivateKey": "/home/runner/work/pulsar-beam/go/src/github.com/kafkaesque-io/pulsar-beam/src/unit-test/example_private_key", 7 | "PbDbInterval": "60s", 8 | "DbConnectionStr": "mongodb://172.17.0.1:27017" 9 | 10 | } -------------------------------------------------------------------------------- /config/prototype-db/default.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Webhooks": [ 4 | { 5 | "URL": "http://localhost:8089/webhook/", 6 | "Headers": [ 7 | "Content-type: application/json" 8 | ] 9 | } 10 | ], 11 | "TopicConfig": { 12 | "TopicFN": "persistent://ming-luo/local-useast1-gcp/test-topic", 13 | "Token": "please provide a valid pulsar token", 14 | "PulsarURL": "pulsar+ssl://useast1.gcp.kafkaesque.io:6651" 15 | } 16 | } 17 | ] 18 | -------------------------------------------------------------------------------- /config/pulsar_beam.json: -------------------------------------------------------------------------------- 1 | { 2 | "PORT": "8085", 3 | "CLUSTER": "localhost", 4 | "PbDbType": "mongo", 5 | "PulsarPublicKey": "./unit-test/example_public_key.pub", 6 | "PulsarPrivateKey": "./unit-test/example_private_key", 7 | "PbDbInterval": "60s", 8 | "DbConnectionStr": "mongodb://localhost:27017" 9 | } -------------------------------------------------------------------------------- /config/pulsar_beam.yml: -------------------------------------------------------------------------------- 1 | --- 2 | PORT: 8085 3 | CLUSTER: localhost 4 | PbDbType: inmemory 5 | PulsarPublicKey: ./unit-test/example_public_key.pub 6 | PulsarPrivateKey: ./unit-test/example_private_key 7 | PbDbInterval: 10s 8 | DbConnectionStr: mongodb://localhost:27017 9 | DbName: 10 | DbPassword: 11 | TrustStore: "/etc/ssl/certs/ca-bundle.crt" -------------------------------------------------------------------------------- /config/pulsar_beam_inmemory_db.yml: -------------------------------------------------------------------------------- 1 | --- 2 | PORT: 8085 3 | CLUSTER: localhost 4 | PbDbType: inmemory 5 | PulsarPublicKey: ../config/example_public_key.pub 6 | PulsarPrivateKey: ../config/example_private_key 7 | PbDbInterval: 10s 8 | DbConnectionStr: mongodb://localhost:27017 9 | DbName: 10 | DbPassword: -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | beam: 4 | build: . 5 | ports: 6 | - "8080:8080" 7 | volumes: 8 | - /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:/etc/ssl/certs/ca-bundle.crt 9 | - /home/ming/go/src/github.com/pulsar-beam/config:/root/config 10 | rqlite: 11 | image: rqlite/rqlite -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kafkaesque-io/pulsar-beam 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/apache/pulsar-client-go v0.8.1 7 | github.com/ghodss/yaml v1.0.0 8 | github.com/golang-jwt/jwt v3.2.2+incompatible 9 | github.com/google/gops v0.3.24 10 | github.com/gorilla/mux v1.8.0 11 | github.com/hashicorp/go-retryablehttp v0.7.1 12 | github.com/prometheus/client_golang v1.12.2 13 | github.com/rs/cors v1.8.2 14 | github.com/sirupsen/logrus v1.8.1 15 | go.mongodb.org/mongo-driver v1.9.1 16 | ) 17 | 18 | require ( 19 | github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect 20 | github.com/99designs/keyring v1.2.1 // indirect 21 | github.com/AthenZ/athenz v1.11.3 // indirect 22 | github.com/DataDog/zstd v1.5.2 // indirect 23 | github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220623212449-0f7041ffa908 // indirect 24 | github.com/ardielle/ardielle-go v1.5.2 // indirect 25 | github.com/beorn7/perks v1.0.1 // indirect 26 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 27 | github.com/danieljoos/wincred v1.1.2 // indirect 28 | github.com/davecgh/go-spew v1.1.1 // indirect 29 | github.com/dvsekhvalnov/jose2go v1.5.0 // indirect 30 | github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect 31 | github.com/go-stack/stack v1.8.1 // indirect 32 | github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect 33 | github.com/gogo/protobuf v1.3.2 // indirect 34 | github.com/golang/protobuf v1.5.2 // indirect 35 | github.com/golang/snappy v0.0.4 // indirect 36 | github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect 37 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 38 | github.com/keybase/go-keychain v0.0.0-20220610143837-c2ce06069005 // indirect 39 | github.com/klauspost/compress v1.15.6 // indirect 40 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect 41 | github.com/linkedin/goavro/v2 v2.11.1 // indirect 42 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 43 | github.com/mitchellh/go-homedir v1.1.0 // indirect 44 | github.com/mtibben/percent v0.2.1 // indirect 45 | github.com/pierrec/lz4 v2.6.1+incompatible // indirect 46 | github.com/pkg/errors v0.9.1 // indirect 47 | github.com/pmezard/go-difflib v1.0.0 // indirect 48 | github.com/prometheus/client_model v0.2.0 // indirect 49 | github.com/prometheus/common v0.35.0 // indirect 50 | github.com/prometheus/procfs v0.7.3 // indirect 51 | github.com/spaolacci/murmur3 v1.1.0 // indirect 52 | github.com/stretchr/testify v1.7.5 // indirect 53 | github.com/tidwall/pretty v1.0.1 // indirect 54 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 55 | github.com/xdg-go/scram v1.1.1 // indirect 56 | github.com/xdg-go/stringprep v1.0.3 // indirect 57 | github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect 58 | go.uber.org/atomic v1.9.0 // indirect 59 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect 60 | golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect 61 | golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 // indirect 62 | golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect 63 | golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect 64 | golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect 65 | golang.org/x/text v0.3.7 // indirect 66 | google.golang.org/appengine v1.6.7 // indirect 67 | google.golang.org/protobuf v1.28.0 // indirect 68 | gopkg.in/yaml.v2 v2.4.0 // indirect 69 | gopkg.in/yaml.v3 v3.0.1 // indirect 70 | ) 71 | -------------------------------------------------------------------------------- /scripts/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Run the CI flow and build the binary 5 | # Prerequisite - 6 | # 1. Go runtime 7 | # 8 | 9 | # absolute directory 10 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 11 | 12 | BASE_PKG_DIR="github.com/kafkaesque-io/pulsar-beam/src/" 13 | ALL_PKGS="" 14 | 15 | cd $DIR/../src 16 | # test lint, vet, and build as basic build steps in CI 17 | echo run golint 18 | golint ./... 19 | echo run go vet 20 | go vet ./... 21 | 22 | echo run go build 23 | mkdir -p ${DIR}/../bin 24 | rm -f ${DIR}/../bin/pulsar-beam 25 | go build -o ${DIR}/../bin/pulsar-beam . 26 | -------------------------------------------------------------------------------- /scripts/secret-post-process.py: -------------------------------------------------------------------------------- 1 | #!/user/bin/env python3 2 | 3 | # 4 | # this file takes output of Yelp's secret-detect from standard input and process it 5 | # 6 | # detect-secrets scan | python3 ./scripts/secret-post-process.py ; echo $? 7 | # 8 | 9 | import json 10 | import sys 11 | import os 12 | 13 | whiteList = ["go.sum", "src/unit-test/example_private_key"] 14 | 15 | stdin='' 16 | 17 | for line in sys.stdin: 18 | if line == "\n": 19 | lb += 1 20 | if lb == 2: 21 | break 22 | else: 23 | lb = 0 24 | stdin += line 25 | 26 | results = json.loads(stdin) 27 | 28 | foundGoSum = False 29 | error = False 30 | for (k, v) in results["results"].items(): 31 | if k == "go.sum": 32 | foundGoSum = True 33 | if k not in whiteList: 34 | print(k, v) 35 | error = True 36 | 37 | if not foundGoSum: 38 | print("Error: fail to process expected go.sum") 39 | os._exit(2) 40 | 41 | if error: 42 | print("Error: above secret detected") 43 | os._exit(3) 44 | else: 45 | print("successful") 46 | -------------------------------------------------------------------------------- /scripts/swagger.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This script has instructions to build swagger document. 4 | # The swagger document is currently hosted on Github pages by another repo, 5 | # https://github.com/kafkaesque-io/pulsar-beam-swagger 6 | # 7 | 8 | # absolute directory 9 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 10 | 11 | # 12 | # Set up vendor package if it is not in use 13 | # 14 | #GO111MODULE=on go mod vendor 15 | 16 | # Install go-swagger 17 | # 18 | # This is required to run the swagger executable in the next step. 19 | # 20 | #GO111MODULE=off go get -u github.com/go-swagger/go-swagger/cmd/swagger 21 | 22 | # 23 | # Generate swagger 24 | # 25 | cd $DIR/.. 26 | GO111MODULE=off swagger generate spec -o ./swagger.yaml --scan-models 27 | 28 | # 29 | # test locally 30 | # swagger must be installed 31 | # it will start a webserver with swagger documents 32 | # 33 | swagger serve -F=swagger swagger.yaml 34 | 35 | # 36 | # swagger.yaml is required to submit to the repo https://github.com/kafkaesque-io/pulsar-beam-swagger 37 | # the swagger document is published at https://kafkaesque-io.github.io/pulsar-beam-swagger/ 38 | # 39 | 40 | -------------------------------------------------------------------------------- /scripts/test_coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Run unit test and generate test coverage report 5 | # 6 | 7 | # absolute directory 8 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 9 | 10 | BASE_PKG_DIR="github.com/kafkaesque-io/pulsar-beam/src/" 11 | ALL_PKGS="" 12 | 13 | cd $DIR/../src 14 | for d in */ ; do 15 | if [[ ${d} != "unit-test/" && ${d} != "e2e/" && ${d} != "docs/" ]] # exclude unit-test for test coverage 16 | then 17 | pkg=${d%/} 18 | ALL_PKGS=${ALL_PKGS}","${BASE_PKG_DIR}${pkg} 19 | fi 20 | done 21 | 22 | ALL_PKGS=$(echo $ALL_PKGS | sed 's/^,//') 23 | echo $ALL_PKGS 24 | 25 | cd $DIR/../src/unit-test 26 | 27 | go test ./... -coverpkg=$ALL_PKGS -covermode=count -coverprofile coverage.out 28 | # to be uploaded to covercov 29 | cp coverage.out $DIR/../coverage.txt 30 | go tool cover -func coverage.out > /tmp/coverage2.txt 31 | 32 | coverPercent=$(cat /tmp/coverage2.txt | grep total: | awk '{print $3}' | sed 's/%$//g') 33 | 34 | echo "Current test coverage is at ${coverPercent}%" 35 | echo "TODO add code coverage verdict" 36 | -------------------------------------------------------------------------------- /src/broker/sse-broker.go: -------------------------------------------------------------------------------- 1 | package broker 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/apache/pulsar-client-go/pulsar" 8 | "github.com/kafkaesque-io/pulsar-beam/src/model" 9 | "github.com/kafkaesque-io/pulsar-beam/src/pulsardriver" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // GetPulsarClientConsumer returns Puslar client and consumer interface objects 14 | func GetPulsarClientConsumer(url, token, topic, subscriptionName string, subType pulsar.SubscriptionType, subInitPos pulsar.SubscriptionInitialPosition) (pulsar.Client, pulsar.Consumer, error) { 15 | client, err := pulsardriver.NewPulsarClient(url, token) 16 | if err != nil { 17 | return nil, nil, err 18 | } 19 | 20 | consumer, err := client.Subscribe(pulsar.ConsumerOptions{ 21 | Topic: topic, 22 | SubscriptionName: subscriptionName, 23 | SubscriptionInitialPosition: subInitPos, 24 | Type: subType, 25 | }) 26 | if err != nil { 27 | return nil, nil, err 28 | } 29 | 30 | return client, consumer, nil 31 | } 32 | 33 | // PollBatchMessages polls a batch of consumer messages 34 | func PollBatchMessages(url, token, topic, subscriptionName string, subType pulsar.SubscriptionType, size, perMessageTimeoutMs int) (model.PulsarMessages, error) { 35 | log.Infof("getbatchmessages called") 36 | client, consumer, err := GetPulsarClientConsumer(url, token, topic, subscriptionName, subType, pulsar.SubscriptionPositionEarliest) 37 | if err != nil { 38 | return model.NewPulsarMessages(size), err 39 | } 40 | if strings.HasPrefix(subscriptionName, model.NonResumable) { 41 | defer consumer.Unsubscribe() 42 | } 43 | defer consumer.Close() 44 | defer client.Close() 45 | 46 | messages := model.NewPulsarMessages(size) 47 | consumChan := consumer.Chan() 48 | for i := 0; i < size; i++ { 49 | select { 50 | case msg := <-consumChan: 51 | // log.Infof("received message %s on topic %s", string(msg.Payload()), msg.Topic()) 52 | messages.AddPulsarMessage(msg) 53 | consumer.Ack(msg) 54 | 55 | case <-time.After(time.Duration(perMessageTimeoutMs) * time.Millisecond): //TODO: this should be configurable 56 | i = size 57 | } 58 | } 59 | 60 | return messages, nil 61 | } 62 | -------------------------------------------------------------------------------- /src/broker/webhook.go: -------------------------------------------------------------------------------- 1 | package broker 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | "github.com/apache/pulsar-client-go/pulsar" 15 | "github.com/hashicorp/go-retryablehttp" 16 | "github.com/kafkaesque-io/pulsar-beam/src/db" 17 | "github.com/kafkaesque-io/pulsar-beam/src/model" 18 | "github.com/kafkaesque-io/pulsar-beam/src/pulsardriver" 19 | "github.com/kafkaesque-io/pulsar-beam/src/util" 20 | 21 | log "github.com/sirupsen/logrus" 22 | ) 23 | 24 | type WebhookBroker struct { 25 | // key is the hash of topic full name and pulsar url, and subscription name 26 | webhooks map[string]chan *SubCloseSignal 27 | dbHandler db.Db 28 | l *log.Entry 29 | sync.RWMutex 30 | } 31 | 32 | // SubCloseSignal is a signal object to pass for channel 33 | type SubCloseSignal struct{} 34 | 35 | func NewWebhookBroker(config *util.Configuration) *WebhookBroker { 36 | return &WebhookBroker{ 37 | dbHandler: db.NewDbWithPanic(config.PbDbType), 38 | webhooks: make(map[string]chan *SubCloseSignal), 39 | l: log.WithFields(log.Fields{"app": "webhookbroker"}), 40 | } 41 | } 42 | 43 | // ReadWebhook reads a thread safe map 44 | func (wb *WebhookBroker) ReadWebhook(key string) (chan *SubCloseSignal, bool) { 45 | wb.RLock() 46 | defer wb.RUnlock() 47 | c, ok := wb.webhooks[key] 48 | return c, ok 49 | } 50 | 51 | // WriteWebhook writes a key/value to a thread safe map 52 | func (wb *WebhookBroker) WriteWebhook(key string, c chan *SubCloseSignal) { 53 | wb.Lock() 54 | defer wb.Unlock() 55 | wb.webhooks[key] = c 56 | } 57 | 58 | // DeleteWebhook deletes a key from a thread safe map 59 | func (wb *WebhookBroker) DeleteWebhook(key string) bool { 60 | wb.Lock() 61 | defer wb.Unlock() 62 | if c, ok := wb.webhooks[key]; ok { 63 | c <- &SubCloseSignal{} 64 | delete(wb.webhooks, key) 65 | //channel is deleted where it's been created with `defer` 66 | return ok 67 | } 68 | return false 69 | } 70 | 71 | // Init initializes webhook configuration database 72 | func Init(config *util.Configuration) { 73 | svr := NewWebhookBroker(config) 74 | durationStr := util.AssignString(config.PbDbInterval, "180s") 75 | duration, err := time.ParseDuration(durationStr) 76 | if err != nil { 77 | svr.l.Errorf("specified duration %s error %v", durationStr, err) 78 | duration, _ = time.ParseDuration("180s") 79 | } 80 | svr.l.Infof("beam database pull every %.0f seconds", duration.Seconds()) 81 | 82 | go func() { 83 | svr.run() 84 | ticker := time.NewTicker(duration) 85 | defer ticker.Stop() 86 | for { 87 | select { 88 | case <-ticker.C: 89 | svr.run() 90 | } 91 | } 92 | }() 93 | } 94 | 95 | // pushWebhook sends data to a webhook interface 96 | func pushWebhook(url string, data []byte, headers []string) (int, *http.Response) { 97 | 98 | client := retryablehttp.NewClient() 99 | client.RetryWaitMin = 2 * time.Second 100 | client.RetryWaitMax = 28 * time.Second 101 | client.RetryMax = 1 102 | 103 | req, err := retryablehttp.NewRequest("POST", url, data) 104 | if err != nil { 105 | log.Errorf("url request error %s", err.Error()) 106 | return http.StatusInternalServerError, nil 107 | } 108 | 109 | for _, h := range headers { 110 | // since : is allowed in header's value 111 | l := strings.SplitAfterN(h, ":", 2) 112 | if len(l) == 2 { 113 | headerKey := strings.TrimSpace(strings.Replace(l[0], ":", "", -1)) 114 | req.Header.Set(headerKey, strings.TrimSpace(l[1])) 115 | } 116 | //discard any misformed headers 117 | } 118 | 119 | res, err := client.Do(req) 120 | if err != nil { 121 | log.Debugf("webhook post error %s", err.Error()) 122 | return http.StatusInternalServerError, nil 123 | } 124 | 125 | if log.GetLevel() == log.DebugLevel { 126 | log.Debugf("webhook endpoint resp status code %d", res.StatusCode) 127 | } 128 | return res.StatusCode, res 129 | } 130 | 131 | func toPulsar(r *http.Response) { 132 | token, topicFN, pulsarURL, err := util.ReceiverHeader(util.AllowedPulsarURLs, &r.Header) 133 | if err != nil { 134 | return 135 | } 136 | if log.GetLevel() == log.DebugLevel { 137 | log.Debugf("topicURL %s pulsarURL %s", topicFN, pulsarURL) 138 | } 139 | 140 | b, err2 := ioutil.ReadAll(r.Body) 141 | defer r.Body.Close() 142 | if err2 != nil { 143 | log.Errorf("failed to read webhook resp body %s\n", err2.Error()) 144 | return 145 | } 146 | 147 | err3 := pulsardriver.SendToPulsar(pulsarURL, token, topicFN, b, true) 148 | if err3 != nil { 149 | return 150 | } 151 | } 152 | 153 | func pushAndAck(c pulsar.Consumer, msg pulsar.Message, url string, data []byte, headers []string) { 154 | code, res := pushWebhook(url, data, headers) 155 | if (code >= 200 && code < 300) || code == http.StatusUnprocessableEntity { 156 | c.Ack(msg) 157 | 158 | if code >= 200 && code < 300 { 159 | go toPulsar(res) 160 | } 161 | } else { 162 | if log.GetLevel() == log.DebugLevel { 163 | // replying on Pulsar to redeliver 164 | log.Errorf("webhook returns non-OK statuscode %d\n", code) 165 | } 166 | } 167 | } 168 | 169 | // ConsumeLoop consumes data from Pulsar topic 170 | // Do not use context since go vet will puke that requires cancel invoked in the same function 171 | func (wb *WebhookBroker) ConsumeLoop(url, token, topic, subscriptionKey string, whCfg model.WebhookConfig) error { 172 | headers := whCfg.Headers 173 | _, err := model.GetSubscriptionType(whCfg.SubscriptionType) 174 | if err != nil { 175 | return err 176 | } 177 | _, err = model.GetInitialPosition(whCfg.InitialPosition) 178 | if err != nil { 179 | return err 180 | } 181 | c, err := pulsardriver.GetPulsarConsumer(url, token, topic, whCfg.Subscription, whCfg.InitialPosition, whCfg.SubscriptionType, subscriptionKey) 182 | if err != nil { 183 | return fmt.Errorf("failed to create Pulsar subscription %v", err) 184 | } 185 | 186 | terminate := make(chan *SubCloseSignal, 2) 187 | wb.WriteWebhook(subscriptionKey, terminate) 188 | defer close(terminate) 189 | ctx := context.Background() 190 | 191 | // infinite loop to receive messages 192 | // TODO receive can starve stop channel if it waits for the next message indefinitely 193 | retry := 0 194 | retryMax := 3 195 | for { 196 | if retry > retryMax { 197 | wb.cancelConsumer(subscriptionKey) 198 | return fmt.Errorf("consumer retried %d times, max reached", retryMax) 199 | } 200 | msg, err := c.Receive(ctx) 201 | if err != nil { 202 | wb.l.Infof("error from consumer loop receive: %v\n", err) 203 | retry++ 204 | ticker := time.NewTicker(time.Duration(2*retry) * time.Second) 205 | defer ticker.Stop() 206 | select { 207 | case <-terminate: 208 | wb.l.Infof("subscription %s received signal to exit consumer loop", subscriptionKey) 209 | return nil 210 | case <-ticker.C: 211 | //reconnect after error 212 | c, err = pulsardriver.GetPulsarConsumer(url, token, topic, whCfg.Subscription, whCfg.InitialPosition, whCfg.SubscriptionType, subscriptionKey) 213 | if err != nil { 214 | return fmt.Errorf("Retry failed to create Pulsar subscription %v", err) 215 | } 216 | } 217 | } else if msg != nil { 218 | retry = 0 219 | if wb.l.Level == log.DebugLevel { 220 | wb.l.Debugf("PulsarMessageId:%v", msg.ID()) 221 | } 222 | headers = append(headers, fmt.Sprintf("PulsarMessageId:%#v", msg.ID())) 223 | headers = append(headers, "PulsarPublishedTime:"+msg.PublishTime().String()) 224 | headers = append(headers, "PulsarTopic:"+msg.Topic()) 225 | nilTime := time.Time{} 226 | if msg.EventTime() != nilTime { 227 | headers = append(headers, "PulsarEventTime:"+msg.EventTime().String()) 228 | } 229 | for k, v := range msg.Properties() { 230 | headers = append(headers, "PulsarProperties-"+k+":"+v) 231 | } 232 | 233 | data := msg.Payload() 234 | if json.Valid(data) { 235 | headers = append(headers, "content-type:application/json") 236 | } 237 | pushAndAck(c, msg, whCfg.URL, data, headers) 238 | } 239 | } 240 | 241 | } 242 | 243 | func (wb *WebhookBroker) run() { 244 | // key is hash of topic name and pulsar url, and subscription name 245 | subscriptionSet := make(map[string]bool) 246 | 247 | for _, cfg := range wb.LoadConfig() { 248 | for _, whCfg := range cfg.Webhooks { 249 | topic := cfg.TopicFullName 250 | token := cfg.Token 251 | url := cfg.PulsarURL 252 | subscriptionKey := cfg.Key + whCfg.URL 253 | status := whCfg.WebhookStatus 254 | _, ok := wb.ReadWebhook(subscriptionKey) 255 | if status == model.Activated { 256 | subscriptionSet[subscriptionKey] = true 257 | if !ok { 258 | wb.l.Infof("start activated webhook for topic subscription %v", subscriptionKey) 259 | go wb.ConsumeLoop(url, token, topic, subscriptionKey, whCfg) 260 | } 261 | } 262 | } 263 | } 264 | 265 | // cancel any webhook which is no longer required to be activated by the database 266 | for k := range wb.webhooks { 267 | if !subscriptionSet[k] { 268 | wb.l.Infof("cancel webhook consumer subscription key %s", k) 269 | wb.cancelConsumer(k) 270 | } 271 | } 272 | wb.l.Infof("load webhooks size %d", len(wb.webhooks)) 273 | } 274 | 275 | // LoadConfig loads the entire topic documents from the database 276 | func (wb *WebhookBroker) LoadConfig() []*model.TopicConfig { 277 | cfgs, err := wb.dbHandler.Load() 278 | if err != nil { 279 | wb.l.Errorf("failed to load topics from database error %v", err.Error()) 280 | } 281 | 282 | return cfgs 283 | } 284 | 285 | func (wb *WebhookBroker) cancelConsumer(key string) error { 286 | ok := wb.DeleteWebhook(key) 287 | if ok { 288 | wb.l.Infof("cancel consumer %v", key) 289 | pulsardriver.CancelPulsarConsumer(key) 290 | return nil 291 | } 292 | return errors.New("topic does not exist " + key) 293 | } 294 | -------------------------------------------------------------------------------- /src/db/in-memory.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "github.com/kafkaesque-io/pulsar-beam/src/model" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | /** 13 | * An in memory database implmentation of the restful API data store 14 | * no data is persisted. 15 | * This is for testing only. 16 | */ 17 | 18 | // InMemoryHandler is the in memory cache driver 19 | type InMemoryHandler struct { 20 | topics map[string]model.TopicConfig 21 | logger *log.Entry 22 | } 23 | 24 | //Init is a Db interface method. 25 | func (s *InMemoryHandler) Init() error { 26 | s.logger = log.WithFields(log.Fields{"app": "inmemory-db"}) 27 | s.topics = make(map[string]model.TopicConfig) 28 | return nil 29 | } 30 | 31 | //Sync is a Db interface method. 32 | func (s *InMemoryHandler) Sync() error { 33 | return nil 34 | } 35 | 36 | //Health is a Db interface method 37 | func (s *InMemoryHandler) Health() bool { 38 | return true 39 | } 40 | 41 | // Close closes database 42 | func (s *InMemoryHandler) Close() error { 43 | return nil 44 | } 45 | 46 | //NewInMemoryHandler initialize a Mongo Db 47 | func NewInMemoryHandler() (*InMemoryHandler, error) { 48 | handler := InMemoryHandler{} 49 | err := handler.Init() 50 | return &handler, err 51 | } 52 | 53 | // Create creates a new document 54 | func (s *InMemoryHandler) Create(topicCfg *model.TopicConfig) (string, error) { 55 | key, err := getKey(topicCfg) 56 | if err != nil { 57 | return key, err 58 | } 59 | 60 | if _, ok := s.topics[key]; ok { 61 | return key, errors.New(DocAlreadyExisted) 62 | } 63 | 64 | topicCfg.Key = key 65 | topicCfg.CreatedAt = time.Now() 66 | topicCfg.UpdatedAt = topicCfg.CreatedAt 67 | 68 | s.topics[topicCfg.Key] = *topicCfg 69 | return key, nil 70 | } 71 | 72 | // GetByTopic gets a document by the topic name and pulsar URL 73 | func (s *InMemoryHandler) GetByTopic(topicFullName, pulsarURL string) (*model.TopicConfig, error) { 74 | key, err := model.GetKeyFromNames(topicFullName, pulsarURL) 75 | if err != nil { 76 | return &model.TopicConfig{}, err 77 | } 78 | return s.GetByKey(key) 79 | } 80 | 81 | // GetByKey gets a document by the key 82 | func (s *InMemoryHandler) GetByKey(hashedTopicKey string) (*model.TopicConfig, error) { 83 | if v, ok := s.topics[hashedTopicKey]; ok { 84 | return &v, nil 85 | } 86 | return &model.TopicConfig{}, errors.New(DocNotFound) 87 | } 88 | 89 | // Load loads the entire database as a list 90 | func (s *InMemoryHandler) Load() ([]*model.TopicConfig, error) { 91 | results := []*model.TopicConfig{} 92 | for _, v := range s.topics { 93 | results = append(results, &v) 94 | } 95 | return results, nil 96 | } 97 | 98 | // Update updates or creates a topic config document 99 | func (s *InMemoryHandler) Update(topicCfg *model.TopicConfig) (string, error) { 100 | key, err := getKey(topicCfg) 101 | if err != nil { 102 | return key, err 103 | } 104 | 105 | if _, ok := s.topics[key]; !ok { 106 | return s.Create(topicCfg) 107 | } 108 | 109 | v := s.topics[key] 110 | v.Token = topicCfg.Token 111 | v.Tenant = topicCfg.Tenant 112 | v.Notes = topicCfg.Notes 113 | v.TopicStatus = topicCfg.TopicStatus 114 | v.UpdatedAt = time.Now() 115 | v.Webhooks = topicCfg.Webhooks 116 | 117 | s.logger.Infof("upsert %s", key) 118 | s.topics[topicCfg.Key] = *topicCfg 119 | return key, nil 120 | 121 | } 122 | 123 | // Delete deletes a document 124 | func (s *InMemoryHandler) Delete(topicFullName, pulsarURL string) (string, error) { 125 | key, err := model.GetKeyFromNames(topicFullName, pulsarURL) 126 | if err != nil { 127 | return "", err 128 | } 129 | return s.DeleteByKey(key) 130 | } 131 | 132 | // DeleteByKey deletes a document based on key 133 | func (s *InMemoryHandler) DeleteByKey(hashedTopicKey string) (string, error) { 134 | if _, ok := s.topics[hashedTopicKey]; !ok { 135 | return "", errors.New(DocNotFound) 136 | } 137 | 138 | delete(s.topics, hashedTopicKey) 139 | return hashedTopicKey, nil 140 | } 141 | -------------------------------------------------------------------------------- /src/db/interface.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/kafkaesque-io/pulsar-beam/src/model" 7 | 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | // dbConn is a singlton of Db instance 12 | var dbConn Db 13 | 14 | // Crud interface specifies typical CRUD opertaions for database 15 | type Crud interface { 16 | GetByTopic(topicFullName, pulsarURL string) (*model.TopicConfig, error) 17 | GetByKey(hashedTopicKey string) (*model.TopicConfig, error) 18 | Update(topicCfg *model.TopicConfig) (string, error) 19 | Create(topicCfg *model.TopicConfig) (string, error) 20 | Delete(topicFullName, pulsarURL string) (string, error) 21 | DeleteByKey(hashedTopicKey string) (string, error) 22 | 23 | // Load is invoked by the webhook.go to start new wekbooks and stop deleted ones 24 | Load() ([]*model.TopicConfig, error) 25 | } 26 | 27 | // Ops interface specifies required database access operations 28 | type Ops interface { 29 | Init() error 30 | Sync() error 31 | Close() error 32 | Health() bool 33 | } 34 | 35 | // Db interface embeds two other database interfaces 36 | type Db interface { 37 | Crud 38 | Ops 39 | } 40 | 41 | // NewDb is a database factory pattern to create a new database 42 | func NewDb(reqDbType string) (Db, error) { 43 | if dbConn != nil { 44 | return dbConn, nil 45 | } 46 | 47 | var err error 48 | switch reqDbType { 49 | case "mongo": 50 | dbConn, err = NewMongoDb() 51 | case "pulsarAsDb": 52 | dbConn, err = NewPulsarHandler() 53 | case "inmemory": 54 | dbConn, err = NewInMemoryHandler() 55 | default: 56 | err = errors.New("unsupported db type") 57 | } 58 | return dbConn, err 59 | } 60 | 61 | // NewDbWithPanic ensures a database is returned panic otherwise 62 | func NewDbWithPanic(reqDbType string) Db { 63 | newDb, err := NewDb(reqDbType) 64 | if err != nil { 65 | log.Fatalf("init db with error %s", err.Error()) 66 | } 67 | return newDb 68 | } 69 | 70 | // DocNotFound means no document found in the database 71 | var DocNotFound = "no document found" 72 | 73 | // DocAlreadyExisted means document already existed in the database when a new creation is requested 74 | var DocAlreadyExisted = "document already existed" 75 | -------------------------------------------------------------------------------- /src/db/mongo.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "time" 7 | 8 | "github.com/kafkaesque-io/pulsar-beam/src/model" 9 | "github.com/kafkaesque-io/pulsar-beam/src/util" 10 | 11 | log "github.com/sirupsen/logrus" 12 | 13 | "go.mongodb.org/mongo-driver/bson" 14 | "go.mongodb.org/mongo-driver/mongo" 15 | "go.mongodb.org/mongo-driver/mongo/options" 16 | "go.mongodb.org/mongo-driver/mongo/readpref" 17 | ) 18 | 19 | // MongoDb is the mongo database driver 20 | type MongoDb struct { 21 | client *mongo.Client 22 | collection *mongo.Collection 23 | logger *log.Entry 24 | } 25 | 26 | var connectionString string = "mongodb://localhost:27017" 27 | var dbName string = "localhost" 28 | var collectionName string = "topics" 29 | 30 | //Init is a Db interface method. 31 | func (s *MongoDb) Init() error { 32 | s.logger = log.WithFields(log.Fields{"app": "mongodb"}) 33 | dbName = util.AssignString(util.Config.CLUSTER, dbName) 34 | connectionString = util.AssignString(util.Config.DbConnectionStr, connectionString) 35 | s.logger.Warnf("connecting to %s", connectionString) 36 | var err error 37 | clientOptions := options.Client().ApplyURI(connectionString) 38 | // ctx, _ := context.WithTimeout(context.Background(), 4*time.Second) 39 | s.client, err = mongo.Connect(context.TODO(), clientOptions) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | err = s.client.Ping(context.TODO(), readpref.Primary()) 45 | if err != nil { 46 | s.logger.Errorf("mongodb ping failed %s", err.Error()) 47 | return err 48 | } 49 | 50 | s.logger.Infof("connected to mongodb %s", dbName) 51 | 52 | s.collection = s.client.Database(dbName).Collection(collectionName) 53 | 54 | indexView := s.collection.Indexes() 55 | indexMode := mongo.IndexModel{ 56 | Keys: bson.M{"Key": 1}, 57 | Options: options.Index().SetUnique(true), 58 | } 59 | _, err = indexView.CreateOne(context.Background(), indexMode) 60 | if err != nil { 61 | s.logger.Errorf("database index creation failed %s", err.Error()) 62 | return err 63 | } 64 | 65 | s.logger.Infof("mongo database name %v, collection %v", dbName, collectionName) 66 | return nil 67 | } 68 | 69 | //Sync is a Db interface method. 70 | func (s *MongoDb) Sync() error { 71 | s.logger.Infof("sync") 72 | return nil 73 | } 74 | 75 | //Health is a Db interface method 76 | func (s *MongoDb) Health() bool { 77 | ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) 78 | defer cancel() 79 | err := s.client.Ping(ctx, readpref.Primary()) 80 | if err != nil { 81 | return false 82 | } 83 | return true 84 | } 85 | 86 | // Close closes database 87 | func (s *MongoDb) Close() error { 88 | return s.client.Disconnect(context.TODO()) 89 | } 90 | 91 | //NewMongoDb initialize a Mongo Db 92 | func NewMongoDb() (*MongoDb, error) { 93 | mongoDb := MongoDb{} 94 | err := mongoDb.Init() 95 | return &mongoDb, err 96 | } 97 | 98 | // Create creates a new document 99 | func (s *MongoDb) Create(topicCfg *model.TopicConfig) (string, error) { 100 | key, err := getKey(topicCfg) 101 | if err != nil { 102 | return key, err 103 | } 104 | 105 | topicCfg.Key = key 106 | topicCfg.CreatedAt = time.Now() 107 | topicCfg.UpdatedAt = topicCfg.CreatedAt 108 | insertResult, err := s.collection.InsertOne(context.Background(), topicCfg) 109 | 110 | if err != nil { 111 | return "", err 112 | } 113 | 114 | if log.GetLevel() == log.DebugLevel { 115 | s.logger.Debugf("Inserted a Single Record %v, %s", insertResult.InsertedID, topicCfg.Key) 116 | } 117 | return topicCfg.Key, nil 118 | } 119 | 120 | // GetByTopic gets a document by the topic name and pulsar URL 121 | func (s *MongoDb) GetByTopic(topicFullName, pulsarURL string) (*model.TopicConfig, error) { 122 | key, err := model.GetKeyFromNames(topicFullName, pulsarURL) 123 | if err != nil { 124 | return &model.TopicConfig{}, err 125 | } 126 | return s.GetByKey(key) 127 | } 128 | 129 | // GetByKey gets a document by the key 130 | func (s *MongoDb) GetByKey(hashedTopicKey string) (*model.TopicConfig, error) { 131 | var doc model.TopicConfig 132 | result := s.collection.FindOne(context.TODO(), bson.M{"key": hashedTopicKey}) 133 | 134 | err := result.Decode(&doc) 135 | if err != nil { 136 | if err.Error() == "mongo: no documents in result" { 137 | err = errors.New(DocNotFound) 138 | } 139 | return &model.TopicConfig{}, err 140 | } 141 | return &doc, nil 142 | } 143 | 144 | // Load loads the entire database into memory 145 | func (s *MongoDb) Load() ([]*model.TopicConfig, error) { 146 | var results []*model.TopicConfig 147 | 148 | findOptions := options.Find() 149 | cursor, err := s.collection.Find(context.TODO(), bson.D{{}}, findOptions) 150 | if err != nil { 151 | return results, err 152 | } 153 | 154 | defer cursor.Close(context.TODO()) 155 | 156 | for cursor.Next(context.TODO()) { 157 | var ele model.TopicConfig 158 | err := cursor.Decode(&ele) 159 | if err != nil { 160 | s.logger.Errorf("failed to decode document %s", err.Error()) 161 | } else { 162 | results = append(results, &ele) 163 | if log.GetLevel() == log.DebugLevel { 164 | s.logger.Debugf("from mongo %s %s %s", ele.TopicFullName, ele.PulsarURL, ele.Webhooks[0].URL) 165 | } 166 | } 167 | } 168 | 169 | return results, nil 170 | } 171 | 172 | // Update updates or creates a topic config document 173 | func (s *MongoDb) Update(topicCfg *model.TopicConfig) (string, error) { 174 | key, err := getKey(topicCfg) 175 | if err != nil { 176 | return key, err 177 | } 178 | 179 | exists, err := exists(key, s.collection) 180 | if err != nil { 181 | return "", err 182 | } 183 | 184 | if !exists { 185 | if log.GetLevel() == log.DebugLevel { 186 | s.logger.Debugf("not exists so to create one") 187 | } 188 | return s.Create(topicCfg) 189 | } 190 | 191 | filter := bson.M{ 192 | "key": bson.M{ 193 | "$eq": key, // key has to match 194 | }, 195 | } 196 | update := bson.M{ 197 | "$set": bson.M{ 198 | "token": topicCfg.Token, 199 | "tenant": topicCfg.Tenant, 200 | "notes": topicCfg.Notes, 201 | "topicstatus": topicCfg.TopicStatus, 202 | "updatedat": time.Now(), 203 | "webhooks": topicCfg.Webhooks, 204 | }, 205 | } 206 | result, err := s.collection.UpdateOne( 207 | context.TODO(), 208 | filter, 209 | update, 210 | ) 211 | if err != nil { 212 | return "", err 213 | } 214 | if log.GetLevel() == log.DebugLevel { 215 | s.logger.Debugf("upsert %v", result) 216 | } 217 | return key, nil 218 | 219 | } 220 | 221 | // Delete deletes a document 222 | func (s *MongoDb) Delete(topicFullName, pulsarURL string) (string, error) { 223 | key, err := model.GetKeyFromNames(topicFullName, pulsarURL) 224 | if err != nil { 225 | return "", err 226 | } 227 | return s.DeleteByKey(key) 228 | } 229 | 230 | // DeleteByKey deletes a document based on key 231 | func (s *MongoDb) DeleteByKey(hashedTopicKey string) (string, error) { 232 | if ok, _ := exists(hashedTopicKey, s.collection); !ok { 233 | return "", errors.New("topic does not exist") 234 | } 235 | result, err := s.collection.DeleteMany(context.TODO(), bson.M{"key": hashedTopicKey}) 236 | if err != nil { 237 | return "", err 238 | } 239 | 240 | if result.DeletedCount > 1 { 241 | return "", errors.New("many documents match the same key") //this is impossible 242 | } 243 | return hashedTopicKey, nil 244 | } 245 | 246 | func exists(key string, coll *mongo.Collection) (bool, error) { 247 | var doc model.TopicConfig 248 | result := coll.FindOne(context.TODO(), bson.M{"key": key}) 249 | 250 | err := result.Decode(&doc) 251 | if err != nil { 252 | if err.Error() == "mongo: no documents in result" { 253 | return false, nil 254 | } 255 | return false, err 256 | } 257 | return true, nil 258 | } 259 | 260 | func getKey(topicCfg *model.TopicConfig) (string, error) { 261 | return model.GetKeyFromNames(topicCfg.TopicFullName, topicCfg.PulsarURL) 262 | } 263 | -------------------------------------------------------------------------------- /src/db/pulsardb.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/apache/pulsar-client-go/pulsar" 12 | "github.com/kafkaesque-io/pulsar-beam/src/model" 13 | "github.com/kafkaesque-io/pulsar-beam/src/pulsardriver" 14 | "github.com/kafkaesque-io/pulsar-beam/src/util" 15 | 16 | log "github.com/sirupsen/logrus" 17 | ) 18 | 19 | /** 20 | * Data design - we use a topic as a database table to store document per user topics basis 21 | * ll non-acked events are received by a consumer; processed to build an in memory database. 22 | * Creation creates a single document with user topic access information and webhook url and headers 23 | , this becomes an event sent by producer 24 | * Read reads 25 | * A topic prefix for the webhook configuration database 26 | **/ 27 | 28 | // the signal to track if the liveness of the reader process 29 | type liveSignal struct{} 30 | 31 | // a map of TopicConfig struct with Key, hash of pulsar URL and topic full name, is the key 32 | // var topics = make(map[string]model.TopicConfig) 33 | 34 | // PulsarHandler is the Pulsar database driver 35 | type PulsarHandler struct { 36 | PulsarURL string 37 | PulsarToken string 38 | TopicName string 39 | topicsLock sync.RWMutex 40 | client pulsar.Client 41 | producer pulsar.Producer 42 | topics map[string]model.TopicConfig 43 | logger *log.Entry 44 | } 45 | 46 | //Init is a Db interface method. 47 | func (s *PulsarHandler) Init() error { 48 | s.logger = log.WithFields(log.Fields{"app": "pulsardb"}) 49 | s.topics = make(map[string]model.TopicConfig) 50 | 51 | s.logger.Infof("database pulsar URL: %s", s.PulsarURL) 52 | if log.GetLevel() == log.DebugLevel { 53 | s.logger.Debugf("database pulsar token string is %s", s.PulsarToken) 54 | } 55 | 56 | var err error 57 | s.client, err = pulsardriver.NewPulsarClient(s.PulsarURL, s.PulsarToken) 58 | if err != nil { 59 | // this would be a serious problem so that we return with error 60 | return err 61 | } 62 | 63 | err = s.createProducer() 64 | if err != nil { 65 | // this would be a serious problem so that we return with error 66 | log.Errorf("failed to create producer error %v", err) 67 | return err 68 | } 69 | 70 | // a loop to receive and recover from failure 71 | go func() { 72 | sig := make(chan *liveSignal) 73 | go s.dbListener(sig) 74 | for { 75 | select { 76 | case <-sig: 77 | go s.dbListener(sig) 78 | } 79 | } 80 | }() 81 | 82 | return nil 83 | } 84 | 85 | //DbListener listens db updates 86 | func (s *PulsarHandler) dbListener(sig chan *liveSignal) error { 87 | defer func(termination chan *liveSignal) { 88 | s.logger.Errorf("tenant db listener terminated") 89 | termination <- &liveSignal{} 90 | }(sig) 91 | s.logger.Infof("listens to pulsar wh database changes") 92 | reader, err := s.client.CreateReader(pulsar.ReaderOptions{ 93 | Topic: s.TopicName, 94 | StartMessageID: pulsar.EarliestMessageID(), 95 | ReadCompacted: true, 96 | }) 97 | 98 | if err != nil { 99 | log.Errorf("dbListener failed to create reader, error %v", err) 100 | return err 101 | } 102 | defer reader.Close() 103 | 104 | ctx := context.Background() 105 | // infinite loop to receive messages 106 | for { 107 | data, err := reader.Next(ctx) 108 | if err != nil { 109 | log.Errorf("dbListener reader.Next() error %v", err) 110 | return err 111 | } 112 | doc := model.TopicConfig{} 113 | if err = json.Unmarshal(data.Payload(), &doc); err != nil { 114 | s.logger.Errorf("dblistener reader unmarshal error %v", err) 115 | // ignore error and move on 116 | } else { 117 | s.topicsLock.Lock() 118 | defer s.topicsLock.Unlock() 119 | if doc.TopicStatus != model.Deleted { 120 | s.logger.Infof("add topic configuration %s", doc.Key) 121 | s.topics[doc.Key] = doc 122 | } else { 123 | delete(s.topics, doc.Key) 124 | } 125 | } 126 | } 127 | } 128 | 129 | func (s *PulsarHandler) createProducer() error { 130 | var err error 131 | s.producer, err = s.client.CreateProducer(pulsar.ProducerOptions{ 132 | Topic: s.TopicName, 133 | DisableBatching: true, 134 | }) 135 | return err 136 | } 137 | 138 | //Sync is a Db interface method. 139 | func (s *PulsarHandler) Sync() error { 140 | return errors.New("Unsupported since this is automatically sync-ed") 141 | } 142 | 143 | //Health is a Db interface method 144 | func (s *PulsarHandler) Health() bool { 145 | return true 146 | } 147 | 148 | // Close closes database 149 | func (s *PulsarHandler) Close() error { 150 | s.producer.Close() 151 | // s.client.Close() 152 | // Here is a Client object leak 153 | return nil 154 | } 155 | 156 | //NewPulsarHandler initialize a Pulsar Db 157 | func NewPulsarHandler() (*PulsarHandler, error) { 158 | handler := PulsarHandler{ 159 | logger: log.WithFields(log.Fields{"app": "pulsardb"}), 160 | } 161 | handler.PulsarURL = util.GetConfig().PulsarBrokerURL 162 | if strings.HasPrefix(util.GetConfig().DbConnectionStr, "pulsar") { 163 | handler.PulsarURL = util.GetConfig().DbConnectionStr 164 | } 165 | handler.TopicName = util.GetConfig().DbName 166 | handler.PulsarToken = util.GetConfig().DbPassword 167 | err := handler.Init() 168 | return &handler, err 169 | } 170 | 171 | // Create creates a new document 172 | func (s *PulsarHandler) Create(topicCfg *model.TopicConfig) (string, error) { 173 | key, err := getKey(topicCfg) 174 | if err != nil { 175 | return key, err 176 | } 177 | 178 | if _, ok := s.topics[key]; ok { 179 | return key, errors.New(DocAlreadyExisted) 180 | } 181 | 182 | topicCfg.Key = key 183 | topicCfg.CreatedAt = time.Now() 184 | topicCfg.UpdatedAt = topicCfg.CreatedAt 185 | 186 | return s.updateCacheAndPulsar(topicCfg) 187 | } 188 | 189 | func (s *PulsarHandler) updateCacheAndPulsar(topicCfg *model.TopicConfig) (string, error) { 190 | 191 | ctx := context.Background() 192 | data, err := json.Marshal(*topicCfg) 193 | if err != nil { 194 | return "", err 195 | } 196 | msg := pulsar.ProducerMessage{ 197 | Payload: data, 198 | Key: topicCfg.Key, 199 | } 200 | 201 | if _, err = s.producer.Send(ctx, &msg); err != nil { 202 | return "", err 203 | } 204 | // s.producer.Flush() do not use it's a blocking call 205 | 206 | s.logger.Infof("send to Pulsar %s", topicCfg.Key) 207 | 208 | s.topics[topicCfg.Key] = *topicCfg 209 | return topicCfg.Key, nil 210 | } 211 | 212 | // GetByTopic gets a document by the topic name and pulsar URL 213 | func (s *PulsarHandler) GetByTopic(topicFullName, pulsarURL string) (*model.TopicConfig, error) { 214 | key, err := model.GetKeyFromNames(topicFullName, pulsarURL) 215 | if err != nil { 216 | return &model.TopicConfig{}, err 217 | } 218 | return s.GetByKey(key) 219 | } 220 | 221 | // GetByKey gets a document by the key 222 | func (s *PulsarHandler) GetByKey(hashedTopicKey string) (*model.TopicConfig, error) { 223 | if v, ok := s.topics[hashedTopicKey]; ok { 224 | return &v, nil 225 | } 226 | return &model.TopicConfig{}, errors.New(DocNotFound) 227 | } 228 | 229 | // Load loads the entire database into memory 230 | func (s *PulsarHandler) Load() ([]*model.TopicConfig, error) { 231 | results := []*model.TopicConfig{} 232 | for _, v := range s.topics { 233 | results = append(results, &v) 234 | } 235 | return results, nil 236 | } 237 | 238 | // Update updates or creates a topic config document 239 | func (s *PulsarHandler) Update(topicCfg *model.TopicConfig) (string, error) { 240 | key, err := getKey(topicCfg) 241 | if err != nil { 242 | return key, err 243 | } 244 | 245 | if _, ok := s.topics[key]; !ok { 246 | return s.Create(topicCfg) 247 | } 248 | 249 | v := s.topics[key] 250 | v.Token = topicCfg.Token 251 | v.Tenant = topicCfg.Tenant 252 | v.Notes = topicCfg.Notes 253 | v.TopicStatus = topicCfg.TopicStatus 254 | v.UpdatedAt = time.Now() 255 | v.Webhooks = topicCfg.Webhooks 256 | 257 | s.logger.Infof("upsert %s", key) 258 | return s.updateCacheAndPulsar(topicCfg) 259 | 260 | } 261 | 262 | // Delete deletes a document 263 | func (s *PulsarHandler) Delete(topicFullName, pulsarURL string) (string, error) { 264 | key, err := model.GetKeyFromNames(topicFullName, pulsarURL) 265 | if err != nil { 266 | return "", err 267 | } 268 | return s.DeleteByKey(key) 269 | } 270 | 271 | // DeleteByKey deletes a document based on key 272 | func (s *PulsarHandler) DeleteByKey(hashedTopicKey string) (string, error) { 273 | if _, ok := s.topics[hashedTopicKey]; !ok { 274 | return "", errors.New(DocNotFound) 275 | } 276 | 277 | v := s.topics[hashedTopicKey] 278 | v.TopicStatus = model.Deleted 279 | 280 | ctx := context.Background() 281 | data, err := json.Marshal(v) 282 | if err != nil { 283 | return "", err 284 | } 285 | 286 | msg := pulsar.ProducerMessage{ 287 | Payload: data, 288 | Key: v.Key, 289 | } 290 | 291 | if _, err = s.producer.Send(ctx, &msg); err != nil { 292 | return "", err 293 | } 294 | 295 | delete(s.topics, v.Key) 296 | return hashedTopicKey, nil 297 | } 298 | -------------------------------------------------------------------------------- /src/docs/api.go: -------------------------------------------------------------------------------- 1 | package docs 2 | 3 | import ( 4 | "github.com/kafkaesque-io/pulsar-beam/src/model" 5 | "github.com/kafkaesque-io/pulsar-beam/src/util" 6 | ) 7 | 8 | // swagger:operation POST /v2/firehose/{persistent}/{tenant}/{namespace}/{topic} Send-Messages idOfFirehoseEndpoint 9 | // 10 | // The endpoint receives a message in HTTP body that will be sent to Pulsar. 11 | // 12 | // --- 13 | // headers: 14 | // responses: 15 | // '200': 16 | // description: successfully sent messages 17 | // '401': 18 | // description: authentication failure 19 | // schema: 20 | // "$ref": "#/definitions/errorResponse" 21 | // '422': 22 | // description: invalid request parameters 23 | // schema: 24 | // "$ref": "#/definitions/errorResponse" 25 | // '500': 26 | // description: failed to read the http body 27 | // schema: 28 | // "$ref": "#/definitions/errorResponse" 29 | // '503': 30 | // description: failed to send messages to Pulsar 31 | // schema: 32 | // "$ref": "#/definitions/errorResponse" 33 | 34 | // swagger:operation GET /v2/sse/{persistent}/{tenant}/{namespace}/{topic} SSE-Event-Streaming idOfHTTPSeverSentEvent 35 | // The HTTP SSE endpoint receives messages in HTTP body from a Pulsar topic. 36 | // 37 | // --- 38 | // produces: 39 | // - text/event-stream 40 | // headers: 41 | // - name: PulsarURL 42 | // description: Specify a pulsar cluster. This can be ignored by the server side to enforce connecting to a local Pulsar cluster. 43 | // required: false 44 | // parameters: 45 | // - name: SubscriptionInitialPosition 46 | // in: query 47 | // description: specify subscription initial position in either latest or earliest, the default is latest 48 | // type: string 49 | // required: false 50 | // - name: SubscriptionType 51 | // in: query 52 | // description: specify subscription type in exclusive, shared, keyshared, or failover, the default is exclusive 53 | // type: string 54 | // required: false 55 | // - name: SubscriptionName 56 | // in: query 57 | // description: subscription name in minimum 5 charaters, a random subscription will be generated if not specified 58 | // type: string 59 | // required: false 60 | // responses: 61 | // '401': 62 | // description: authentication failure 63 | // schema: 64 | // "$ref": "#/definitions/errorResponse" 65 | // '422': 66 | // description: invalid request parameters 67 | // schema: 68 | // "$ref": "#/definitions/errorResponse" 69 | // '500': 70 | // description: failed to subscribe or receive messages from Pulsar 71 | // schema: 72 | // "$ref": "#/definitions/errorResponse" 73 | 74 | // swagger:operation GET /v2/poll/{persistent}/{tenant}/{namespace}/{topic} Long-Polling idOfHTTPLongPolling 75 | // The long polling endpoint receives messages in HTTP body from a Pulsar topic. 76 | // 77 | // --- 78 | // produces: 79 | // - text/event-poll 80 | // headers: 81 | // - name: PulsarURL 82 | // description: Specify a pulsar cluster. This can be ignored by the server side to enforce connecting to a local Pulsar cluster. 83 | // required: false 84 | // parameters: 85 | // - name: SubscriptionType 86 | // in: query 87 | // description: specify subscription type in exclusive, shared, keyshared, or failover, the default is exclusive 88 | // type: string 89 | // required: false 90 | // - name: SubscriptionName 91 | // in: query 92 | // description: subscription name in minimum 5 charaters, a random subscription will be generated if not specified 93 | // type: string 94 | // required: false 95 | // - name: batchSize 96 | // in: query 97 | // description: the batch size of the message list. The poll responds to the client When the batch size is reached. The default is 10 messages 98 | // type: integer 99 | // required: false 100 | // - name: perMessageTimeoutMs 101 | // in: query 102 | // description: Per message time out in milliseconds to wait the message from the Pulsar topic. The default is 300 millisecond 103 | // type: integer 104 | // required: false 105 | // responses: 106 | // '200': 107 | // description: successfully subscribed and received messages from a Pulsar topic 108 | // '204': 109 | // description: successfully subscribed to a Pulsar topic but receives no messages 110 | // '401': 111 | // description: authentication failure 112 | // schema: 113 | // "$ref": "#/definitions/errorResponse" 114 | // '422': 115 | // description: invalid request parameters 116 | // schema: 117 | // "$ref": "#/definitions/errorResponse" 118 | // '500': 119 | // description: failed to subscribe or receive messages from Pulsar 120 | // schema: 121 | // "$ref": "#/definitions/errorResponse" 122 | 123 | // swagger:route GET /v2/topic Get-Topic idOfGetTopic 124 | // Get a topic configuration based on the topic name. 125 | // 126 | // headers: 127 | // responses: 128 | // 200: topicGetResponse 129 | // 403: 130 | // 404: errorResponse 131 | // 422: errorResponse 132 | // 500: errorResponse 133 | 134 | // swagger:route GET /v2/topic/{topicKey} Get-Topic idOfGetTopicKey 135 | // Get a topic configuration based on topic key. 136 | // 137 | // headers: 138 | // responses: 139 | // 200: topicGetResponse 140 | // 403: 141 | // 404: errorResponse 142 | // 422: errorResponse 143 | // 500: errorResponse 144 | 145 | // swagger:route POST /v2/topic Create-or-Update-Topic idOfUpdateTopic 146 | // Create or update a topic configuration. 147 | // Please do NOT specifiy key. The topic status must be for 1 for activation. 148 | // 149 | // responses: 150 | // 201: topicUpdateResponse 151 | // 403: 152 | // 409: errorResponse 153 | // 422: errorResponse 154 | // 500: errorResponse 155 | 156 | // swagger:route DELETE /v2/topic Delete-Topic idOfDeleteTopicKey 157 | // Delete a topic configuration based on topic name. 158 | // 159 | // headers: 160 | // responses: 161 | // 200: topicDeleteResponse 162 | // 403: errorResponse 163 | // 404: errorResponse 164 | // 422: errorResponse 165 | // 500: errorResponse 166 | 167 | // swagger:route DELETE /v2/topic/{topicKey} Delete-Topic idOfDeleteTopic 168 | // Delete a topic configuration based on topic key. 169 | // 170 | // headers: 171 | // responses: 172 | // 200: topicDeleteResponse 173 | // 403: errorResponse 174 | // 404: errorResponse 175 | // 422: errorResponse 176 | // 500: errorResponse 177 | 178 | type sseQueryParams struct { 179 | } 180 | 181 | // swagger:parameters idOfGetTopic 182 | type topicGetParams struct { 183 | // in:body 184 | Body model.TopicKey 185 | } 186 | 187 | // swagger:parameters idOfDeleteTopic 188 | type topicDeleteParams struct { 189 | // in:body 190 | Body model.TopicKey 191 | } 192 | 193 | // swagger:response topicGetResponse 194 | type topicGetResponse struct { 195 | Body model.TopicConfig 196 | } 197 | 198 | // swagger:response topicDeleteResponse 199 | type topicDeleteResponse struct { 200 | Body model.TopicConfig 201 | } 202 | 203 | // swagger:response topicUpdateResponse 204 | type topicUpdateResponse struct { 205 | Body model.TopicConfig 206 | } 207 | 208 | // swagger:parameters idOfUpdateTopic 209 | type topicUpdateParams struct { 210 | // in:body 211 | Body model.TopicConfig 212 | } 213 | 214 | // swagger:response errorResponse 215 | type errorResponse struct { 216 | // in:body 217 | Body util.ResponseErr 218 | } 219 | 220 | // swagger:model errorResponse 221 | type errorResponse2 struct { 222 | // required: true 223 | Body util.ResponseErr 224 | } 225 | -------------------------------------------------------------------------------- /src/docs/docs.go: -------------------------------------------------------------------------------- 1 | // Package docs Pulsar Beam 2 | // 3 | // Documentation of Pulsar Beam API. 4 | // 5 | // Schemes: http 6 | // BasePath: / 7 | // Version: 0.2.0 8 | // 9 | // License: Apache 2.0 https://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Consumes: 12 | // - application/json 13 | // 14 | // Produces: 15 | // - application/json 16 | // 17 | // Schemes: 18 | // - https 19 | // 20 | // Security: 21 | // - bearerAuth: [] 22 | // 23 | // SecurityDefinitions: 24 | // bearerAuth: 25 | // type: bearer 26 | // bearerFormat: JWT 27 | // 28 | // swagger:meta 29 | package docs 30 | -------------------------------------------------------------------------------- /src/e2e/e2etest.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "github.com/apache/pulsar-client-go/pulsar" 15 | "github.com/kafkaesque-io/pulsar-beam/src/model" 16 | "github.com/kafkaesque-io/pulsar-beam/src/util" 17 | ) 18 | 19 | // This is an end to end test program. It does these steps in order 20 | // - Create a topic and webhook integration with Pulsar Beam 21 | // - Trigger Pulsar Beam to load the new configuration 22 | // - Start a pulsar consumer listens to the sink topic 23 | // - Send an event to pulsar beam 24 | // - Verify the listener can reveive the event on the sink topic 25 | // - Delete the topic configuration including webhook 26 | // This test uses the default dev setting including cluster name, 27 | // RSA key pair, and mongo database access. 28 | 29 | var pulsarToken, 30 | pulsarURL, 31 | webhookTopic, 32 | restAPIToken, 33 | webhookURL, 34 | functionSinkTopic string 35 | 36 | var restURL = "http://localhost:8085/v2/topic" 37 | 38 | type received struct{} 39 | 40 | func init() { 41 | // required parameters from os.env 42 | pulsarToken = getEnvPanic("PULSAR_TOKEN") 43 | pulsarURL = getEnvPanic("PULSAR_URI") 44 | webhookTopic = getEnvPanic("WEBHOOK_TOPIC") 45 | restAPIToken = getEnvPanic("REST_API_TOKEN") 46 | webhookURL = getEnvPanic("WEBHOOK2_URL") 47 | functionSinkTopic = getEnvPanic("FN_SINK_TOPIC") 48 | 49 | } 50 | 51 | func getEnvPanic(key string) string { 52 | if value, ok := os.LookupEnv(key); ok { 53 | return value 54 | } 55 | log.Panic("missing required env " + key) 56 | return "" 57 | } 58 | 59 | func eval(val bool, verdict string) { 60 | if !val { 61 | log.Panic("Failed verdict " + verdict) 62 | } 63 | } 64 | 65 | func errNil(err error) { 66 | if err != nil { 67 | log.Panic(err) 68 | } 69 | } 70 | 71 | // returns the key of the topic 72 | func addWebhookToDb() string { 73 | // Create a topic and webhook via REST 74 | topicConfig, err := model.NewTopicConfig(webhookTopic, pulsarURL, pulsarToken) 75 | errNil(err) 76 | 77 | // log.Printf("register webhook %s\nwith pulsar %s and topic %s\n", webhookURL, pulsarURL, webhookTopic) 78 | wh := model.NewWebhookConfig(webhookURL) 79 | wh.InitialPosition = "earliest" 80 | wh.Subscription = "my-subscription" 81 | topicConfig.Webhooks = append(topicConfig.Webhooks, wh) 82 | 83 | if _, err = model.ValidateTopicConfig(topicConfig); err != nil { 84 | log.Fatal("Invalid topic config ", err) 85 | } 86 | 87 | reqJSON, err := json.Marshal(topicConfig) 88 | if err != nil { 89 | log.Fatal("Topic marshalling error Error reading request. ", err) 90 | } 91 | log.Println("create topic and webhook with REST call") 92 | req, err := http.NewRequest("POST", restURL, bytes.NewBuffer(reqJSON)) 93 | if err != nil { 94 | log.Fatal("Error reading request. ", err) 95 | } 96 | 97 | req.Header.Set("Authorization", restAPIToken) 98 | 99 | // Set client timeout 100 | client := &http.Client{Timeout: time.Second * 10} 101 | 102 | // Send request 103 | resp, err := client.Do(req) 104 | if err != nil { 105 | log.Fatal("Error reading response from Beam. ", err) 106 | } 107 | defer resp.Body.Close() 108 | 109 | log.Printf("post call to rest API statusCode %d", resp.StatusCode) 110 | eval(resp.StatusCode == 201, "expected rest api status code is 201") 111 | return topicConfig.Key 112 | } 113 | 114 | func deleteWebhook(key string) { 115 | log.Printf("delete topic and webhook with REST call with key %s\n", key) 116 | req, err := http.NewRequest("DELETE", restURL+"/"+key, nil) 117 | errNil(err) 118 | 119 | req.Header.Set("Authorization", restAPIToken) 120 | 121 | // Set client timeout 122 | client := &http.Client{Timeout: time.Second * 10} 123 | 124 | // Send request 125 | resp, err := client.Do(req) 126 | errNil(err) 127 | defer resp.Body.Close() 128 | 129 | log.Printf("delete topic %s rest API statusCode %d", key, resp.StatusCode) 130 | eval(resp.StatusCode == 200, "expected delete status code is 200") 131 | } 132 | 133 | func produceMessage(sentMessage string) string { 134 | 135 | beamReceiverURL := "http://localhost:8085/v1/firehose" 136 | originalData := []byte(`{"Data": "` + sentMessage + `"}`) 137 | log.Printf("send to topic %s with message %s \n", webhookTopic, string(originalData)) 138 | 139 | //Send to Pulsar Beam 140 | req, err := http.NewRequest("POST", beamReceiverURL, bytes.NewBuffer(originalData)) 141 | if err != nil { 142 | log.Fatal("Error reading request. ", err) 143 | } 144 | 145 | req.Header.Set("Authorization", pulsarToken) 146 | req.Header.Set("PulsarUrl", pulsarURL) 147 | req.Header.Set("TopicFn", webhookTopic) 148 | 149 | // Set client timeout 150 | client := &http.Client{Timeout: time.Second * 10} 151 | 152 | // Send request 153 | resp, err := client.Do(req) 154 | if resp != nil { 155 | defer resp.Body.Close() 156 | } 157 | if err != nil { 158 | log.Fatal("Error reading response from Beam. ", err) 159 | } 160 | 161 | eval(resp.StatusCode == 200, fmt.Sprintf("expected receiver status code is 200 but received %d", resp.StatusCode)) 162 | 163 | return sentMessage 164 | } 165 | 166 | func subscribe(verifyStr string, verified chan received) { 167 | subscriptionName := "my-subscription" 168 | // log.Printf("Pulsar Consumer sink topic %s\n", functionSinkTopic) 169 | log.Printf("Pulsar Consumer subscribe to %s\n", subscriptionName) 170 | 171 | // Configuration variables pertaining to this consumer 172 | trustStore := util.AssignString(os.Getenv("TrustStore"), "/etc/ssl/certs/ca-bundle.crt") 173 | log.Printf("trust store %v", trustStore) 174 | 175 | token := pulsar.NewAuthenticationToken(pulsarToken) 176 | 177 | // Pulsar client 178 | client, err := pulsar.NewClient(pulsar.ClientOptions{ 179 | URL: pulsarURL, 180 | Authentication: token, 181 | TLSTrustCertsFilePath: trustStore, 182 | }) 183 | errNil(err) 184 | 185 | consumer, err := client.Subscribe(pulsar.ConsumerOptions{ 186 | Topic: functionSinkTopic, 187 | SubscriptionName: subscriptionName, 188 | SubscriptionInitialPosition: pulsar.SubscriptionPositionLatest, 189 | }) 190 | errNil(err) 191 | defer consumer.Close() 192 | defer client.Close() 193 | 194 | ctx, cancel := context.WithTimeout(context.Background(), 181*time.Second) 195 | defer cancel() 196 | 197 | receivedStr := "" 198 | 199 | // replied string has suffix 200 | log.Printf("expect received string %s", verifyStr) 201 | for !strings.HasSuffix(receivedStr, verifyStr) { 202 | msg, err := consumer.Receive(ctx) 203 | errNil(err) 204 | 205 | receivedStr = string(msg.Payload()) 206 | log.Printf("Received message : %v\n", receivedStr) 207 | 208 | consumer.Ack(msg) 209 | } 210 | 211 | verified <- received{} 212 | } 213 | 214 | func main() { 215 | 216 | receivedChan := make(chan received, 1) 217 | sentMessage := fmt.Sprintf("hello-from-e2e-test %d", time.Now().Unix()) 218 | 219 | key := addWebhookToDb() 220 | log.Printf("add webhook %s", key) 221 | go subscribe(sentMessage, receivedChan) 222 | time.Sleep(15 * time.Second) 223 | 224 | produceMessage(sentMessage) 225 | 226 | select { 227 | case <-receivedChan: 228 | log.Printf("successful received and verified") 229 | deleteWebhook(key) 230 | case <-time.Tick(121 * time.Second): 231 | deleteWebhook(key) 232 | log.Fatal("failed to receive expected message, timed out") 233 | } 234 | 235 | } 236 | -------------------------------------------------------------------------------- /src/e2e/source_this_env.sh: -------------------------------------------------------------------------------- 1 | # 2 | # These env should be provided by CI runtime env variable. 3 | # Only source this file for local testing purpose. 4 | # 5 | export PULSAR_TOKEN="eyJhbG" 6 | export PULSAR_URI="pulsar+ssl://useast1.gcp.kafkaesque.io:6651" 7 | export WEBHOOK_TOPIC="persistent://" 8 | export REST_API_TOKEN="eyJhbG" 9 | export WEBHOOK2_URL="http://localhost:8080/wh" 10 | export FN_SINK_TOPIC="persistent://" 11 | 12 | # a compacted Pulsar Topic can be used as Database table for RESTful API 13 | export REST_DB_TABLE_TOPIC="persistent:////" 14 | -------------------------------------------------------------------------------- /src/icrypto/icrypto.go: -------------------------------------------------------------------------------- 1 | package icrypto 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "crypto/rand" 7 | "crypto/rsa" 8 | "crypto/x509" 9 | "errors" 10 | "io" 11 | ) 12 | 13 | const ( 14 | // AesAlgo enum for AES 15 | AesAlgo int = iota 16 | // RsaAlgo enum for RSA 17 | RsaAlgo 18 | ) 19 | 20 | // AsymKeys asymmetric keys 21 | type AsymKeys interface { 22 | getPrivateKey() ([]byte, error) 23 | getPublicKey() ([]byte, error) 24 | } 25 | 26 | // Encrypto encryption 27 | type Encrypto interface { 28 | encrypt(plaintext []byte, key []byte) ([]byte, error) 29 | encryptWithDefaultKey(plaintext []byte) ([]byte, error) 30 | } 31 | 32 | // Decrypto decryption 33 | type Decrypto interface { 34 | decrypt(ciphertext []byte, key []byte) ([]byte, error) 35 | decryptWithDefaultKey(plaintext []byte) ([]byte, error) 36 | } 37 | 38 | // AES struct implementation including encryption and decryption 39 | type AES struct { 40 | DefaultSalt string 41 | } 42 | 43 | // Encrypt encrypts with asymmetric key 44 | func (a *AES) Encrypt(plaintext []byte, key []byte) ([]byte, error) { 45 | c, err := aes.NewCipher(key) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | gcm, err := cipher.NewGCM(c) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | nonce := make([]byte, gcm.NonceSize()) 56 | if _, err = io.ReadFull(rand.Reader, nonce); err != nil { 57 | return nil, err 58 | } 59 | 60 | return gcm.Seal(nonce, nonce, plaintext, nil), nil 61 | } 62 | 63 | // EncryptWithDefaultKey encrypts with a default key 64 | func (a *AES) EncryptWithDefaultKey(plaintext []byte) ([]byte, error) { 65 | defaultKey := []byte(a.DefaultSalt) 66 | return a.Encrypt(plaintext, defaultKey) 67 | } 68 | 69 | // Decrypt decrypts with asymmetric key 70 | func (a *AES) Decrypt(ciphertext []byte, key []byte) ([]byte, error) { 71 | c, err := aes.NewCipher(key) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | gcm, err := cipher.NewGCM(c) 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | nonceSize := gcm.NonceSize() 82 | if len(ciphertext) < nonceSize { 83 | return nil, errors.New("ciphertext too short") 84 | } 85 | 86 | nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] 87 | return gcm.Open(nil, nonce, ciphertext, nil) 88 | } 89 | 90 | // DecryptWithDefaultKey decrypts with a default key 91 | func (a *AES) DecryptWithDefaultKey(plaintext []byte) ([]byte, error) { 92 | defaultKey := []byte(a.DefaultSalt) 93 | return a.Decrypt(plaintext, defaultKey) 94 | } 95 | 96 | //RSA struct implementation including encryption and decryption 97 | type RSA struct { 98 | MyPrivateKey *rsa.PrivateKey 99 | MyPublicKey *rsa.PublicKey 100 | } 101 | 102 | // NewRSAWithKeys with private and public keys 103 | func NewRSAWithKeys(priv []byte, pub []byte) (RSA, error) { 104 | var privKey *rsa.PrivateKey 105 | var pubKey *rsa.PublicKey 106 | var err error 107 | if priv != nil { 108 | //pemblock, _ := pem.Decode(priv) 109 | privKey, err = x509.ParsePKCS1PrivateKey(priv) 110 | if err != nil { 111 | return RSA{}, err 112 | } 113 | } 114 | if pub != nil { 115 | //pemblock2, _ := pem.Decode(pub) 116 | pubKey, err = x509.ParsePKCS1PublicKey(pub) 117 | if err != nil { 118 | return RSA{}, err 119 | } 120 | } 121 | return RSA{MyPrivateKey: privKey, MyPublicKey: pubKey}, nil 122 | } 123 | 124 | // NewRSA creates RSA keys pair 125 | func NewRSA() (RSA, error) { 126 | keys := RSA{} 127 | var err error 128 | 129 | keys.MyPrivateKey, err = rsa.GenerateKey(rand.Reader, 2048) 130 | 131 | if err != nil { 132 | return RSA{}, err 133 | } 134 | 135 | keys.MyPublicKey = &(keys.MyPrivateKey.PublicKey) 136 | return keys, nil 137 | } 138 | 139 | // GetPublicKey gets the public RSA key 140 | func (a *RSA) GetPublicKey() ([]byte, error) { 141 | key := x509.MarshalPKCS1PublicKey(a.MyPublicKey) 142 | return key, nil 143 | } 144 | 145 | // GetPrivateKey gets the private RSA key 146 | func (a *RSA) GetPrivateKey() ([]byte, error) { 147 | key := x509.MarshalPKCS1PrivateKey(a.MyPrivateKey) 148 | return key, nil 149 | } 150 | 151 | // Encrypt encrypts with RSA key 152 | func (a *RSA) Encrypt(plaintext []byte, key []byte) ([]byte, error) { 153 | 154 | return nil, errors.New("unsupported") 155 | } 156 | 157 | // EncryptWithDefaultKey encrypts with a default RSA key 158 | func (a *RSA) EncryptWithDefaultKey(plaintext []byte) ([]byte, error) { 159 | ciphertext, err := rsa.EncryptPKCS1v15(rand.Reader, a.MyPublicKey, plaintext) 160 | if err != nil { 161 | return nil, err 162 | } 163 | return ciphertext, nil 164 | } 165 | 166 | // Decrypt decrypts with RSA key 167 | func (a *RSA) Decrypt(ciphertext []byte, key []byte) ([]byte, error) { 168 | 169 | return nil, errors.New("unsupported") 170 | } 171 | 172 | // DecryptWithDefaultKey decrypts with a default RSA key 173 | func (a *RSA) DecryptWithDefaultKey(ciphertext []byte) ([]byte, error) { 174 | plaintext, err := rsa.DecryptPKCS1v15(rand.Reader, a.MyPrivateKey, ciphertext) 175 | if err != nil { 176 | return nil, err 177 | } 178 | return plaintext, nil 179 | } 180 | -------------------------------------------------------------------------------- /src/icrypto/pulsar-jwt.go: -------------------------------------------------------------------------------- 1 | package icrypto 2 | 3 | // This is JWT sign/verify with the same key algo used in Pulsar. 4 | 5 | import ( 6 | "bufio" 7 | "crypto/rsa" 8 | "crypto/x509" 9 | "encoding/binary" 10 | "encoding/pem" 11 | "errors" 12 | "io/ioutil" 13 | "os" 14 | "time" 15 | 16 | "github.com/golang-jwt/jwt" 17 | log "github.com/sirupsen/logrus" 18 | ) 19 | 20 | // RSAKeyPair for JWT token sign and verification 21 | type RSAKeyPair struct { 22 | PrivateKey *rsa.PrivateKey 23 | PublicKey *rsa.PublicKey 24 | } 25 | 26 | const ( 27 | tokenDuration = 24 28 | expireOffset = 3600 29 | ) 30 | 31 | var jwtRsaKeys *RSAKeyPair 32 | 33 | // NewRSAKeyPair creates a pair of RSA key for JWT token sign and verification 34 | func NewRSAKeyPair(privateKeyPath, publicKeyPath string) *RSAKeyPair { 35 | if jwtRsaKeys == nil { 36 | jwtRsaKeys = &RSAKeyPair{ 37 | PrivateKey: getPrivateKey(privateKeyPath), 38 | PublicKey: getPublicKey(publicKeyPath), 39 | } 40 | } 41 | 42 | return jwtRsaKeys 43 | } 44 | 45 | // GenerateToken generates token with user defined subject 46 | func (keys *RSAKeyPair) GenerateToken(userSubject string) (string, error) { 47 | token := jwt.New(jwt.SigningMethodRS256) 48 | token.Claims = jwt.MapClaims{ 49 | // "exp": time.Now().Add(time.Hour * time.Duration(24)).Unix(), 50 | // "iat": time.Now().Unix(), 51 | "sub": userSubject, 52 | } 53 | tokenString, err := token.SignedString(keys.PrivateKey) 54 | if err != nil { 55 | return "", err 56 | } 57 | return tokenString, nil 58 | } 59 | 60 | // DecodeToken decodes a token string 61 | func (keys *RSAKeyPair) DecodeToken(tokenStr string) (*jwt.Token, error) { 62 | token, err := jwt.Parse(tokenStr, func(token *jwt.Token) (interface{}, error) { 63 | return jwtRsaKeys.PublicKey, nil 64 | }) 65 | 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | if token.Valid { 71 | return token, nil 72 | } 73 | 74 | return nil, errors.New("invalid token") 75 | } 76 | 77 | //TODO: support multiple subjects in claims 78 | 79 | // GetTokenSubject gets the subjects from a token 80 | func (keys *RSAKeyPair) GetTokenSubject(tokenStr string) (string, error) { 81 | token, err := keys.DecodeToken(tokenStr) 82 | if err != nil { 83 | return "", err 84 | } 85 | claims := token.Claims.(jwt.MapClaims) 86 | subjects, ok := claims["sub"] 87 | if ok { 88 | return subjects.(string), nil 89 | } 90 | return "", errors.New("missing subjects") 91 | } 92 | 93 | // VerifyTokenSubject verifies a token string based on required matching subject 94 | func (keys *RSAKeyPair) VerifyTokenSubject(tokenStr, subject string) (bool, error) { 95 | token, err := keys.DecodeToken(tokenStr) 96 | 97 | if err != nil { 98 | return false, err 99 | } 100 | 101 | claims := token.Claims.(jwt.MapClaims) 102 | 103 | if subject == claims["sub"] { 104 | return true, nil 105 | } 106 | 107 | return false, errors.New("incorrect sub") 108 | } 109 | 110 | // GetTokenRemainingValidity is the remaining seconds before token expires 111 | func (keys *RSAKeyPair) GetTokenRemainingValidity(timestamp interface{}) int { 112 | if validity, ok := timestamp.(float64); ok { 113 | tm := time.Unix(int64(validity), 0) 114 | remainer := tm.Sub(time.Now()) 115 | if remainer > 0 { 116 | return int(remainer.Seconds() + expireOffset) 117 | } 118 | } 119 | return expireOffset 120 | } 121 | 122 | // supports pk12 jks binary format 123 | func readPK12(file string) ([]byte, error) { 124 | osFile, err := os.Open(file) 125 | if err != nil { 126 | return nil, err 127 | } 128 | reader := bufio.NewReaderSize(osFile, 4) 129 | 130 | return ioutil.ReadAll(reader) 131 | } 132 | 133 | // decode PEM format to array of bytes 134 | func decodePEM(pemFilePath string) ([]byte, error) { 135 | keyFile, err := os.Open(pemFilePath) 136 | defer keyFile.Close() 137 | if err != nil { 138 | return nil, err 139 | } 140 | 141 | pemfileinfo, _ := keyFile.Stat() 142 | pembytes := make([]byte, pemfileinfo.Size()) 143 | 144 | buffer := bufio.NewReader(keyFile) 145 | _, err = buffer.Read(pembytes) 146 | 147 | data, _ := pem.Decode([]byte(pembytes)) 148 | return data.Bytes, err 149 | } 150 | 151 | func parseX509PKCS8PrivateKey(data []byte) *rsa.PrivateKey { 152 | key, err := x509.ParsePKCS8PrivateKey(data) 153 | 154 | if err != nil { 155 | panic(err) 156 | } 157 | 158 | rsaPrivate, ok := key.(*rsa.PrivateKey) 159 | if !ok { 160 | log.Fatalf("expected key to be of type *ecdsa.PrivateKey, but actual was %T", key) 161 | } 162 | 163 | return rsaPrivate 164 | } 165 | 166 | func parseX509PKIXPublicKey(data []byte) *rsa.PublicKey { 167 | publicKeyImported, err := x509.ParsePKIXPublicKey(data) 168 | 169 | if err != nil { 170 | panic(err) 171 | } 172 | 173 | rsaPub, ok := publicKeyImported.(*rsa.PublicKey) 174 | if !ok { 175 | panic(err) 176 | } 177 | 178 | return rsaPub 179 | } 180 | 181 | // Since we support PEM And binary fomat of PKCS12/X509 keys, 182 | // this function tries to determine which format 183 | func fileFormat(file string) (string, error) { 184 | osFile, err := os.Open(file) 185 | if err != nil { 186 | return "", err 187 | } 188 | reader := bufio.NewReaderSize(osFile, 4) 189 | // attempt to guess based on first 4 bytes of input 190 | data, err := reader.Peek(4) 191 | if err != nil { 192 | return "", err 193 | } 194 | 195 | magic := binary.BigEndian.Uint32(data) 196 | if magic == 0x2D2D2D2D || magic == 0x434f4e4e { 197 | // Starts with '----' or 'CONN' (what s_client prints...) 198 | return "PEM", nil 199 | } 200 | if magic&0xFFFF0000 == 0x30820000 { 201 | // Looks like the input is DER-encoded, so it's either PKCS12 or X.509. 202 | if magic&0x0000FF00 == 0x0300 { 203 | // Probably X.509 204 | return "DER", nil 205 | } 206 | return "PKCS12", nil 207 | } 208 | 209 | return "", errors.New("undermined format") 210 | } 211 | 212 | func getDataFromKeyFile(file string) ([]byte, error) { 213 | format, err := fileFormat(file) 214 | if err != nil { 215 | return nil, err 216 | } 217 | 218 | switch format { 219 | case "PEM": 220 | return decodePEM(file) 221 | case "PKCS12": 222 | return readPK12(file) 223 | default: 224 | return nil, errors.New("unsupported format") 225 | } 226 | } 227 | 228 | func getPrivateKey(file string) *rsa.PrivateKey { 229 | data, err := getDataFromKeyFile(file) 230 | if err != nil { 231 | log.Fatalf("failed to load private key %v", err) 232 | } 233 | 234 | return parseX509PKCS8PrivateKey(data) 235 | } 236 | 237 | func getPublicKey(file string) *rsa.PublicKey { 238 | data, err := getDataFromKeyFile(file) 239 | if err != nil { 240 | log.Fatalf("failed to load public key %v", err) 241 | } 242 | 243 | return parseX509PKIXPublicKey(data) 244 | } 245 | -------------------------------------------------------------------------------- /src/icrypto/util.go: -------------------------------------------------------------------------------- 1 | package icrypto 2 | 3 | // encryption and decryption utility functions 4 | import ( 5 | "encoding/base64" 6 | "math/rand" 7 | 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | var e AES 12 | 13 | const defaultSymKey string = "popl4190LKOI4862" //16 character length 14 | 15 | var defaultRunes = []rune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 16 | 17 | // init 18 | func init() { 19 | e = AES{DefaultSalt: defaultSymKey} 20 | } 21 | 22 | // EncryptWithBase64 encrypts a string with AES default key and returns 64encoded string 23 | func EncryptWithBase64(str string) (string, error) { 24 | text := []byte(str) 25 | encrypted, err := e.EncryptWithDefaultKey(text) 26 | if err != nil { 27 | return "", err 28 | } 29 | encoded := base64.StdEncoding.EncodeToString(encrypted) 30 | return encoded, nil 31 | } 32 | 33 | // DecryptWithBase64 a 64encoded string with the default key AES 34 | func DecryptWithBase64(str string) (string, error) { 35 | decoded, err1 := base64.StdEncoding.DecodeString(str) 36 | if err1 != nil { 37 | log.Errorf("base64 decode error: %v", err1) 38 | return "", err1 39 | } 40 | decrypted, err := e.DecryptWithDefaultKey(decoded) 41 | if err != nil { 42 | return "", err 43 | } 44 | return string(decrypted), nil 45 | } 46 | 47 | // RandKey generates a random key in n length 48 | func RandKey(n int) string { 49 | b := make([]rune, n) 50 | for i := range b { 51 | b[i] = defaultRunes[rand.Intn(len(defaultRunes))] 52 | } 53 | return string(b) 54 | } 55 | 56 | // GenTopicKey generates a random key in 24 char length. 57 | func GenTopicKey() string { 58 | return RandKey(24) 59 | } 60 | -------------------------------------------------------------------------------- /src/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "runtime" 7 | 8 | "github.com/google/gops/agent" 9 | "github.com/kafkaesque-io/pulsar-beam/src/broker" 10 | "github.com/kafkaesque-io/pulsar-beam/src/route" 11 | "github.com/kafkaesque-io/pulsar-beam/src/util" 12 | "github.com/rs/cors" 13 | log "github.com/sirupsen/logrus" 14 | 15 | _ "github.com/kafkaesque-io/pulsar-beam/src/docs" // This line is required for go-swagger to find docs 16 | ) 17 | 18 | var mode = util.AssignString(os.Getenv("ProcessMode"), *flag.String("mode", "hybrid", "server running mode")) 19 | 20 | func main() { 21 | // runtime.GOMAXPROCS does not the container's CPU quota in Kubernetes 22 | // therefore, it requires to be set explicitly 23 | runtime.GOMAXPROCS(util.GetEnvInt("GOMAXPROCS", 1)) 24 | 25 | // gops debug instrument 26 | if err := agent.Listen(agent.Options{}); err != nil { 27 | log.Panicf("gops instrument error %v", err) 28 | } 29 | 30 | exit := make(chan bool) // future use to exit the main program if in broker only mode 31 | config := util.Init() 32 | 33 | flag.Parse() 34 | log.Warnf("start server mode %s", mode) 35 | if !util.IsValidMode(&mode) { 36 | log.Panic("Unsupported server mode") 37 | } 38 | 39 | if util.IsBrokerRequired(&mode) { 40 | broker.Init(config) 41 | } 42 | if util.IsHTTPRouterRequired(&mode) { 43 | route.Init() 44 | 45 | c := cors.New(cors.Options{ 46 | AllowedOrigins: []string{"http://localhost:8085", "http://localhost:8080"}, 47 | AllowCredentials: true, 48 | AllowedHeaders: []string{"Authorization", "PulsarTopicUrl"}, 49 | }) 50 | 51 | router := route.NewRouter(&mode) 52 | 53 | handler := c.Handler(router) 54 | config := util.GetConfig() 55 | port := util.AssignString(config.PORT, "8085") 56 | certFile := util.GetConfig().CertFile 57 | keyFile := util.GetConfig().KeyFile 58 | log.Fatal(util.ListenAndServeTLS(":"+port, certFile, keyFile, handler)) 59 | } 60 | 61 | for util.IsBroker(&mode) { 62 | select { 63 | case <-exit: 64 | os.Exit(2) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/middleware/middleware.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | //middleware includes auth, rate limit, and etc. 4 | import ( 5 | "net/http" 6 | "strings" 7 | 8 | "github.com/kafkaesque-io/pulsar-beam/src/util" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | var ( 14 | // Rate is the default global rate limit 15 | // This rate only limits the rate hitting on endpoint 16 | // It does not limit the underline resource access 17 | Rate = NewSema(200) 18 | ) 19 | 20 | // AuthFunc is a function type to allow pluggable authentication middleware 21 | type AuthFunc func(next http.Handler) http.Handler 22 | 23 | // AuthVerifyJWT Authenticate middleware function 24 | func AuthVerifyJWT(next http.Handler) http.Handler { 25 | switch util.GetConfig().HTTPAuthImpl { 26 | case "noauth": 27 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 28 | r.Header.Set("injectedSubs", util.SuperRoles[0]) 29 | next.ServeHTTP(w, r) 30 | }) 31 | default: 32 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 33 | tokenStr := strings.TrimSpace(strings.Replace(r.Header.Get("Authorization"), "Bearer", "", 1)) 34 | subjects, err := util.JWTAuth.GetTokenSubject(tokenStr) 35 | 36 | if err == nil { 37 | log.Infof("Authenticated with subjects %s", subjects) 38 | r.Header.Set("injectedSubs", subjects) 39 | next.ServeHTTP(w, r) 40 | } else { 41 | http.Error(w, "Unauthorized", http.StatusUnauthorized) 42 | } 43 | 44 | }) 45 | } 46 | } 47 | 48 | // AuthHeaderRequired is a very weak auth to verify token existence only. 49 | func AuthHeaderRequired(next http.Handler) http.Handler { 50 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 51 | tokenStr := strings.TrimSpace(strings.Replace(r.Header.Get("Authorization"), "Bearer", "", 1)) 52 | 53 | if len(tokenStr) > 1 { 54 | next.ServeHTTP(w, r) 55 | } else { 56 | http.Error(w, "Unauthorized", http.StatusUnauthorized) 57 | } 58 | 59 | }) 60 | } 61 | 62 | // NoAuth bypasses the auth middleware 63 | func NoAuth(next http.Handler) http.Handler { 64 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 65 | next.ServeHTTP(w, r) 66 | }) 67 | } 68 | 69 | // LimitRate rate limites against http handler 70 | // use semaphore as a simple rate limiter 71 | func LimitRate(next http.Handler) http.Handler { 72 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 73 | err := Rate.Acquire() 74 | if err != nil { 75 | http.Error(w, "Too many requests", http.StatusTooManyRequests) 76 | } else { 77 | next.ServeHTTP(w, r) 78 | } 79 | Rate.Release() 80 | }) 81 | } 82 | -------------------------------------------------------------------------------- /src/middleware/semaphore.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // Sema the semaphore object used internally 8 | type Sema struct { 9 | Size int 10 | Ch chan int 11 | } 12 | 13 | // NewSema creates a new semaphore 14 | func NewSema(Length int) Sema { 15 | obj := Sema{} 16 | InitChan := make(chan int, Length) 17 | obj.Size = Length 18 | obj.Ch = InitChan 19 | return obj 20 | } 21 | 22 | // Acquire aquires a semaphore lock 23 | func (s *Sema) Acquire() error { 24 | select { 25 | case s.Ch <- 1: 26 | return nil 27 | default: 28 | return errors.New("all semaphore buffer full") 29 | } 30 | } 31 | 32 | // Release release a semaphore lock 33 | func (s *Sema) Release() error { 34 | select { 35 | case <-s.Ch: 36 | return nil 37 | default: 38 | return errors.New("all semaphore buffer empty") 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/model/message.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/apache/pulsar-client-go/pulsar" 8 | ) 9 | 10 | // PulsarMessage is the Pulsar Message type 11 | type PulsarMessage struct { 12 | Payload []byte `json:"payload"` 13 | Topic string `json:"topic"` 14 | EventTime time.Time `json:"eventTime"` 15 | PublishTime time.Time `json:"publishTime"` 16 | MessageID string `json:"messageId"` 17 | Key string `json:"key"` 18 | } 19 | 20 | // PulsarMessages encapsulates a list of messages to be returned to a client 21 | type PulsarMessages struct { 22 | Limit int `json:"limit"` 23 | Size int `json:"size"` 24 | Messages []PulsarMessage `json:"messages"` 25 | } 26 | 27 | // NewPulsarMessages create a PulsarMessages object 28 | func NewPulsarMessages(initSize int) PulsarMessages { 29 | return PulsarMessages{ 30 | Limit: initSize, 31 | Size: 0, 32 | Messages: make([]PulsarMessage, 0), 33 | } 34 | } 35 | 36 | // AddPulsarMessage adds a Pulsar Message to the payload, return true if reaches capacity 37 | func (msgs *PulsarMessages) AddPulsarMessage(msg pulsar.Message) bool { 38 | if msgs.Size >= msgs.Limit { 39 | return true 40 | } 41 | msgs.Messages = append(msgs.Messages, PulsarMessage{ 42 | Payload: msg.Payload(), 43 | Topic: msg.Topic(), 44 | EventTime: msg.EventTime(), 45 | PublishTime: msg.PublishTime(), 46 | MessageID: fmt.Sprintf("%+v", msg.ID()), 47 | Key: msg.Key(), 48 | }) 49 | msgs.Size++ 50 | 51 | return msgs.Size >= msgs.Limit 52 | } 53 | 54 | // IsEmpty checks if the message list is empty 55 | func (msgs *PulsarMessages) IsEmpty() bool { 56 | return msgs.Size == 0 57 | } 58 | -------------------------------------------------------------------------------- /src/model/topic.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "crypto/sha1" 5 | "encoding/hex" 6 | "errors" 7 | "fmt" 8 | "net/url" 9 | "regexp" 10 | "strings" 11 | "time" 12 | 13 | "github.com/apache/pulsar-client-go/pulsar" 14 | "github.com/kafkaesque-io/pulsar-beam/src/icrypto" 15 | ) 16 | 17 | // Status can be used for webhook status 18 | type Status int 19 | 20 | // state machine of webhook state 21 | const ( 22 | // Deactivated is the beginning state 23 | Deactivated Status = iota 24 | // Activated is the only active state 25 | Activated 26 | // Suspended is the state between Activated and Deleted 27 | Suspended 28 | // Deleted is the end of state 29 | Deleted 30 | ) 31 | 32 | // WebhookConfig - a configuration for webhook 33 | type WebhookConfig struct { 34 | URL string `json:"url"` 35 | Headers []string `json:"headers"` 36 | Subscription string `json:"subscription"` 37 | SubscriptionType string `json:"subscriptionType"` 38 | InitialPosition string `json:"initialPosition"` 39 | WebhookStatus Status `json:"webhookStatus"` 40 | CreatedAt time.Time `json:"createdAt"` 41 | UpdatedAt time.Time `json:"updatedAt"` 42 | DeletedAt time.Time `json:"deletedAt"` 43 | } 44 | 45 | //TODO add state of Webhook replies 46 | 47 | // TopicConfig - a configuraion for topic and its webhook configuration. 48 | type TopicConfig struct { 49 | TopicFullName string 50 | PulsarURL string 51 | Token string 52 | Tenant string 53 | Key string 54 | Notes string 55 | TopicStatus Status 56 | Webhooks []WebhookConfig 57 | CreatedAt time.Time 58 | UpdatedAt time.Time 59 | } 60 | 61 | // TopicKey represents a struct to identify a topic 62 | type TopicKey struct { 63 | TopicFullName string `json:"TopicFullName"` 64 | PulsarURL string `json:"PulsarURL"` 65 | } 66 | 67 | // 68 | const ( 69 | NonResumable = "NonResumable" 70 | ) 71 | 72 | // NewTopicConfig creates a topic configuration struct. 73 | func NewTopicConfig(topicFullName, pulsarURL, token string) (TopicConfig, error) { 74 | cfg := TopicConfig{} 75 | cfg.TopicFullName = topicFullName 76 | cfg.PulsarURL = pulsarURL 77 | cfg.Token = token 78 | cfg.Webhooks = make([]WebhookConfig, 0, 10) //Good to have a limit to budget threads 79 | 80 | var err error 81 | cfg.Key, err = GetKeyFromNames(topicFullName, pulsarURL) 82 | if err != nil { 83 | return cfg, err 84 | } 85 | cfg.CreatedAt = time.Now() 86 | cfg.UpdatedAt = time.Now() 87 | return cfg, nil 88 | } 89 | 90 | // NewWebhookConfig creates a new webhook config 91 | func NewWebhookConfig(URL string) WebhookConfig { 92 | cfg := WebhookConfig{} 93 | cfg.URL = URL 94 | cfg.Subscription = fmt.Sprintf("%s%s%d", NonResumable, icrypto.GenTopicKey(), time.Now().UnixNano()) 95 | cfg.WebhookStatus = Activated 96 | cfg.SubscriptionType = "exclusive" 97 | cfg.InitialPosition = "latest" 98 | cfg.CreatedAt = time.Now() 99 | cfg.UpdatedAt = time.Now() 100 | return cfg 101 | } 102 | 103 | // GetKeyFromNames generate topic key based on topic full name and pulsar url 104 | func GetKeyFromNames(topicFullName, pulsarURL string) (string, error) { 105 | url := strings.TrimSpace(pulsarURL) 106 | name := strings.TrimSpace(topicFullName) 107 | if url == "" || name == "" { 108 | return "", errors.New("missing PulsarURL or TopicFullName") 109 | } 110 | 111 | re := regexp.MustCompile(`^(pulsar|pulsar\+ssl)?:\/\/[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*(:[0-9]{0,6})?$`) 112 | if !re.MatchString(url) { 113 | return "", fmt.Errorf("incorrect pulsar url format %s", url) 114 | } 115 | return GenKey(name, url), nil 116 | } 117 | 118 | // GenKey generates a unique key based on pulsar url and topic full name 119 | func GenKey(topicFullName, pulsarURL string) string { 120 | h := sha1.New() 121 | h.Write([]byte(topicFullName + pulsarURL)) 122 | return hex.EncodeToString(h.Sum(nil)) 123 | } 124 | 125 | // GetInitialPosition returns the initial position for subscription 126 | func GetInitialPosition(pos string) (pulsar.SubscriptionInitialPosition, error) { 127 | switch strings.ToLower(pos) { 128 | case "latest", "": 129 | return pulsar.SubscriptionPositionLatest, nil 130 | case "earliest": 131 | return pulsar.SubscriptionPositionEarliest, nil 132 | default: 133 | return -1, fmt.Errorf("invalid subscription initial position %s", pos) 134 | } 135 | } 136 | 137 | // GetSubscriptionType converts string based subscription type to Pulsar subscription type 138 | func GetSubscriptionType(subType string) (pulsar.SubscriptionType, error) { 139 | switch strings.ToLower(subType) { 140 | case "exclusive", "": 141 | return pulsar.Exclusive, nil 142 | case "shared": 143 | return pulsar.Shared, nil 144 | case "keyshared": 145 | return pulsar.KeyShared, nil 146 | case "failover": 147 | return pulsar.Failover, nil 148 | default: 149 | return -1, fmt.Errorf("unsupported subscription type %s", subType) 150 | } 151 | } 152 | 153 | // ValidateWebhookConfig validates WebhookConfig object 154 | // I'd write explicit validation code rather than any off the shelf library, 155 | // which are just DSL and sometime these library just like fit square peg in a round hole. 156 | // Explicit validation has no dependency and very specific. 157 | func ValidateWebhookConfig(whs []WebhookConfig) error { 158 | // keeps track of exclusive subscription name 159 | exclusiveSubs := make(map[string]bool) 160 | for _, wh := range whs { 161 | if !isURL(wh.URL) { 162 | return fmt.Errorf("not a URL %s", wh.URL) 163 | } 164 | if strings.TrimSpace(wh.Subscription) == "" { 165 | return fmt.Errorf("subscription name is missing") 166 | } 167 | if subType, err := GetSubscriptionType(wh.SubscriptionType); err == nil { 168 | if subType == pulsar.Exclusive { 169 | if exclusiveSubs[wh.Subscription] { 170 | return fmt.Errorf("exclusive subscription %s cannot be shared between multiple webhooks", wh.Subscription) 171 | } 172 | exclusiveSubs[wh.Subscription] = true 173 | } 174 | } else { 175 | return err 176 | } 177 | if _, err := GetInitialPosition(wh.InitialPosition); err != nil { 178 | return err 179 | } 180 | } 181 | return nil 182 | 183 | } 184 | 185 | // ValidateTopicConfig validates the TopicConfig and returns the key to identify this topic 186 | func ValidateTopicConfig(top TopicConfig) (string, error) { 187 | if err := ValidateWebhookConfig(top.Webhooks); err != nil { 188 | return "", err 189 | } 190 | 191 | return GetKeyFromNames(top.TopicFullName, top.PulsarURL) 192 | } 193 | 194 | func isURL(str string) bool { 195 | u, err := url.Parse(str) 196 | return err == nil && u.Scheme != "" && u.Host != "" 197 | } 198 | -------------------------------------------------------------------------------- /src/pulsardriver/pulsar-client.go: -------------------------------------------------------------------------------- 1 | package pulsardriver 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | "github.com/apache/pulsar-client-go/pulsar" 11 | "github.com/kafkaesque-io/pulsar-beam/src/util" 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | // ClientCache caches a list Pulsar clients 16 | var ClientCache = make(map[string]*PulsarClient) 17 | 18 | // clientSync protects the ClientCache access 19 | var clientSync = &sync.RWMutex{} 20 | 21 | var ( 22 | clientOpsTimeout = util.GetEnvInt("PulsarClientOperationTimeout", 30) 23 | clientConnectTimeout = util.GetEnvInt("PulsarClientConnectionTimeout", 30) 24 | ) 25 | 26 | // GetPulsarClient gets a Pulsar client object 27 | func GetPulsarClient(pulsarURL, pulsarToken string, reset bool) (pulsar.Client, error) { 28 | key := pulsarURL + pulsarToken 29 | clientSync.Lock() 30 | driver, ok := ClientCache[key] 31 | if !ok { 32 | driver = &PulsarClient{} 33 | driver.createdAt = time.Now() 34 | driver.pulsarURL = pulsarURL 35 | driver.token = pulsarToken 36 | ClientCache[key] = driver 37 | } 38 | clientSync.Unlock() 39 | if reset { 40 | return driver.Reconnect() 41 | } 42 | return driver.GetClient(pulsarURL, pulsarToken) 43 | 44 | } 45 | 46 | // PulsarClient encapsulates the Pulsar Client object 47 | type PulsarClient struct { 48 | client pulsar.Client 49 | pulsarURL string 50 | token string 51 | createdAt time.Time 52 | lastUsed time.Time 53 | sync.Mutex 54 | } 55 | 56 | // GetClient acquires a new pulsar client 57 | func (c *PulsarClient) GetClient(url, tokenStr string) (pulsar.Client, error) { 58 | c.Lock() 59 | defer c.Unlock() 60 | 61 | if c.client != nil { 62 | return c.client, nil 63 | } 64 | 65 | driver, err := NewPulsarClient(url, tokenStr) 66 | if err != nil { 67 | log.Errorf("failed instantiate pulsar client %v", err) 68 | return nil, fmt.Errorf("Could not instantiate Pulsar client: %v", err) 69 | } 70 | if log.GetLevel() == log.DebugLevel { 71 | log.Debugf("pulsar client url %s\n token %s", url, tokenStr) 72 | } 73 | 74 | c.client = driver 75 | return driver, nil 76 | } 77 | 78 | // UpdateTime updates all time stamps in the object 79 | func (c *PulsarClient) UpdateTime() { 80 | c.lastUsed = time.Now() 81 | } 82 | 83 | // Close closes the Pulsar client 84 | func (c *PulsarClient) Close() { 85 | c.Lock() 86 | defer c.Unlock() 87 | if c.client != nil { 88 | c.client.Close() 89 | c.client = nil 90 | } 91 | } 92 | 93 | // Reconnect closes the current connection and reconnects again 94 | func (c *PulsarClient) Reconnect() (pulsar.Client, error) { 95 | c.Close() 96 | return c.GetClient(c.pulsarURL, c.token) 97 | } 98 | 99 | // NewPulsarClient always creates a new pulsar.Client connection 100 | func NewPulsarClient(url, tokenStr string) (pulsar.Client, error) { 101 | clientOpt := pulsar.ClientOptions{ 102 | URL: url, 103 | OperationTimeout: time.Duration(clientOpsTimeout) * time.Second, 104 | ConnectionTimeout: time.Duration(clientConnectTimeout) * time.Second, 105 | } 106 | 107 | if tokenStr != "" { 108 | clientOpt.Authentication = pulsar.NewAuthenticationToken(tokenStr) 109 | } 110 | 111 | if strings.HasPrefix(url, "pulsar+ssl://") { 112 | trustStore := os.Getenv("TrustStore") //"/etc/ssl/certs/ca-bundle.crt" all Config is also written back to OS ENV 113 | if trustStore == "" { 114 | return nil, fmt.Errorf("this is fatal that we are missing trustStore while pulsar+ssl is required") 115 | } 116 | clientOpt.TLSTrustCertsFilePath = trustStore 117 | } 118 | 119 | // default is false for these two configuration parameters 120 | clientOpt.TLSAllowInsecureConnection = util.StringToBool(os.Getenv("PulsarTLSAllowInsecureConnection")) 121 | clientOpt.TLSValidateHostname = util.StringToBool(os.Getenv("PulsarTLSValidateHostname")) 122 | 123 | driver, err := pulsar.NewClient(clientOpt) 124 | 125 | if err != nil { 126 | log.Errorf("failed instantiate pulsar client %v", err) 127 | return nil, fmt.Errorf("Could not instantiate Pulsar client: %v", err) 128 | } 129 | if log.GetLevel() == log.DebugLevel { 130 | log.Debugf("pulsar client url %s\n token %s", url, tokenStr) 131 | } 132 | 133 | return driver, nil 134 | } 135 | -------------------------------------------------------------------------------- /src/pulsardriver/pulsar-consumer.go: -------------------------------------------------------------------------------- 1 | package pulsardriver 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | "time" 7 | 8 | "github.com/apache/pulsar-client-go/pulsar" 9 | "github.com/kafkaesque-io/pulsar-beam/src/model" 10 | "github.com/kafkaesque-io/pulsar-beam/src/util" 11 | 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | // ConsumerCache caches a list Pulsar prudcers 16 | // key is a string concatenated with pulsar url, token, and topic full name 17 | var ConsumerCache = make(map[string]*PulsarConsumer) 18 | 19 | var consumerSync = &sync.RWMutex{} 20 | 21 | // GetPulsarConsumer gets a Pulsar consumer object 22 | func GetPulsarConsumer(pulsarURL, pulsarToken, topic, subName, subInitPos, subType, subKey string) (pulsar.Consumer, error) { 23 | key := subKey 24 | consumerSync.RLock() 25 | prod, ok := ConsumerCache[key] 26 | consumerSync.RUnlock() 27 | if !ok { 28 | prod = &PulsarConsumer{} 29 | prod.createdAt = time.Now() 30 | prod.pulsarURL = pulsarURL 31 | prod.token = pulsarToken 32 | prod.topic = topic 33 | prod.subscriptionName = subName 34 | var err error 35 | prod.subscriptionType, err = model.GetSubscriptionType(subType) 36 | if err != nil { 37 | return nil, err 38 | } 39 | prod.initPosition, err = model.GetInitialPosition(subInitPos) 40 | if err != nil { 41 | return nil, err 42 | } 43 | consumerSync.Lock() 44 | ConsumerCache[key] = prod 45 | consumerSync.Unlock() 46 | } 47 | p, err := prod.GetConsumer() 48 | if err != nil { 49 | // retry to close the client 50 | if _, err = GetPulsarClient(pulsarURL, pulsarToken, true); err != nil { 51 | return nil, err 52 | } 53 | if p, err = prod.GetConsumer(); err != nil { 54 | return nil, err 55 | } 56 | } 57 | return p, nil 58 | } 59 | 60 | // CancelPulsarConsumer closes Pulsar consumer and removes from the ConsumerCache 61 | func CancelPulsarConsumer(key string) { 62 | consumerSync.Lock() 63 | defer consumerSync.Unlock() 64 | c, ok := ConsumerCache[key] 65 | if ok { 66 | if strings.HasPrefix(c.consumer.Subscription(), model.NonResumable) { 67 | util.ReportError(c.consumer.Unsubscribe()) 68 | } 69 | c.Close() 70 | delete(ConsumerCache, key) 71 | } else { 72 | log.Errorf("cancel consumer failed to locate consumer key %v", key) 73 | } 74 | } 75 | 76 | // PulsarConsumer encapsulates the Pulsar Consumer object 77 | type PulsarConsumer struct { 78 | consumer pulsar.Consumer 79 | pulsarURL string 80 | token string 81 | topic string 82 | subscriptionName string 83 | subscriptionKey string 84 | initPosition pulsar.SubscriptionInitialPosition 85 | subscriptionType pulsar.SubscriptionType 86 | createdAt time.Time 87 | lastUsed time.Time 88 | sync.Mutex 89 | } 90 | 91 | // GetConsumer acquires a new pulsar consumer 92 | func (c *PulsarConsumer) GetConsumer() (pulsar.Consumer, error) { 93 | c.Lock() 94 | defer c.Unlock() 95 | 96 | if c.consumer != nil { 97 | return c.consumer, nil 98 | } 99 | 100 | driver, err := GetPulsarClient(c.pulsarURL, c.token, false) 101 | if err != nil { 102 | return nil, err 103 | } 104 | 105 | if log.GetLevel() == log.DebugLevel { 106 | log.Debugf("topic %s, subscriptionName %s\ninitPosition %v, subscriptionType %v\n", c.topic, c.subscriptionName, c.initPosition, c.subscriptionType) 107 | } 108 | c.consumer, err = driver.Subscribe(pulsar.ConsumerOptions{ 109 | Topic: c.topic, 110 | SubscriptionName: c.subscriptionName, 111 | SubscriptionInitialPosition: c.initPosition, 112 | Type: c.subscriptionType, 113 | }) 114 | if err != nil { 115 | log.Errorf("consumer subscribe error:%s\n", err.Error()) 116 | return nil, err 117 | } 118 | 119 | return c.consumer, nil 120 | } 121 | 122 | // UpdateTime updates all time stamps in the object 123 | func (c *PulsarConsumer) UpdateTime() { 124 | c.lastUsed = time.Now() 125 | } 126 | 127 | // Close closes the Pulsar client 128 | func (c *PulsarConsumer) Close() { 129 | c.Lock() 130 | defer c.Unlock() 131 | if c.consumer != nil { 132 | c.consumer.Close() 133 | c.consumer = nil 134 | } 135 | } 136 | 137 | // Reconnect closes the current connection and reconnects again 138 | func (c *PulsarConsumer) Reconnect() (pulsar.Consumer, error) { 139 | c.Close() 140 | return c.GetConsumer() 141 | } 142 | -------------------------------------------------------------------------------- /src/pulsardriver/pulsar-producer.go: -------------------------------------------------------------------------------- 1 | package pulsardriver 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strconv" 7 | "sync" 8 | "time" 9 | 10 | "github.com/apache/pulsar-client-go/pulsar" 11 | "github.com/kafkaesque-io/pulsar-beam/src/util" 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | var producerCacheTTL = util.GetEnvInt("ProducerCacheTTL", 900) 16 | 17 | // ProducerCache is the cache for Producer objects 18 | var ProducerCache = util.NewCache(util.CacheOption{ 19 | TTL: time.Duration(producerCacheTTL) * time.Second, 20 | CleanInterval: time.Duration(producerCacheTTL+2) * time.Second, 21 | ExpireCallback: func(key string, value interface{}) { 22 | if obj, ok := value.(*PulsarProducer); ok { 23 | obj.Close() 24 | } else { 25 | log.Errorf("wrong PulsarProducer object type stored in Cache") 26 | } 27 | }, 28 | }) 29 | 30 | // GetPulsarProducer gets a Pulsar producer object 31 | func GetPulsarProducer(pulsarURL, pulsarToken, topic string) (pulsar.Producer, error) { 32 | key := pulsarURL + pulsarToken + topic 33 | obj, exists := ProducerCache.Get(key) 34 | if exists { 35 | if driver, ok := obj.(*PulsarProducer); ok { 36 | return driver.GetProducer() 37 | } 38 | } 39 | prod := &PulsarProducer{ 40 | createdAt: time.Now(), 41 | pulsarURL: pulsarURL, 42 | token: pulsarToken, 43 | topic: topic, 44 | } 45 | p, err := prod.GetProducer() 46 | if err != nil { 47 | // retry to close the client 48 | if _, err = GetPulsarClient(pulsarURL, pulsarToken, true); err != nil { 49 | return nil, err 50 | } 51 | if p, err = prod.GetProducer(); err != nil { 52 | return nil, err 53 | } 54 | } 55 | ProducerCache.Set(key, prod) 56 | return p, nil 57 | } 58 | 59 | // PulsarProducer encapsulates the Pulsar Producer object 60 | type PulsarProducer struct { 61 | producer pulsar.Producer 62 | pulsarURL string 63 | token string 64 | topic string 65 | createdAt time.Time 66 | lastUsed time.Time 67 | sync.Mutex 68 | } 69 | 70 | // SendToPulsar sends data to a Pulsar producer. 71 | func SendToPulsar(url, token, topic string, data []byte, async bool) error { 72 | p, err := GetPulsarProducer(url, token, topic) 73 | if err != nil { 74 | log.Errorf("Failed to create Pulsar produce err: %v", err) 75 | return errors.New("Failed to create Pulsar producer") 76 | } 77 | 78 | ctx := context.Background() 79 | 80 | id, err := util.NewUUID() 81 | if err != nil { 82 | // this is very bad if happens 83 | log.Warnf("NewUUID generation error %v", err) 84 | id = strconv.FormatInt(time.Now().Unix(), 10) 85 | } 86 | prop := map[string]string{"PulsarBeamId": id} 87 | //TODO: add cluster origin and maybe other properties 88 | 89 | message := pulsar.ProducerMessage{ 90 | Payload: data, 91 | EventTime: time.Now(), 92 | Properties: prop, 93 | } 94 | 95 | if async { 96 | p.SendAsync(ctx, &message, func(messageId pulsar.MessageID, msg *pulsar.ProducerMessage, err error) { 97 | if err != nil { 98 | log.Warnf("send to Pulsar err %v", err) 99 | // TODO: push to a queue for retry 100 | } 101 | }) 102 | return nil 103 | } 104 | _, err = p.Send(ctx, &message) 105 | return err 106 | } 107 | 108 | // GetProducer acquires a new pulsar producer 109 | func (c *PulsarProducer) GetProducer() (pulsar.Producer, error) { 110 | c.Lock() 111 | defer c.Unlock() 112 | 113 | if c.producer != nil { 114 | return c.producer, nil 115 | } 116 | 117 | driver, err := GetPulsarClient(c.pulsarURL, c.token, false) 118 | if err != nil { 119 | return nil, err 120 | } 121 | p, err := driver.CreateProducer(pulsar.ProducerOptions{ 122 | Topic: c.topic, 123 | }) 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | c.producer = p 129 | return p, nil 130 | } 131 | 132 | // UpdateTime updates all time stamps in the object 133 | func (c *PulsarProducer) UpdateTime() { 134 | c.lastUsed = time.Now() 135 | } 136 | 137 | // Close closes the Pulsar client 138 | func (c *PulsarProducer) Close() { 139 | c.Lock() 140 | defer c.Unlock() 141 | if c.producer != nil { 142 | c.producer.Close() 143 | c.producer = nil 144 | } 145 | } 146 | 147 | // Reconnect closes the current connection and reconnects again 148 | func (c *PulsarProducer) Reconnect() (pulsar.Producer, error) { 149 | c.Close() 150 | return c.GetProducer() 151 | } 152 | -------------------------------------------------------------------------------- /src/route/error.go: -------------------------------------------------------------------------------- 1 | package route 2 | 3 | //This is a model for HTTP response 4 | 5 | // ResponseErr - Error struct for Http response 6 | type ResponseErr struct { 7 | Error string `json:"error"` 8 | } 9 | -------------------------------------------------------------------------------- /src/route/handlers.go: -------------------------------------------------------------------------------- 1 | package route 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | "strings" 12 | "compress/gzip" 13 | 14 | "github.com/apache/pulsar-client-go/pulsar" 15 | "github.com/gorilla/mux" 16 | "github.com/kafkaesque-io/pulsar-beam/src/broker" 17 | "github.com/kafkaesque-io/pulsar-beam/src/db" 18 | "github.com/kafkaesque-io/pulsar-beam/src/model" 19 | "github.com/kafkaesque-io/pulsar-beam/src/pulsardriver" 20 | "github.com/kafkaesque-io/pulsar-beam/src/util" 21 | 22 | log "github.com/sirupsen/logrus" 23 | ) 24 | 25 | var singleDb db.Db 26 | 27 | const subDelimiter = "-" 28 | 29 | // Init initializes database 30 | func Init() { 31 | singleDb = db.NewDbWithPanic(util.GetConfig().PbDbType) 32 | } 33 | 34 | // TokenServerResponse is the json object for token server response 35 | type TokenServerResponse struct { 36 | Subject string `json:"subject"` 37 | Token string `json:"token"` 38 | } 39 | 40 | // TokenSubjectHandler issues new token 41 | func TokenSubjectHandler(w http.ResponseWriter, r *http.Request) { 42 | vars := mux.Vars(r) 43 | subject, ok := vars["sub"] 44 | if !ok { 45 | w.WriteHeader(http.StatusUnprocessableEntity) 46 | return 47 | } 48 | 49 | if util.StrContains(util.SuperRoles, util.AssignString(r.Header.Get("injectedSubs"), "BOGUSROLE")) { 50 | tokenString, err := util.JWTAuth.GenerateToken(subject) 51 | if err != nil { 52 | util.ResponseErrorJSON(errors.New("failed to generate token"), w, http.StatusInternalServerError) 53 | } else { 54 | respJSON, err := json.Marshal(&TokenServerResponse{ 55 | Subject: subject, 56 | Token: tokenString, 57 | }) 58 | if err != nil { 59 | util.ResponseErrorJSON(errors.New("failed to marshal token response json object"), w, http.StatusInternalServerError) 60 | return 61 | } 62 | w.Write(respJSON) 63 | } 64 | return 65 | } 66 | util.ResponseErrorJSON(errors.New("incorrect subject"), w, http.StatusUnauthorized) 67 | return 68 | } 69 | 70 | // StatusPage replies with basic status code 71 | func StatusPage(w http.ResponseWriter, r *http.Request) { 72 | w.WriteHeader(http.StatusOK) 73 | return 74 | } 75 | 76 | // Full message structure that contains all request information 77 | type InfoRichMessage struct { 78 | Headers http.Header `json:"headers"` 79 | Body string `json:"body"` 80 | } 81 | 82 | // ReceiveHandler - the message receiver handler 83 | func ReceiveHandler(w http.ResponseWriter, r *http.Request) { 84 | var b []byte 85 | var err error 86 | if r.Header.Get("Content-Encoding") == "gzip" { 87 | g, gerr := gzip.NewReader(r.Body) 88 | if gerr != nil { 89 | util.ResponseErrorJSON(gerr, w, http.StatusInternalServerError) 90 | return 91 | } 92 | b, err = ioutil.ReadAll(g) 93 | } else { 94 | b, err = ioutil.ReadAll(r.Body) 95 | } 96 | defer r.Body.Close() 97 | if err != nil { 98 | util.ResponseErrorJSON(err, w, http.StatusInternalServerError) 99 | return 100 | } 101 | token, topic, pulsarURL, err := util.ReceiverHeader(util.AllowedPulsarURLs, &r.Header) 102 | if err != nil { 103 | util.ResponseErrorJSON(err, w, http.StatusUnauthorized) 104 | return 105 | } 106 | 107 | // Include headers information into the message payload if url has includeHeaders=true 108 | includeHeaders, isInfoRichMessage := r.URL.Query()["includeHeaders"] 109 | 110 | var infoRichMessage *InfoRichMessage 111 | 112 | if isInfoRichMessage && includeHeaders[0] != "false" { 113 | infoRichMessage = new(InfoRichMessage) 114 | infoRichMessage.Headers = r.Header 115 | infoRichMessage.Body = string(b) 116 | } 117 | 118 | if infoRichMessage != nil { 119 | b, _ = json.Marshal(infoRichMessage) 120 | } 121 | 122 | topicFN, err2 := GetTopicFnFromRoute(mux.Vars(r)) 123 | if topic == "" && err2 != nil { 124 | // only read topic from routes 125 | util.ResponseErrorJSON(err2, w, http.StatusUnprocessableEntity) 126 | return 127 | } 128 | topicFN = util.AssignString(topic, topicFN) // header topicFn overwrites topic specified in the routes 129 | log.Infof("topicFN %s pulsarURL %s", topicFN, pulsarURL) 130 | 131 | pulsarAsync := r.URL.Query().Get("mode") == "async" 132 | err = pulsardriver.SendToPulsar(pulsarURL, token, topicFN, b, pulsarAsync) 133 | if err != nil { 134 | util.ResponseErrorJSON(err, w, http.StatusServiceUnavailable) 135 | return 136 | } 137 | w.WriteHeader(http.StatusOK) 138 | return 139 | } 140 | 141 | // recoverHandler a function recovers from panic 142 | func recoverHandler(r *http.Request) { 143 | if r := recover(); r != nil { 144 | fmt.Printf("Recovered in http handler crash %v", r) 145 | } else { 146 | fmt.Printf("exit http handler") 147 | } 148 | } 149 | 150 | // PollHandler polls messages from a Pulsar topic. 151 | func PollHandler(w http.ResponseWriter, r *http.Request) { 152 | defer recoverHandler(r) 153 | 154 | u, _ := url.Parse(r.URL.String()) 155 | params := u.Query() 156 | token, topicFN, pulsarURL, subName, _, subType, err := ConsumerConfigFromHTTPParts(util.AllowedPulsarURLs, &r.Header, mux.Vars(r), params) 157 | if err != nil { 158 | util.ResponseErrorJSON(err, w, http.StatusUnprocessableEntity) 159 | return 160 | } 161 | w.Header().Set("Content-Type", "application/json; charset=UTF-8") 162 | 163 | size := util.QueryParamInt(params, "batchSize", 10) 164 | perMessageTimeoutMs := util.QueryParamInt(params, "perMessageTimeoutMs", 300) 165 | 166 | // subscription initial position is always set to earliest since this is short poll 167 | msgs, err := broker.PollBatchMessages(pulsarURL, token, topicFN, subName, subType, size, perMessageTimeoutMs) 168 | if err != nil { 169 | util.ResponseErrorJSON(err, w, http.StatusInternalServerError) 170 | return 171 | } 172 | 173 | if msgs.IsEmpty() { 174 | w.WriteHeader(http.StatusNoContent) 175 | return 176 | } 177 | 178 | data, err := json.Marshal(msgs) 179 | if err != nil { 180 | util.ResponseErrorJSON(err, w, http.StatusInternalServerError) 181 | return 182 | } 183 | w.WriteHeader(http.StatusOK) 184 | w.Write(data) 185 | } 186 | 187 | // SSEHandler is the HTTP SSE handler 188 | func SSEHandler(w http.ResponseWriter, r *http.Request) { 189 | defer recoverHandler(r) 190 | 191 | u, _ := url.Parse(r.URL.String()) 192 | params := u.Query() 193 | token, topicFN, pulsarURL, subName, subInitPos, subType, err := ConsumerConfigFromHTTPParts(util.AllowedPulsarURLs, &r.Header, mux.Vars(r), params) 194 | if err != nil { 195 | util.ResponseErrorJSON(err, w, http.StatusUnprocessableEntity) 196 | return 197 | } 198 | 199 | // Make sure that the writer supports flushing. 200 | flusher, ok := w.(http.Flusher) 201 | if !ok { 202 | http.Error(w, "Streaming unsupported", http.StatusInternalServerError) 203 | return 204 | } 205 | 206 | w.Header().Set("Content-Type", "text/event-stream") 207 | w.Header().Set("Cache-Control", "no-cache") 208 | w.Header().Set("Connection", "keep-alive") 209 | w.Header().Set("Access-Control-Allow-Origin", "*") // allow connection from different domain 210 | 211 | client, consumer, err := broker.GetPulsarClientConsumer(pulsarURL, token, topicFN, subName, subType, subInitPos) 212 | if err != nil { 213 | util.ResponseErrorJSON(err, w, http.StatusInternalServerError) 214 | return 215 | } 216 | defer client.Close() 217 | defer consumer.Close() 218 | if strings.HasPrefix(subName, model.NonResumable) { 219 | defer consumer.Unsubscribe() 220 | } 221 | 222 | consumChan := consumer.Chan() 223 | for { 224 | select { 225 | case msg := <-consumChan: 226 | // log.Infof("received message %s on topic %s", string(msg.Payload()), topicFN) 227 | consumer.Ack(msg) 228 | 229 | // ledgerId, entryId, batchId, partitionIndex, reserved, consumerId 230 | fmt.Fprintf(w, strings.Replace(fmt.Sprintf("id: %v\n", msg.Message.ID()), "&", "", 1)) 231 | fmt.Fprintf(w, "data: %s\n\n", msg.Payload()) 232 | flusher.Flush() 233 | case <-r.Context().Done(): 234 | return 235 | } 236 | } 237 | } 238 | 239 | // GetTopicHandler gets the topic details 240 | func GetTopicHandler(w http.ResponseWriter, r *http.Request) { 241 | topicKey, err := GetTopicKey(r) 242 | if err != nil { 243 | util.ResponseErrorJSON(err, w, http.StatusUnprocessableEntity) 244 | return 245 | } 246 | 247 | // TODO: we may fix the problem that allows negatively look up by another tenant 248 | doc, err := singleDb.GetByKey(topicKey) 249 | if err != nil { 250 | log.Errorf("get topic error %v", err) 251 | util.ResponseErrorJSON(err, w, http.StatusNotFound) 252 | return 253 | } 254 | if !VerifySubjectBasedOnTopic(doc.TopicFullName, r.Header.Get("injectedSubs"), ExtractEvalTenant) { 255 | w.WriteHeader(http.StatusForbidden) 256 | return 257 | } 258 | 259 | resJSON, err := json.Marshal(doc) 260 | if err != nil { 261 | util.ResponseErrorJSON(err, w, http.StatusInternalServerError) 262 | } else { 263 | w.Header().Set("Content-Type", "application/json; charset=UTF-8") 264 | w.Write(resJSON) 265 | } 266 | 267 | } 268 | 269 | // UpdateTopicHandler creates or updates a topic 270 | func UpdateTopicHandler(w http.ResponseWriter, r *http.Request) { 271 | decoder := json.NewDecoder(r.Body) 272 | defer r.Body.Close() 273 | w.Header().Set("Content-Type", "application/json; charset=UTF-8") 274 | 275 | var doc model.TopicConfig 276 | err := decoder.Decode(&doc) 277 | if err != nil { 278 | util.ResponseErrorJSON(err, w, http.StatusUnprocessableEntity) 279 | return 280 | } 281 | 282 | if _, err = model.ValidateTopicConfig(doc); err != nil { 283 | util.ResponseErrorJSON(err, w, http.StatusUnprocessableEntity) 284 | return 285 | } 286 | 287 | if !VerifySubjectBasedOnTopic(doc.TopicFullName, r.Header.Get("injectedSubs"), ExtractEvalTenant) { 288 | w.WriteHeader(http.StatusForbidden) 289 | return 290 | } 291 | 292 | id, err := singleDb.Update(&doc) 293 | if err != nil { 294 | util.ResponseErrorJSON(err, w, http.StatusConflict) 295 | return 296 | } 297 | if len(id) > 1 { 298 | savedDoc, err := singleDb.GetByKey(id) 299 | if err != nil { 300 | util.ResponseErrorJSON(err, w, http.StatusInternalServerError) 301 | return 302 | } 303 | w.WriteHeader(http.StatusCreated) 304 | resJSON, err := json.Marshal(savedDoc) 305 | if err != nil { 306 | util.ResponseErrorJSON(err, w, http.StatusInternalServerError) 307 | return 308 | } 309 | w.Write(resJSON) 310 | return 311 | } 312 | util.ResponseErrorJSON(fmt.Errorf("failed to update"), w, http.StatusInternalServerError) 313 | return 314 | } 315 | 316 | // DeleteTopicHandler deletes a topic 317 | func DeleteTopicHandler(w http.ResponseWriter, r *http.Request) { 318 | topicKey, err := GetTopicKey(r) 319 | if err != nil { 320 | util.ResponseErrorJSON(err, w, http.StatusUnprocessableEntity) 321 | return 322 | } 323 | 324 | doc, err := singleDb.GetByKey(topicKey) 325 | if err != nil { 326 | log.Errorf("failed to get topic based on key %s err: %v", topicKey, err) 327 | util.ResponseErrorJSON(err, w, http.StatusNotFound) 328 | return 329 | } 330 | if !VerifySubjectBasedOnTopic(doc.TopicFullName, r.Header.Get("injectedSubs"), ExtractEvalTenant) { 331 | w.WriteHeader(http.StatusForbidden) 332 | return 333 | } 334 | 335 | deletedKey, err := singleDb.DeleteByKey(topicKey) 336 | if err != nil { 337 | util.ResponseErrorJSON(err, w, http.StatusNotFound) 338 | return 339 | } 340 | resJSON, err := json.Marshal(deletedKey) 341 | if err != nil { 342 | w.WriteHeader(http.StatusInternalServerError) 343 | } else { 344 | w.Header().Set("Content-Type", "application/json; charset=UTF-8") 345 | w.Write(resJSON) 346 | } 347 | } 348 | 349 | // GetTopicKey gets the topic key from the request body or url sub route 350 | func GetTopicKey(r *http.Request) (string, error) { 351 | var err error 352 | vars := mux.Vars(r) 353 | topicKey, ok := vars["topicKey"] 354 | if !ok { 355 | var topic model.TopicKey 356 | decoder := json.NewDecoder(r.Body) 357 | defer r.Body.Close() 358 | 359 | err := decoder.Decode(&topic) 360 | switch { 361 | case err == io.EOF: 362 | return "", errors.New("missing topic key or topic names in body") 363 | case err != nil: 364 | return "", err 365 | } 366 | topicKey, err = model.GetKeyFromNames(topic.TopicFullName, topic.PulsarURL) 367 | if err != nil { 368 | return "", err 369 | } 370 | } 371 | return topicKey, err 372 | } 373 | 374 | // VerifySubjectBasedOnTopic verifies the subject can meet the requirement. 375 | func VerifySubjectBasedOnTopic(topicFN, tokenSub string, evalTenant func(tenant, subjects string) bool) bool { 376 | parts := strings.Split(topicFN, "/") 377 | if len(parts) < 4 { 378 | return false 379 | } 380 | tenant := parts[2] 381 | if len(tenant) < 1 { 382 | log.Infof(" auth verify tenant %s token sub %s", tenant, tokenSub) 383 | return false 384 | } 385 | return VerifySubject(tenant, tokenSub, evalTenant) 386 | } 387 | 388 | // VerifySubject verifies the subject can meet the requirement. 389 | // Subject verification requires role or tenant name in the jwt subject 390 | func VerifySubject(requiredSubject, tokenSubjects string, evalTenant func(tenant, subjects string) bool) bool { 391 | for _, v := range strings.Split(tokenSubjects, ",") { 392 | if util.StrContains(util.SuperRoles, v) { 393 | return true 394 | } 395 | if requiredSubject == v { 396 | return true 397 | } 398 | if evalTenant(requiredSubject, v) { 399 | return true 400 | } 401 | } 402 | 403 | return false 404 | } 405 | 406 | // ExtractEvalTenant is a customized function to evaluate subject against tenant 407 | func ExtractEvalTenant(requiredSubject, tokenSub string) bool { 408 | // expect - in subject unless it is superuser 409 | var sub string 410 | parts := strings.Split(tokenSub, subDelimiter) 411 | if len(parts) < 2 { 412 | sub = parts[0] 413 | } 414 | 415 | validLength := len(parts) - 1 416 | sub = strings.Join(parts[:validLength], subDelimiter) 417 | if sub != "" && requiredSubject == sub { 418 | return true 419 | } 420 | return false 421 | } 422 | 423 | // GetTopicFnFromRoute builds a valida topic fullname from the http route 424 | func GetTopicFnFromRoute(vars map[string]string) (string, error) { 425 | tenant, ok := vars["tenant"] 426 | namespace, ok2 := vars["namespace"] 427 | topic, ok3 := vars["topic"] 428 | persistent, ok4 := vars["persistent"] 429 | if !(ok && ok2 && ok3 && ok4) { 430 | return "", fmt.Errorf("missing topic parts") 431 | } 432 | topicFn, err := util.BuildTopicFn(persistent, tenant, namespace, topic) 433 | if err != nil { 434 | return "", err 435 | } 436 | return topicFn, nil 437 | } 438 | 439 | // ConsumerParams returns a configuration parameters for Pulsar consumer 440 | func ConsumerParams(params url.Values) (subName string, subInitPos pulsar.SubscriptionInitialPosition, subType pulsar.SubscriptionType, err error) { 441 | subType, err = model.GetSubscriptionType(util.QueryParamString(params, "SubscriptionType", "exclusive")) 442 | if err != nil { 443 | return "", -1, -1, err 444 | } 445 | subInitPos, err = model.GetInitialPosition(util.QueryParamString(params, "SubscriptionInitialPosition", "latest")) 446 | if err != nil { 447 | return "", -1, -1, err 448 | } 449 | 450 | subName = util.QueryParamString(params, "SubscriptionName", "") 451 | if len(subName) == 0 { 452 | name, err := util.NewUUID() 453 | if err != nil { 454 | return "", -1, -1, fmt.Errorf("failed to generate uuid error %v", err) 455 | } 456 | return model.NonResumable + name, subInitPos, subType, nil 457 | } else if len(subName) < 5 { 458 | return "", -1, -1, fmt.Errorf("subscription name must be more than 4 characters") 459 | } 460 | return subName, subInitPos, subType, nil 461 | } 462 | 463 | // ConsumerConfigFromHTTPParts returns configuration parameters required to generate Pulsar Client and Consumer 464 | func ConsumerConfigFromHTTPParts(allowedClusters []string, h *http.Header, vars map[string]string, params url.Values) (token, topicFN, pulsarURL, subName string, subInitPos pulsar.SubscriptionInitialPosition, subType pulsar.SubscriptionType, err error) { 465 | token, _, pulsarURL, err = util.ReceiverHeader(allowedClusters, h) 466 | if err != nil { 467 | return "", "", "", "", -1, -1, err 468 | } 469 | 470 | topicFN, err = GetTopicFnFromRoute(vars) 471 | if err != nil { 472 | return "", "", "", "", -1, -1, err 473 | } 474 | 475 | subName, subInitPos, subType, err = ConsumerParams(params) 476 | if err != nil { 477 | return "", "", "", "", -1, -1, err 478 | } 479 | 480 | return token, topicFN, pulsarURL, subName, subInitPos, subType, nil 481 | } 482 | -------------------------------------------------------------------------------- /src/route/logger.go: -------------------------------------------------------------------------------- 1 | package route 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | // Logger logs http traffic. 10 | func Logger(inner http.Handler, name string) http.Handler { 11 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 12 | start := time.Now() 13 | 14 | inner.ServeHTTP(w, r) 15 | 16 | log.Printf( 17 | "%s\t%s\t%s\t%s", 18 | r.Method, 19 | r.RequestURI, 20 | name, 21 | time.Since(start), 22 | ) 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /src/route/router.go: -------------------------------------------------------------------------------- 1 | package route 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gorilla/mux" 7 | 8 | "github.com/kafkaesque-io/pulsar-beam/src/middleware" 9 | "github.com/kafkaesque-io/pulsar-beam/src/util" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // NewRouter - create new router for HTTP routing 14 | func NewRouter(mode *string) *mux.Router { 15 | 16 | router := mux.NewRouter().StrictSlash(true) 17 | for _, route := range GetEffectiveRoutes(mode) { 18 | var handler http.Handler 19 | 20 | handler = route.HandlerFunc 21 | handler = Logger(handler, route.Name) 22 | 23 | router. 24 | Methods(route.Method). 25 | Path(route.Pattern). 26 | Name(route.Name). 27 | Handler(route.AuthFunc(handler)) 28 | 29 | } 30 | // TODO rate limit can be added per route basis 31 | router.Use(middleware.LimitRate) 32 | 33 | log.Infof("router added") 34 | return router 35 | } 36 | 37 | // GetEffectiveRoutes gets effective routes 38 | func GetEffectiveRoutes(mode *string) Routes { 39 | return append(PprofRoute, append(PrometheusRoute, getRoutes(mode)...)...) 40 | } 41 | 42 | func getRoutes(mode *string) Routes { 43 | switch *mode { 44 | case util.Hybrid: 45 | return append(ReceiverRoutes, RestRoutes...) 46 | case util.Receiver: 47 | return ReceiverRoutes 48 | case util.HTTPOnly: 49 | return append(ReceiverRoutes, append(RestRoutes, TokenServerRoutes...)...) 50 | case util.TokenServer: 51 | return TokenServerRoutes 52 | case util.HTTPWithNoRest: 53 | return append(ReceiverRoutes, TokenServerRoutes...) 54 | default: 55 | return RestRoutes 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/route/router_test.go: -------------------------------------------------------------------------------- 1 | package route 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestEffectiveRoutes(t *testing.T) { 10 | receiverRoutesLen := len(ReceiverRoutes) 11 | restRoutesLen := len(RestRoutes) 12 | prometheusLen := len(PrometheusRoute) 13 | // mode := "hybrid" 14 | // assert.Equal(t, len(GetEffectiveRoutes(&mode)), (receiverRoutesLen + restRoutesLen + prometheusLen)) 15 | mode := "rest" 16 | assert.Equal(t, len(GetEffectiveRoutes(&mode)), (receiverRoutesLen + restRoutesLen + prometheusLen)) 17 | mode = "receiver" 18 | assert.Equal(t, len(GetEffectiveRoutes(&mode)), (receiverRoutesLen + restRoutesLen + prometheusLen)) 19 | } 20 | -------------------------------------------------------------------------------- /src/route/routes.go: -------------------------------------------------------------------------------- 1 | package route 2 | 3 | import ( 4 | "net/http" 5 | "net/http/pprof" 6 | 7 | "github.com/gorilla/mux" 8 | "github.com/kafkaesque-io/pulsar-beam/src/middleware" 9 | "github.com/prometheus/client_golang/prometheus/promhttp" 10 | ) 11 | 12 | // Route - HTTP Route 13 | type Route struct { 14 | Name string 15 | Method string 16 | Pattern string 17 | HandlerFunc http.HandlerFunc 18 | AuthFunc mux.MiddlewareFunc 19 | } 20 | 21 | // Routes list of HTTP Routes 22 | type Routes []Route 23 | 24 | // TokenServerRoutes definition 25 | var TokenServerRoutes = Routes{ 26 | Route{ 27 | "token server", 28 | http.MethodGet, 29 | "/subject/{sub}", 30 | TokenSubjectHandler, 31 | middleware.AuthVerifyJWT, 32 | }, 33 | } 34 | 35 | // PrometheusRoute definition 36 | var PrometheusRoute = Routes{ 37 | Route{ 38 | "Prometeus metrics", 39 | http.MethodGet, 40 | "/metrics", 41 | promhttp.Handler().ServeHTTP, 42 | middleware.NoAuth, 43 | }, 44 | } 45 | 46 | var PprofRoute = Routes{ 47 | Route{ 48 | "Pprof Index", 49 | http.MethodGet, 50 | "/debug/pprof/", 51 | pprof.Index, 52 | middleware.NoAuth, 53 | }, 54 | Route{ 55 | "Pprof Cmdline", 56 | http.MethodGet, 57 | "/debug/pprof/cmdline", 58 | pprof.Cmdline, 59 | middleware.NoAuth, 60 | }, 61 | Route{ 62 | "Pprof Profile", 63 | http.MethodGet, 64 | "/debug/pprof/profile", 65 | pprof.Profile, 66 | middleware.NoAuth, 67 | }, 68 | Route{ 69 | "Pprof Symbol", 70 | http.MethodGet, 71 | "/debug/pprof/symbol", 72 | pprof.Symbol, 73 | middleware.NoAuth, 74 | }, 75 | Route{ 76 | "Pprof Trace", 77 | http.MethodGet, 78 | "/debug/pprof/trace", 79 | pprof.Trace, 80 | middleware.NoAuth, 81 | }, 82 | } 83 | 84 | // ReceiverRoutes definition 85 | var ReceiverRoutes = Routes{ 86 | Route{ 87 | "status", 88 | "GET", 89 | "/status", 90 | StatusPage, 91 | middleware.AuthHeaderRequired, 92 | }, 93 | Route{ 94 | "Receive", 95 | "POST", 96 | "/v1/firehose", 97 | ReceiveHandler, 98 | middleware.NoAuth, 99 | }, 100 | Route{ 101 | "Receive", 102 | "POST", 103 | "/v2/firehose/{persistent}/{tenant}/{namespace}/{topic}", 104 | ReceiveHandler, 105 | middleware.AuthVerifyJWT, 106 | }, 107 | Route{ 108 | "http-sse", 109 | "GET", 110 | "/v2/sse/{persistent}/{tenant}/{namespace}/{topic}", 111 | SSEHandler, 112 | middleware.AuthVerifyJWT, 113 | }, 114 | Route{ 115 | "poll-messages", 116 | http.MethodGet, 117 | "/v2/poll/{persistent}/{tenant}/{namespace}/{topic}", 118 | PollHandler, 119 | middleware.AuthVerifyJWT, 120 | }, 121 | } 122 | 123 | // RestRoutes definition 124 | var RestRoutes = Routes{ 125 | Route{ 126 | "Get a topic with key", 127 | "GET", 128 | "/v2/topic/{topicKey}", 129 | GetTopicHandler, 130 | middleware.AuthVerifyJWT, 131 | }, 132 | Route{ 133 | "Get a topic", 134 | "GET", 135 | "/v2/topic", 136 | GetTopicHandler, 137 | middleware.AuthVerifyJWT, 138 | }, 139 | Route{ 140 | "Update a topic", 141 | "POST", 142 | "/v2/topic", 143 | UpdateTopicHandler, 144 | middleware.AuthVerifyJWT, 145 | }, 146 | Route{ 147 | "Delete a topic with key", 148 | "DELETE", 149 | "/v2/topic/{topicKey}", 150 | DeleteTopicHandler, 151 | middleware.AuthVerifyJWT, 152 | }, 153 | Route{ 154 | "Delete a topic", 155 | "DELETE", 156 | "/v2/topic", 157 | DeleteTopicHandler, 158 | middleware.AuthVerifyJWT, 159 | }, 160 | } 161 | -------------------------------------------------------------------------------- /src/unit-test/crypto_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | . "github.com/kafkaesque-io/pulsar-beam/src/icrypto" 9 | ) 10 | 11 | func TestAES(t *testing.T) { 12 | //https://golang.org/src/crypto/aes/cipher.go 13 | Key := "test1234test1234" //only 16/24/32 key length is supported 14 | e := AES{DefaultSalt: Key} 15 | var s string 16 | s = "this is a test string" 17 | text := []byte(s) 18 | key := []byte(Key) 19 | 20 | ciphertext, err := e.Encrypt(text, key) 21 | errNil(t, err) 22 | 23 | decryptedText, err := e.Decrypt(ciphertext, key) 24 | errNil(t, err) 25 | equals(t, text, decryptedText) 26 | } 27 | 28 | func TestRSA(t *testing.T) { 29 | //https://golang.org/src/crypto/rsa/rsa_test.go 30 | rsa, err := NewRSA() 31 | errNil(t, err) 32 | var s string 33 | s = "this is a test string" 34 | text := []byte(s) 35 | 36 | ciphertext, err := rsa.EncryptWithDefaultKey(text) 37 | errNil(t, err) 38 | 39 | decryptedText, err := rsa.DecryptWithDefaultKey(ciphertext) 40 | errNil(t, err) 41 | equals(t, text, decryptedText) 42 | 43 | bytes, err := rsa.GetPrivateKey() 44 | pstr := string(bytes) 45 | errNil(t, err) 46 | assert(t, len(pstr) > 512, "min length") 47 | 48 | //test to be encrypted by another object 49 | _, err = rsa.GetPrivateKey() 50 | errNil(t, err) 51 | 52 | pubKey, err := rsa.GetPublicKey() 53 | errNil(t, err) 54 | 55 | rsaDe, err := NewRSAWithKeys(nil, pubKey) 56 | errNil(t, err) 57 | 58 | text2 := []byte("another test to encrypted by one obj then decrypted by the original obj") 59 | ciphertext2, err := rsaDe.EncryptWithDefaultKey(text2) 60 | errNil(t, err) 61 | 62 | decrypted2, err := rsa.DecryptWithDefaultKey(ciphertext2) 63 | errNil(t, err) 64 | equals(t, text2, decrypted2) 65 | 66 | // test unsupported functions 67 | _, err = rsa.Decrypt([]byte{}, []byte{}) 68 | equals(t, err.Error(), "unsupported") 69 | 70 | _, err = rsa.Encrypt([]byte{}, []byte{}) 71 | equals(t, err.Error(), "unsupported") 72 | 73 | } 74 | 75 | func TestController64EncodeWithEncryption(t *testing.T) { 76 | s := "mockpassword1234" 77 | en, err := EncryptWithBase64(s) 78 | errNil(t, err) 79 | 80 | de, err2 := DecryptWithBase64(en) 81 | errNil(t, err2) 82 | equals(t, s, de) 83 | 84 | // failure cases 85 | _, err2 = DecryptWithBase64(fmt.Sprintf("%sy", en)) 86 | assert(t, err2 != nil, "expect error with incorrect decryption") 87 | } 88 | 89 | func TestGenWriteKey(t *testing.T) { 90 | // Create 1 million to make sure no duplicates 91 | size := 100000 92 | set := make(map[string]bool) 93 | for i := 0; i < size; i++ { 94 | id := GenTopicKey() 95 | set[id] = true 96 | } 97 | assert(t, size == len(set), "WriteKey duplicates found") 98 | 99 | } 100 | 101 | func TestJWTRSASignAndVerifyWithPEMKey(t *testing.T) { 102 | privateKeyPath := "./example_private_key" 103 | publicKeyPath := "./example_public_key.pub" 104 | authen := NewRSAKeyPair(privateKeyPath, publicKeyPath) 105 | 106 | tokenString, err := authen.GenerateToken("myadmin") 107 | errNil(t, err) 108 | assert(t, len(tokenString) > 1, "a token string can be generated") 109 | 110 | token, err0 := authen.DecodeToken(tokenString) 111 | errNil(t, err0) 112 | assert(t, token.Valid, "validate a valid token") 113 | 114 | valid, _ := authen.VerifyTokenSubject("bogustokenstr", "myadmin") 115 | assert(t, valid == false, "validate token fails test") 116 | 117 | valid, _ = authen.VerifyTokenSubject(tokenString, "myadmin") 118 | assert(t, valid, "validate token's expected subject") 119 | 120 | valid, _ = authen.VerifyTokenSubject(tokenString, "admin") 121 | assert(t, valid == false, "validate token's mismatched subject") 122 | 123 | pulsarGeneratedToken := "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJwaWNhc3NvIn0.TZilYXJOeeCLwNOHICCYyFxUlwOLxa_kzVjKcoQRTJm2xqmNzTn-s9zjbuaNMCDj1U7gRPHKHkWNDb2W4MwQd6Nkc543E_cIHlJG82eKKIsGfAEQpnPJLpzz2zytgmRON6HCPDsQDAKIXHriKmbmCzHLOILziks0oOCadBGC79iddb9DjPku6sU0nByS8r8_oIrRCqV_cNsH1MInA6CRNYkPJaJI0T8i77ND7azTXwH0FTX_KE_yRmOkXnejJ14GEEcBM99dPGg8jCp-zOyfvrMIJjWsWzjXYExxjKaC85779ciu59YO3cXd0Lk2LzlyB4kDKZgPyqOgyQFIfQ1eiA" // pragma: allowlist secret 124 | valid, _ = authen.VerifyTokenSubject(pulsarGeneratedToken, "picasso") 125 | assert(t, valid, "validate pulsar generated token and subject") 126 | 127 | subjects, err := authen.GetTokenSubject(pulsarGeneratedToken) 128 | errNil(t, err) 129 | equals(t, subjects, "picasso") 130 | 131 | t2 := time.Now().Add(time.Hour * 1) 132 | expireOffset := authen.GetTokenRemainingValidity(t2) 133 | equals(t, expireOffset, 3600) 134 | 135 | } 136 | 137 | func TestJWTRSASignAndVerifyWithPKCS12Keys(t *testing.T) { 138 | privateKeyPath := "./pk12-binary-private.key" 139 | publicKeyPath := "./pk12-binary-public.key" 140 | authen := NewRSAKeyPair(privateKeyPath, publicKeyPath) 141 | 142 | tokenString, err := authen.GenerateToken("myadmin") 143 | errNil(t, err) 144 | assert(t, len(tokenString) > 1, "a token string can be generated") 145 | 146 | token, err0 := authen.DecodeToken(tokenString) 147 | errNil(t, err0) 148 | assert(t, token.Valid, "validate a valid token") 149 | 150 | valid, _ := authen.VerifyTokenSubject("bogustokenstr", "myadmin") 151 | assert(t, valid == false, "validate token fails test") 152 | 153 | valid, _ = authen.VerifyTokenSubject(tokenString, "myadmin") 154 | assert(t, valid, "validate token's expected subject") 155 | 156 | valid, _ = authen.VerifyTokenSubject(tokenString, "admin") 157 | assert(t, valid == false, "validate token's mismatched subject") 158 | 159 | pulsarGeneratedToken := "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJwaWNhc3NvIn0.TZilYXJOeeCLwNOHICCYyFxUlwOLxa_kzVjKcoQRTJm2xqmNzTn-s9zjbuaNMCDj1U7gRPHKHkWNDb2W4MwQd6Nkc543E_cIHlJG82eKKIsGfAEQpnPJLpzz2zytgmRON6HCPDsQDAKIXHriKmbmCzHLOILziks0oOCadBGC79iddb9DjPku6sU0nByS8r8_oIrRCqV_cNsH1MInA6CRNYkPJaJI0T8i77ND7azTXwH0FTX_KE_yRmOkXnejJ14GEEcBM99dPGg8jCp-zOyfvrMIJjWsWzjXYExxjKaC85779ciu59YO3cXd0Lk2LzlyB4kDKZgPyqOgyQFIfQ1eiA" // pragma: allowlist secret 160 | valid, _ = authen.VerifyTokenSubject(pulsarGeneratedToken, "picasso") 161 | assert(t, valid, "validate pulsar generated token and subject") 162 | 163 | subjects, err := authen.GetTokenSubject(pulsarGeneratedToken) 164 | errNil(t, err) 165 | equals(t, subjects, "picasso") 166 | 167 | t2 := time.Now().Add(time.Hour * 1) 168 | expireOffset := authen.GetTokenRemainingValidity(t2) 169 | equals(t, expireOffset, 3600) 170 | 171 | } 172 | -------------------------------------------------------------------------------- /src/unit-test/db_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | . "github.com/kafkaesque-io/pulsar-beam/src/db" 10 | "github.com/kafkaesque-io/pulsar-beam/src/model" 11 | "github.com/kafkaesque-io/pulsar-beam/src/util" 12 | ) 13 | 14 | func TestUnsupportedDbDriver(t *testing.T) { 15 | _, err := NewDb("cockroach") 16 | equals(t, err.Error(), "unsupported db type") 17 | } 18 | 19 | func TestInMemoryDatabase(t *testing.T) { 20 | // a test case 1) connect to a local mongodb 21 | // 2) test with ping 22 | // 3) create a document 23 | // 4) create another document with the same key; expected to fail 24 | // 5) update a document 25 | // 6) load/retrieve all documents, iterate to find a document 26 | // 7) delete a document 27 | // 8) get a document to ensure it's deleted 28 | inmemorydb, err := NewInMemoryHandler() 29 | errNil(t, err) 30 | 31 | err = inmemorydb.Sync() 32 | errNil(t, err) 33 | status := inmemorydb.Health() 34 | equals(t, status, true) 35 | 36 | docs, err := inmemorydb.Load() 37 | errNil(t, err) 38 | equals(t, 0, len(docs)) 39 | 40 | topicFullName := "persistent://mytenant/local-useast1-gcp/yet-another-test-topic" 41 | token := "eyJhbGciOiJSUzI1NiJ9somecrazytokenstring" 42 | pulsarURL := "pulsar+ssl://useast1.gcp.kafkaesque.io:6651" 43 | 44 | // test incorrect arity order 45 | topic, err := model.NewTopicConfig(pulsarURL, topicFullName, token) 46 | equals(t, err != nil, true) 47 | 48 | // test correct arity for topic 49 | topic, err = model.NewTopicConfig(topicFullName, pulsarURL, token) 50 | errNil(t, err) 51 | 52 | wh := model.NewWebhookConfig("http://localhost:8089") 53 | equals(t, wh.URL, "http://localhost:8089") 54 | assert(t, strings.HasPrefix(wh.Subscription, model.NonResumable), "ensure non resumable subscription") 55 | wh.Subscription = "firstsubscription" 56 | equals(t, wh.Subscription, "firstsubscription") 57 | equals(t, wh.WebhookStatus, model.Activated) 58 | headers := []string{ 59 | "Authorization: Bearer anothertoken", 60 | "Content-type: application/json", 61 | } 62 | wh.Headers = headers 63 | topic.Webhooks = append(topic.Webhooks, wh) 64 | equals(t, len(topic.Webhooks), 1) 65 | equals(t, cap(topic.Webhooks), 10) 66 | 67 | _, err = inmemorydb.Create(&topic) 68 | errNil(t, err) 69 | 70 | _, err = inmemorydb.Create(&topic) 71 | equals(t, err != nil, true) 72 | 73 | var key string 74 | key, err = inmemorydb.Update(&topic) 75 | errNil(t, err) 76 | equals(t, key != "", true) 77 | 78 | res, err := inmemorydb.Load() 79 | if err != nil { 80 | log.Fatal(err) 81 | } 82 | found := false 83 | for _, v := range res { 84 | if v.Key == key { 85 | found = true 86 | } 87 | } 88 | equals(t, found, true) 89 | 90 | resTopic, err := inmemorydb.GetByTopic(topic.TopicFullName, topic.PulsarURL) 91 | errNil(t, err) 92 | equals(t, topic.Token, resTopic.Token) 93 | equals(t, topic.PulsarURL, resTopic.PulsarURL) 94 | 95 | deletedKey, err := inmemorydb.Delete(topic.TopicFullName, topic.PulsarURL) 96 | errNil(t, err) 97 | equals(t, deletedKey, key) 98 | 99 | _, err = inmemorydb.GetByKey(resTopic.Key) 100 | assert(t, err != nil, "already deleted so returns error") 101 | equals(t, err.Error(), DocNotFound) 102 | // TODO: find a place to test Close(); need to find out dependencies. 103 | // Comment out because there are other test cases require database. 104 | errNil(t, inmemorydb.Close()) 105 | } 106 | 107 | func TestPulsarDbDriver(t *testing.T) { 108 | util.Config.DbConnectionStr = os.Getenv("PULSAR_URI") 109 | util.Config.DbName = os.Getenv("REST_DB_TABLE_TOPIC") 110 | util.Config.DbPassword = os.Getenv("PULSAR_TOKEN") 111 | util.Config.PbDbType = "pulsarAsDb" 112 | util.Config.TrustStore = os.Getenv("TrustStore") 113 | if util.GetConfig().DbPassword == "" { 114 | util.ReadConfigFile("../" + util.DefaultConfigFile) 115 | return 116 | } 117 | 118 | // a test case 1) connect to a local mongodb 119 | // 2) test with ping 120 | // 3) create a document 121 | // 4) create another document with the same key; expected to fail 122 | // 5) update a document 123 | // 6) load/retrieve all documents, iterate to find a document 124 | // 7) delete a document 125 | // 8) get a document to ensure it's deleted 126 | pulsardb, err := NewPulsarHandler() 127 | errNil(t, err) 128 | 129 | status := pulsardb.Health() 130 | equals(t, status, true) 131 | 132 | docs, err := pulsardb.Load() 133 | errNil(t, err) 134 | equals(t, 0, len(docs)) 135 | 136 | topicFullName := "persistent://mytenant/local-useast1-gcp/yet-another-test-topic" 137 | token := "eyJhbGciOiJSUzI1NiJ9somecrazytokenstring" 138 | pulsarURL := "pulsar+ssl://useast1.gcp.kafkaesque.io:6651" 139 | 140 | // test incorrect arity order 141 | topic, err := model.NewTopicConfig(pulsarURL, topicFullName, token) 142 | equals(t, err != nil, true) 143 | 144 | // test correct arity for topic 145 | topic, err = model.NewTopicConfig(topicFullName, pulsarURL, token) 146 | errNil(t, err) 147 | 148 | wh := model.NewWebhookConfig("http://localhost:8089") 149 | equals(t, wh.URL, "http://localhost:8089") 150 | assert(t, strings.HasPrefix(wh.Subscription, model.NonResumable), "ensure non resumable subscription") 151 | wh.Subscription = "firstsubscription" 152 | equals(t, wh.Subscription, "firstsubscription") 153 | equals(t, wh.WebhookStatus, model.Activated) 154 | headers := []string{ 155 | "Authorization: Bearer anothertoken", 156 | "Content-type: application/json", 157 | } 158 | wh.Headers = headers 159 | topic.Webhooks = append(topic.Webhooks, wh) 160 | equals(t, len(topic.Webhooks), 1) 161 | equals(t, cap(topic.Webhooks), 10) 162 | 163 | // initial creation of a topic config 164 | _, err = pulsardb.Create(&topic) 165 | errNil(t, err) 166 | 167 | // expect error when creation of an already existed topicConfig 168 | _, err = pulsardb.Create(&topic) 169 | equals(t, err != nil, true) 170 | 171 | // however updating an existing topic is allowed 172 | var key string 173 | key, err = pulsardb.Update(&topic) 174 | errNil(t, err) 175 | equals(t, len(key) > 1, true) 176 | 177 | // Load will return a list topicConfig so we can confirm if the one already created exists 178 | res, err := pulsardb.Load() 179 | if err != nil { 180 | log.Fatal(err) 181 | } 182 | found := false 183 | for _, v := range res { 184 | if v.Key == key { 185 | found = true 186 | } 187 | } 188 | equals(t, found, true) 189 | 190 | // Get topic by fullename and pulsar URL 191 | resTopic, err := pulsardb.GetByTopic(topic.TopicFullName, topic.PulsarURL) 192 | errNil(t, err) 193 | equals(t, topic.Token, resTopic.Token) 194 | equals(t, topic.PulsarURL, resTopic.PulsarURL) 195 | 196 | docs2, err2 := pulsardb.Load() 197 | errNil(t, err2) 198 | equals(t, 1, len(docs2)) 199 | 200 | resTopic2, err := pulsardb.GetByKey(resTopic.Key) 201 | errNil(t, err) 202 | equals(t, topic.Token, resTopic2.Token) 203 | equals(t, topic.PulsarURL, resTopic2.PulsarURL) 204 | 205 | deletedKey, err := pulsardb.Delete(topic.TopicFullName, topic.PulsarURL) 206 | errNil(t, err) 207 | equals(t, deletedKey, key) 208 | 209 | resTopic2, err = pulsardb.GetByKey(resTopic.Key) 210 | assert(t, err != nil, "already deleted so returns error") 211 | equals(t, err.Error(), DocNotFound) 212 | 213 | err = pulsardb.Sync() 214 | assert(t, err != nil, "pulsardb sync is not implemented yet") 215 | // TODO: find a place to test Close(); need to find out dependencies. 216 | // Comment out because there are other test cases require database. 217 | errNil(t, pulsardb.Close()) 218 | } 219 | -------------------------------------------------------------------------------- /src/unit-test/example_private_key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDSe16b0yrNY6PepjkCbL/IXKf32/ucjMaGKCnBcHuJ8U3U66WADQXJEDIcQLziKC5w1pbAq3Sv6sIZ/gL+lTe0NGJWoKYF7wIgtL088DN6rWzJLIii2hIPVfkggdGSjfOz935vZKZxthHb9GCweYswifNb23UJdxVHQA1VmMmma6dgZNhGSLPHepFoycdIsT8r3jgxu0/sWLRqlSMU17+Cd8rY2JmIDdPVkjtphPRBZzIYh1YAkfdKbtQ8b4FyRAEtTXQ0/gtlpUH0sZj20MWMERYpMYTDu+l2tahNoRqIFAreLmboRb3i8m0ot2M+nryRtuJQeEO7GXQgAgnxoXCRAgMBAAECggEAQaiyjrGE/KVEjUQVLKh3+yzMSQmap+9STq9gtzM7loBr4yvPaO6nC12+BAo94d7e/dwzEs9piycUfb87d+dchR6CPrvGjrtMUp+PGN9lb7OB6A+4X/TfBWGwYW+dzLXzfASMsHsZYQeG8rJ9JxADV5TxEYqYK+e1/4//mOAcp4kEsuppf5i8mGLCxaUMandLFUhGgdoTE1NK/ZhTqXucEtS/iFzhfij1aVqVodErjhZFIi2pwLawiaph9RGTZz1n2WvmmtFdkM+SVGfrFCiEMTRg0KZXWXdOcb3FS2SLdBR+0IzsnVOozo3S//54GsVDRUDbPO0TukLVsbItzeZqQQKBgQDxPMtfdNp41IsiEQacMkrnW3OBECbGGX/1XuPZiaZjhGgxb+ahEVyY79aS7mHcRcMImW/LTYovsXZUUtbzIWgTWHduT9S+jjlRT8iq3CT/3pFvfPR2uBtKL5EXyn5o51PqC02MLHV2E4Tv/1S8ViC9RKTNe68w93o8VHFzv5odbQKBgQDfXMLW6IsUHd5qYQfp1C/gv2UbrmunqJfAyaKLr/dwJbhqnSG7N80Vmlm/qq+F/FHTvXd2FVkBvsz6VaRFRvtLqTZkYJQiFtGhJ5yNYhoC9sUytMOXY6YiC29oejJiyBH8ZYoWXD3/J6Zelc69ofN6b6E6Ewp6582zQWxt7HQdNQKBgQCBVVazlrKKlWkCmp4wn8YIw77pMv+WtZUkt/rwZhwOvq4d0yCSufwrAmrH24Yb0kr+EUUeejPb3gjSrJcRQpfJx2RAgAxPHXEdZujusZgkle7jFtr5yzrWSyo+1xFurJcQo36v6CYRZ+WdxgZn8sUas/KnN+h1GDkwnxU0OUUnAQKBgQC8JGOOKJS13i7xxkenI28szar59SKDN9STtIQxS4iOM4eybibyjx81mi0M8FYm8xFt3IMauQKfILuCBc9390FHSwIg7OT6DeSH7VjGEqM8aeZTPsYd5/cRaDZwd7WnVUUpJ9J34Tzrhtdxhph9TURMlmjlBRLn0geHfY06zlhEHQKBgQDvHDy6AnFipMkOs9AvHiXrTkLgoscol4kp5gd1+Y4U/rXtYMiY8ac2wRQN9MKl6a4uWLEwP6Vrhspot5XlLoARRumTspGn8Eyz2k4VOASsBQSeQrz/pvU0PfgSLo1Zl+QS4ypcwaXgCibQJMUModQLeodK+0X/exjOWt99U3UEJw== 3 | -----END RSA PRIVATE KEY----- -------------------------------------------------------------------------------- /src/unit-test/example_public_key.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0ntem9MqzWOj3qY5Amy/yFyn99v7nIzGhigpwXB7ifFN1OulgA0FyRAyHEC84igucNaWwKt0r+rCGf4C/pU3tDRiVqCmBe8CILS9PPAzeq1sySyIotoSD1X5IIHRko3zs/d+b2SmcbYR2/RgsHmLMInzW9t1CXcVR0ANVZjJpmunYGTYRkizx3qRaMnHSLE/K944MbtP7Fi0apUjFNe/gnfK2NiZiA3T1ZI7aYT0QWcyGIdWAJH3Sm7UPG+BckQBLU10NP4LZaVB9LGY9tDFjBEWKTGEw7vpdrWoTaEaiBQK3i5m6EW94vJtKLdjPp68kbbiUHhDuxl0IAIJ8aFwkQIDAQAB 3 | -----END PUBLIC KEY----- -------------------------------------------------------------------------------- /src/unit-test/middleware_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "os" 8 | "testing" 9 | "time" 10 | 11 | "github.com/kafkaesque-io/pulsar-beam/src/icrypto" 12 | . "github.com/kafkaesque-io/pulsar-beam/src/middleware" 13 | "github.com/kafkaesque-io/pulsar-beam/src/route" 14 | "github.com/kafkaesque-io/pulsar-beam/src/util" 15 | ) 16 | 17 | func mockHandler(w http.ResponseWriter, r *http.Request) { 18 | w.WriteHeader(http.StatusOK) 19 | return 20 | } 21 | 22 | func mockWorkerHandler(w http.ResponseWriter, r *http.Request) { 23 | time.Sleep(1 * time.Second) 24 | w.WriteHeader(http.StatusOK) 25 | return 26 | } 27 | 28 | func TestAuthJWTMiddleware(t *testing.T) { 29 | // thanks goodness it is singleton 30 | publicKeyPath := "./example_public_key.pub" 31 | privateKeyPath := "./example_private_key" 32 | icrypto.NewRSAKeyPair(privateKeyPath, publicKeyPath) 33 | 34 | handlerTest := AuthVerifyJWT(http.HandlerFunc(mockHandler)) 35 | 36 | req, err := http.NewRequest(http.MethodGet, "http://test", nil) 37 | errNil(t, err) 38 | 39 | rr := httptest.NewRecorder() 40 | 41 | // test missing authorization header 42 | handlerTest.ServeHTTP(rr, req) 43 | equals(t, http.StatusUnauthorized, rr.Code) 44 | 45 | // test a valid token 46 | req.Header.Set("Authorization", "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJwaWNhc3NvIn0.TZilYXJOeeCLwNOHICCYyFxUlwOLxa_kzVjKcoQRTJm2xqmNzTn-s9zjbuaNMCDj1U7gRPHKHkWNDb2W4MwQd6Nkc543E_cIHlJG82eKKIsGfAEQpnPJLpzz2zytgmRON6HCPDsQDAKIXHriKmbmCzHLOILziks0oOCadBGC79iddb9DjPku6sU0nByS8r8_oIrRCqV_cNsH1MInA6CRNYkPJaJI0T8i77ND7azTXwH0FTX_KE_yRmOkXnejJ14GEEcBM99dPGg8jCp-zOyfvrMIJjWsWzjXYExxjKaC85779ciu59YO3cXd0Lk2LzlyB4kDKZgPyqOgyQFIfQ1eiA") // pragma: allowlist secret 47 | rr = httptest.NewRecorder() 48 | handlerTest.ServeHTTP(rr, req) 49 | equals(t, http.StatusOK, rr.Code) 50 | 51 | // test invalid token 52 | req.Header.Set("Authorization", "eeyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJwaWNhc3NvIn0.TZilYXJOeeCLwNOHICCYyFxUlwOLxa_kzVjKcoQRTJm2xqmNzTn-s9zjbuaNMCDj1U7gRPHKHkWNDb2W4MwQd6Nkc543E_cIHlJG82eKKIsGfAEQpnPJLpzz2zytgmRON6HCPDsQDAKIXHriKmbmCzHLOILziks0oOCadBGC79iddb9DjPku6sU0nByS8r8_oIrRCqV_cNsH1MInA6CRNYkPJaJI0T8i77ND7azTXwH0FTX_KE_yRmOkXnejJ14GEEcBM99dPGg8jCp-zOyfvrMIJjWsWzjXYExxjKaC85779ciu59YO3cXd0Lk2LzlyB4kDKZgPyqOgyQFIfQ1eiA") // pragma: allowlist secret 53 | rr = httptest.NewRecorder() 54 | handlerTest.ServeHTTP(rr, req) 55 | equals(t, http.StatusUnauthorized, rr.Code) 56 | 57 | // test valid bearer token 58 | req.Header.Set("Authorization", "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJwaWNhc3NvIn0.TZilYXJOeeCLwNOHICCYyFxUlwOLxa_kzVjKcoQRTJm2xqmNzTn-s9zjbuaNMCDj1U7gRPHKHkWNDb2W4MwQd6Nkc543E_cIHlJG82eKKIsGfAEQpnPJLpzz2zytgmRON6HCPDsQDAKIXHriKmbmCzHLOILziks0oOCadBGC79iddb9DjPku6sU0nByS8r8_oIrRCqV_cNsH1MInA6CRNYkPJaJI0T8i77ND7azTXwH0FTX_KE_yRmOkXnejJ14GEEcBM99dPGg8jCp-zOyfvrMIJjWsWzjXYExxjKaC85779ciu59YO3cXd0Lk2LzlyB4kDKZgPyqOgyQFIfQ1eiA") // pragma: allowlist secret 59 | rr = httptest.NewRecorder() 60 | handlerTest.ServeHTTP(rr, req) 61 | equals(t, http.StatusOK, rr.Code) 62 | } 63 | 64 | func TestAuthHeaderRequiredMiddleware(t *testing.T) { 65 | handlerTest := AuthHeaderRequired(http.HandlerFunc(mockHandler)) 66 | 67 | req, err := http.NewRequest(http.MethodGet, "http://test", nil) 68 | errNil(t, err) 69 | 70 | rr := httptest.NewRecorder() 71 | 72 | // test missing authorization header 73 | handlerTest.ServeHTTP(rr, req) 74 | equals(t, http.StatusUnauthorized, rr.Code) 75 | 76 | // test a valid token 77 | req.Header.Set("Authorization", "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJwaWNhc3NvIn0.TZilYXJOeeCLwNOHICCYyFxUlwOLxa_kzVjKcoQRTJm2xqmNzTn-s9zjbuaNMCDj1U7gRPHKHkWNDb2W4MwQd6Nkc543E_cIHlJG82eKKIsGfAEQpnPJLpzz2zytgmRON6HCPDsQDAKIXHriKmbmCzHLOILziks0oOCadBGC79iddb9DjPku6sU0nByS8r8_oIrRCqV_cNsH1MInA6CRNYkPJaJI0T8i77ND7azTXwH0FTX_KE_yRmOkXnejJ14GEEcBM99dPGg8jCp-zOyfvrMIJjWsWzjXYExxjKaC85779ciu59YO3cXd0Lk2LzlyB4kDKZgPyqOgyQFIfQ1eiA") // pragma: allowlist secret 78 | rr = httptest.NewRecorder() 79 | handlerTest.ServeHTTP(rr, req) 80 | equals(t, http.StatusOK, rr.Code) 81 | 82 | // test invalid token 83 | req.Header.Set("Authorization", "eeyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJwaWNhc3NvIn0.TZilYXJOeeCLwNOHICCYyFxUlwOLxa_kzVjKcoQRTJm2xqmNzTn-s9zjbuaNMCDj1U7gRPHKHkWNDb2W4MwQd6Nkc543E_cIHlJG82eKKIsGfAEQpnPJLpzz2zytgmRON6HCPDsQDAKIXHriKmbmCzHLOILziks0oOCadBGC79iddb9DjPku6sU0nByS8r8_oIrRCqV_cNsH1MInA6CRNYkPJaJI0T8i77ND7azTXwH0FTX_KE_yRmOkXnejJ14GEEcBM99dPGg8jCp-zOyfvrMIJjWsWzjXYExxjKaC85779ciu59YO3cXd0Lk2LzlyB4kDKZgPyqOgyQFIfQ1eiA") // pragma: allowlist secret 84 | rr = httptest.NewRecorder() 85 | handlerTest.ServeHTTP(rr, req) 86 | equals(t, http.StatusOK, rr.Code) 87 | } 88 | 89 | func TestNoAuthMiddleware(t *testing.T) { 90 | handlerTest := NoAuth(http.HandlerFunc(mockHandler)) 91 | 92 | req, err := http.NewRequest(http.MethodGet, "http://test", nil) 93 | errNil(t, err) 94 | 95 | rr := httptest.NewRecorder() 96 | handlerTest.ServeHTTP(rr, req) 97 | equals(t, http.StatusOK, rr.Code) 98 | } 99 | 100 | func TestRateLimitMiddleware(t *testing.T) { 101 | handlerTest := LimitRate(http.HandlerFunc(mockHandler)) 102 | 103 | req, err := http.NewRequest(http.MethodGet, "http://test", nil) 104 | errNil(t, err) 105 | 106 | request := func() { 107 | rr := httptest.NewRecorder() 108 | handlerTest.ServeHTTP(rr, req) 109 | equals(t, http.StatusOK, rr.Code) 110 | } 111 | // 112 | for i := 0; i < 1000; i++ { 113 | go request() 114 | } 115 | 116 | } 117 | 118 | func TestLoggerMiddleware(t *testing.T) { 119 | logger := route.Logger(http.HandlerFunc(mockHandler), "test") 120 | 121 | req, err := http.NewRequest(http.MethodGet, "http://test", nil) 122 | errNil(t, err) 123 | 124 | rr := httptest.NewRecorder() 125 | logger.ServeHTTP(rr, req) 126 | equals(t, http.StatusOK, rr.Code) 127 | } 128 | 129 | func TestAuthJWTMiddlewareWithNoAuth(t *testing.T) { 130 | // thanks goodness it is singleton 131 | publicKeyPath := "./example_public_key.pub" 132 | privateKeyPath := "./example_private_key" 133 | icrypto.NewRSAKeyPair(privateKeyPath, publicKeyPath) 134 | 135 | os.Setenv("HTTPAuthImpl", "noauth") 136 | os.Setenv("SuperRoles", "thisisroot,anotherroot") 137 | util.ReadConfigFile("../" + util.DefaultConfigFile) 138 | 139 | handlerTest := AuthVerifyJWT(http.HandlerFunc(mockHandler)) 140 | 141 | req, err := http.NewRequest(http.MethodGet, "http://test", nil) 142 | errNil(t, err) 143 | 144 | rr := httptest.NewRecorder() 145 | 146 | // test missing authorization header 147 | handlerTest.ServeHTTP(rr, req) 148 | rr = httptest.NewRecorder() 149 | fmt.Printf("subs are %v size %d", rr.Header(), len(rr.Header())) 150 | handlerTest.ServeHTTP(rr, req) 151 | equals(t, "thisisroot", req.Header.Get("injectedSubs")) 152 | equals(t, http.StatusOK, rr.Code) 153 | } 154 | 155 | func TestSemaphore(t *testing.T) { 156 | var sema = NewSema(2) 157 | err := sema.Release() 158 | equals(t, "all semaphore buffer empty", err.Error()) 159 | 160 | err = sema.Acquire() 161 | errNil(t, err) 162 | sema.Acquire() 163 | err = sema.Acquire() 164 | assertErr(t, "all semaphore buffer full", err) 165 | 166 | sema.Release() 167 | errNil(t, sema.Acquire()) 168 | 169 | errNil(t, sema.Release()) 170 | errNil(t, sema.Release()) 171 | 172 | err = sema.Release() 173 | assertErr(t, "all semaphore buffer empty", err) 174 | } 175 | -------------------------------------------------------------------------------- /src/unit-test/model_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/kafkaesque-io/pulsar-beam/src/model" 7 | ) 8 | 9 | func TestPulsarMessages(t *testing.T) { 10 | 11 | messages := NewPulsarMessages(10) 12 | equals(t, messages.Limit, 10) 13 | equals(t, messages.IsEmpty(), true) 14 | } 15 | -------------------------------------------------------------------------------- /src/unit-test/pk12-binary-private.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kafkaesque-io/pulsar-beam/1a2f11041e1ec44d693d77a66fbe16ee494a7549/src/unit-test/pk12-binary-private.key -------------------------------------------------------------------------------- /src/unit-test/pk12-binary-public.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kafkaesque-io/pulsar-beam/1a2f11041e1ec44d693d77a66fbe16ee494a7549/src/unit-test/pk12-binary-public.key -------------------------------------------------------------------------------- /src/unit-test/pulsar_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/kafkaesque-io/pulsar-beam/src/pulsardriver" 9 | "github.com/kafkaesque-io/pulsar-beam/src/util" 10 | ) 11 | 12 | func TestClientCreation(t *testing.T) { 13 | util.Config.DbConnectionStr = os.Getenv("PULSAR_URI") 14 | util.Config.DbName = os.Getenv("REST_DB_TABLE_TOPIC") 15 | util.Config.DbPassword = os.Getenv("PULSAR_TOKEN") 16 | util.Config.PbDbType = "pulsarAsDb" 17 | util.Config.TrustStore = os.Getenv("TrustStore") 18 | 19 | os.Setenv("PulsarClientOperationTimeout", "1") 20 | os.Setenv("PulsarClientConnectionTimeout", "1") 21 | 22 | _, err := pulsardriver.GetPulsarClient("pulsar://test url", "token", false) 23 | assert(t, err != nil, "create pulsar driver with bogus url") 24 | assert(t, strings.HasPrefix(err.Error(), "Could not instantiate Pulsar client: "), "match invalid service URL at pulsar client creation") 25 | 26 | _, err = pulsardriver.GetPulsarClient("pulsar://useast1.do.kafkaesque.io:6650", "token", true) 27 | errNil(t, err) 28 | 29 | _, err = pulsardriver.GetPulsarConsumer("pulsar://test url", "token", "topicname", "sub", "failover", "latest", "subKey") 30 | assert(t, err != nil, "create pulsar consumer with bogus url") 31 | 32 | pulsardriver.CancelPulsarConsumer("testkey") //just to bump up code coverage 33 | } 34 | 35 | func TestProducerObject(t *testing.T) { 36 | os.Setenv("PulsarClientOperationTimeout", "1") 37 | os.Setenv("PulsarClientConnectionTimeout", "1") 38 | 39 | _, err := pulsardriver.GetPulsarProducer("pulsar://test url", "tokenstring", "topicName") 40 | assert(t, err != nil, "create pulsar consumer with bogus url") 41 | 42 | // pulsardriver.SendToPulsar("pulsar://", "tokenstring", "topicName", []byte("payload"), false) 43 | 44 | p := pulsardriver.PulsarProducer{} 45 | p.UpdateTime() 46 | p.Close() 47 | 48 | c := pulsardriver.PulsarConsumer{} 49 | c.UpdateTime() 50 | c.Close() 51 | 52 | clt := pulsardriver.PulsarClient{} 53 | clt.UpdateTime() 54 | clt.Close() 55 | } 56 | -------------------------------------------------------------------------------- /src/unit-test/test_util.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "reflect" 7 | "runtime" 8 | "testing" 9 | ) 10 | 11 | // assert fails the test if the condition is false. 12 | func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { 13 | if !condition { 14 | _, file, line, _ := runtime.Caller(1) 15 | fmt.Printf("[%s:%d: "+msg+"]\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) 16 | tb.FailNow() 17 | } 18 | } 19 | 20 | // test if an err is not nil. 21 | func errNil(tb testing.TB, err error) { 22 | if err != nil { 23 | _, file, line, _ := runtime.Caller(1) 24 | fmt.Printf("[%s:%d: unexpected error: %s]\n\n", filepath.Base(file), line, err.Error()) 25 | tb.FailNow() 26 | } 27 | } 28 | 29 | // test if an err is not nil. 30 | func assertErr(tb testing.TB, exp string, err error) { 31 | if err == nil { 32 | _, file, line, _ := runtime.Caller(1) 33 | fmt.Printf("[%s:%d: error is expected.]\n\n", filepath.Base(file), line) 34 | tb.FailNow() 35 | } else { 36 | equals(tb, exp, err.Error()) 37 | } 38 | } 39 | 40 | // equals fails the test if exp is not equal to act. 41 | func equals(tb testing.TB, exp, act interface{}) { 42 | if !reflect.DeepEqual(exp, act) { 43 | _, file, line, _ := runtime.Caller(1) 44 | fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) 45 | tb.FailNow() 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/unit-test/util_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "os" 9 | "strconv" 10 | "strings" 11 | "testing" 12 | "time" 13 | 14 | "github.com/kafkaesque-io/pulsar-beam/src/broker" 15 | "github.com/kafkaesque-io/pulsar-beam/src/model" 16 | "github.com/kafkaesque-io/pulsar-beam/src/route" 17 | "github.com/kafkaesque-io/pulsar-beam/src/util" 18 | . "github.com/kafkaesque-io/pulsar-beam/src/util" 19 | ) 20 | 21 | func TestUUID(t *testing.T) { 22 | // Create 1 million to make sure no duplicates 23 | size := 100000 24 | set := make(map[string]bool) 25 | for i := 0; i < size; i++ { 26 | id, err := NewUUID() 27 | errNil(t, err) 28 | set[id] = true 29 | } 30 | assert(t, size == len(set), "UUID duplicates found") 31 | 32 | } 33 | 34 | func TestDbKeyGeneration(t *testing.T) { 35 | topicFullName := "persistent://my-topic/local-useast1-gcp/cloudfunction-funnel" 36 | pulsarURL := "pulsar+ssl://useast3.aws.host.io:6651" 37 | first := model.GenKey(topicFullName, pulsarURL) 38 | 39 | assert(t, first == model.GenKey(topicFullName, pulsarURL), "same key generates same hash") 40 | assert(t, first != model.GenKey(topicFullName, "pulsar://useast3.aws.host.io:6650"), "different key generates different hash") 41 | assert(t, first != model.GenKey("persistent://my-topic/local-useast1-gcp/cloudfunction-funnel-a", pulsarURL), "different key generates different hash") 42 | } 43 | 44 | func TestJoinString(t *testing.T) { 45 | part0 := "a" 46 | part1 := "b" 47 | part2 := "cd" 48 | assert(t, JoinString(part0, part2) == part0+part2, "JoinString test two string") 49 | assert(t, JoinString(part1, part0, part2) == part1+part0+part2, "JoinString test three string") 50 | } 51 | func TestAssignString(t *testing.T) { 52 | assert(t, "YesYo" == AssignString("", "", "YesYo"), "test default") 53 | assert(t, "YesYo" == AssignString("", "YesYo", "test"), "test valid string in middle") 54 | assert(t, "YesYo" == AssignString("YesYo", "YesYo2", "test"), "test valid string in head") 55 | assert(t, "" == AssignString("", "", ""), "test no valid string") 56 | } 57 | 58 | func TestLoadConfigFile(t *testing.T) { 59 | 60 | os.Setenv("PORT", "9876543") 61 | ReadConfigFile("../" + DefaultConfigFile) 62 | config := GetConfig() 63 | assert(t, config.PORT == "9876543", "config value overwritteen by env") 64 | assert(t, config.PbDbInterval == "10s", "default config setting") 65 | assert(t, os.Getenv("PbDbInterval") == "10s", "default config setting") 66 | 67 | assert(t, false == util.StringToBool(os.Getenv("PulsarTLSAllowInsecureConnection")), "") 68 | 69 | assert(t, os.Getenv("PulsarTLSValidateHostname") == "", "PulsarTLSValidateHostname default config from env") 70 | assert(t, false == util.StringToBool(os.Getenv("PulsarTLSValidateHostname")), "PulsarTLSValidateHostname false as the default config from env") 71 | assert(t, os.Getenv("PbDbInterval") == "10s", "default config setting") 72 | 73 | dbType := "inmemory2" 74 | os.Setenv("PbDbType", dbType) 75 | ReadConfigFile("../../config/pulsar_beam.yml") 76 | config2 := GetConfig() 77 | assert(t, config2.PORT == "9876543", "config value overwritteen by env") 78 | fmt.Println(config2.PbDbType) 79 | assert(t, config2.PbDbType == dbType, "default config setting") 80 | } 81 | 82 | func TestHTTPRouters(t *testing.T) { 83 | mode := "hybrid" 84 | router := route.NewRouter(&mode) 85 | 86 | routeName := "receive" 87 | route := router.Get(routeName) 88 | assert(t, route == nil, "get route name") 89 | } 90 | 91 | func TestMainControlMode(t *testing.T) { 92 | mode := "receiver" 93 | assert(t, IsBroker(&mode) == false, "") 94 | assert(t, IsValidMode(&mode), "") 95 | mode = "broker" 96 | assert(t, IsBroker(&mode), "") 97 | assert(t, IsBrokerRequired(&mode), "") 98 | assert(t, IsValidMode(&mode), "") 99 | 100 | mode = "hybrid" 101 | assert(t, IsHTTPRouterRequired(&mode), "") 102 | assert(t, IsBrokerRequired(&mode), "") 103 | assert(t, IsValidMode(&mode), "") 104 | 105 | mode = "rest" 106 | assert(t, IsHTTPRouterRequired(&mode), "") 107 | assert(t, IsBrokerRequired(&mode) == false, "") 108 | assert(t, IsBroker(&mode) == false, "") 109 | assert(t, IsValidMode(&mode), "") 110 | 111 | mode = "tokenserver" 112 | assert(t, IsHTTPRouterRequired(&mode), "") 113 | assert(t, IsBrokerRequired(&mode) == false, "") 114 | assert(t, IsBroker(&mode) == false, "") 115 | assert(t, IsValidMode(&mode), "") 116 | 117 | mode = "http2" 118 | assert(t, IsHTTPRouterRequired(&mode), "") 119 | assert(t, IsBrokerRequired(&mode) == false, "") 120 | assert(t, IsBroker(&mode) == false, "") 121 | assert(t, IsValidMode(&mode), "") 122 | 123 | mode = "oops" 124 | assert(t, !IsValidMode(&mode), "test invalid mode") 125 | assert(t, !IsHTTPRouterRequired(&mode), "test invalid HTTPRouterRequired mode") 126 | } 127 | 128 | func TestReceiverHeader(t *testing.T) { 129 | header := http.Header{} 130 | // header.Set("Authorization", "Bearer erfagagagag") 131 | _, _, _, result := ReceiverHeader(strings.Split("", ","), &header) 132 | equals(t, result.Error(), "missing configured Pulsar URL") 133 | 134 | header.Set("PulsarUrl", "http://target.net/route") 135 | var token string 136 | token, _, _, result = ReceiverHeader(strings.Split("", ","), &header) 137 | errNil(t, result) 138 | assert(t, token == "", "test all headers presence") 139 | } 140 | 141 | func TestDefaultPulsarURLInReceiverHeader(t *testing.T) { 142 | allowedPulsarURLs := strings.Split("pulsar+ssl://kafkaesque.net:6651", ",") 143 | header := http.Header{} 144 | header.Set("Authorization", "Bearer erfagagagag") 145 | _, _, pulsarURL, result := ReceiverHeader(allowedPulsarURLs, &header) 146 | errNil(t, result) 147 | equals(t, pulsarURL, "pulsar+ssl://kafkaesque.net:6651") 148 | 149 | header.Set("TopicFn", "http://target.net/route") 150 | var webhook string 151 | _, webhook, pulsarURL, result = ReceiverHeader(allowedPulsarURLs, &header) 152 | errNil(t, result) 153 | assert(t, webhook == header.Get("TopicFn"), "test all headers presence") 154 | assert(t, pulsarURL == "pulsar+ssl://kafkaesque.net:6651", "test all headers presence") 155 | assert(t, "" == header.Get("PulsarUrl"), "ensure PulsarUrl is empty") 156 | } 157 | 158 | func TestThreadSafeMap(t *testing.T) { 159 | // TODO add more goroutine to test concurrency 160 | wb := broker.NewWebhookBroker(&Configuration{}) 161 | 162 | _, rc := wb.ReadWebhook("first") 163 | equals(t, false, rc) 164 | 165 | sig := make(chan *broker.SubCloseSignal, 2) 166 | 167 | wb.WriteWebhook("first", sig) 168 | _, rc = wb.ReadWebhook("first") 169 | equals(t, true, rc) 170 | wb.DeleteWebhook("first") 171 | _, rc = wb.ReadWebhook("first") 172 | equals(t, false, rc) 173 | go func() { 174 | for i := 0; i < 1000; i++ { 175 | sig2 := make(chan *broker.SubCloseSignal, 2) 176 | wb.WriteWebhook("key"+strconv.Itoa(i), sig2) 177 | wb.WriteWebhook("first", sig2) 178 | } 179 | }() 180 | go func() { 181 | for i := 0; i < 1000; i++ { 182 | wb.ReadWebhook("key" + strconv.Itoa(i)) 183 | wb.ReadWebhook("first") 184 | } 185 | }() 186 | go func() { 187 | for i := 0; i < 1000; i++ { 188 | sig3 := make(chan *broker.SubCloseSignal, 2) 189 | wb.DeleteWebhook("key" + strconv.Itoa(i)) 190 | wb.WriteWebhook("first"+strconv.Itoa(i), sig3) 191 | } 192 | }() 193 | } 194 | 195 | func TestReportError(t *testing.T) { 196 | errorStr := "my invented error" 197 | equals(t, errorStr, ReportError(errors.New(errorStr)).Error()) 198 | } 199 | 200 | func TestStrContains(t *testing.T) { 201 | assert(t, StrContains([]string{"foo", "bar", "flying"}, "foo"), "") 202 | assert(t, !StrContains([]string{"foo", "bar", "flying"}, "foobar"), "") 203 | assert(t, !StrContains([]string{}, "foobar"), "") 204 | } 205 | 206 | func TestGetEnvInt(t *testing.T) { 207 | os.Setenv("Kopper", "546") 208 | equals(t, 546, GetEnvInt("Kopper", 90)) 209 | 210 | os.Setenv("Kopper", "5o46") 211 | equals(t, 946, GetEnvInt("Kopper", 946)) 212 | } 213 | 214 | type TestObj struct { 215 | isClosed bool 216 | } 217 | 218 | func (o *TestObj) Close() { 219 | o.isClosed = true 220 | } 221 | 222 | func TestExpiryTTLCache(t *testing.T) { 223 | 224 | cache := NewCache(CacheOption{ 225 | TTL: 10 * time.Millisecond, 226 | CleanInterval: 12 * time.Millisecond, 227 | ExpireCallback: func(key string, value interface{}) { 228 | if obj, ok := value.(*TestObj); ok { 229 | obj.Close() 230 | } else { 231 | assert(t, false, "wrong object type stored in Cache") 232 | } 233 | }, 234 | }) 235 | 236 | object1 := TestObj{} 237 | cache.Set("object1", &object1) 238 | cache.Set("object2", &TestObj{}) 239 | cache.Set("object3", &TestObj{}) 240 | 241 | object4 := TestObj{} 242 | cache.Set("object4", &object4) 243 | 244 | object5 := TestObj{} 245 | cache.SetWithTTL("object5", &object5, 4*time.Millisecond) 246 | 247 | obj, ok := cache.Get("object5") 248 | assert(t, ok, "verify added object exists in cache") 249 | assert(t, !(obj.(*TestObj).isClosed), "ensure object has not been Close()") 250 | assert(t, 5 == cache.Count(), "check the counts of total number of objects in cache") 251 | 252 | // make sure object4 won't expire 253 | time.Sleep(8 * time.Millisecond) 254 | _, ok = cache.Get("object5") 255 | assert(t, !ok, "object5 has already expired") 256 | 257 | // comment out because GitHub action may have different timing that causes this failing 258 | //_, ok = cache.Get("object4") 259 | //assert(t, ok, "object4 still exists") 260 | 261 | // another 2ms expires the default TTL 262 | time.Sleep(2 * time.Millisecond) 263 | assert(t, 4 == cache.Count(), "all objects are expired but not purged yet") 264 | _, ok = cache.Get("object1") 265 | assert(t, !ok, "object has expired") 266 | assert(t, object1.isClosed, "object1 has been Close() by the callback") 267 | assert(t, !object4.isClosed, "object4 has not been Close() by the callback") 268 | assert(t, object5.isClosed, "object5 has not been Close() by the callback") 269 | 270 | //time.Sleep(2 * time.Millisecond) 271 | //assert(t, 1 == cache.Count(), "check the counts of total number of objects in cache") 272 | //assert(t, !object4.isClosed, "object4 has not expired yet") 273 | 274 | } 275 | 276 | func TestInfinityExpiryTTLCache(t *testing.T) { 277 | 278 | cache := NewCache(CacheOption{ 279 | TTL: 2 * time.Millisecond, 280 | CleanInterval: 3 * time.Millisecond, 281 | ExpireCallback: func(key string, value interface{}) { 282 | if obj, ok := value.(*TestObj); ok { 283 | obj.Close() 284 | } else { 285 | assert(t, false, "wrong object type stored in Cache") 286 | } 287 | }, 288 | }) 289 | 290 | object1 := TestObj{} 291 | cache.SetWithTTL("object1", &object1, -1) 292 | 293 | object2 := TestObj{} 294 | cache.Set("object2", &object2) 295 | 296 | obj, ok := cache.Get("object2") 297 | assert(t, ok, "verify added object exists in cache") 298 | assert(t, !(obj.(*TestObj).isClosed), "ensure object has not been Close()") 299 | assert(t, 2 == cache.Count(), "check the counts of total number of objects in cache") 300 | 301 | // make sure object4 won't expire 302 | time.Sleep(4 * time.Millisecond) 303 | _, ok = cache.Get("object1") 304 | assert(t, ok, "object1 still exists") 305 | _, ok = cache.Get("object2") 306 | assert(t, !ok, "object2 already expired") 307 | 308 | assert(t, 1 == cache.Count(), "all objects are expired but not purged yet") 309 | 310 | assert(t, !object1.isClosed, "object1 has not expired yet") 311 | assert(t, object2.isClosed, "object2 already Close()") 312 | 313 | cache.Delete("object1") 314 | assert(t, 0 == cache.Count(), "all objects should be deleted") 315 | cache.Close() 316 | } 317 | 318 | func TestConcurrencyTTLCache(t *testing.T) { 319 | 320 | cache := NewCache(CacheOption{ 321 | TTL: 2 * time.Millisecond, 322 | CleanInterval: 10 * time.Millisecond, 323 | ExpireCallback: func(key string, value interface{}) { 324 | if obj, ok := value.(*TestObj); ok { 325 | obj.Close() 326 | } else { 327 | assert(t, false, "wrong object type stored in Cache") 328 | } 329 | }, 330 | }) 331 | 332 | for i := 0; i < 5; i++ { 333 | go func(index int) { 334 | cache.Set("p_object"+strconv.Itoa(index), &TestObj{}) 335 | fmt.Printf("added entry %s\n", "d_object"+strconv.Itoa(index)) 336 | }(i) 337 | } 338 | for i := 0; i < 8; i++ { 339 | go func(index int) { 340 | cache.SetWithTTL("q_object"+strconv.Itoa(index), &TestObj{}, 5*time.Millisecond) 341 | fmt.Printf("added entry %s\n", "object"+strconv.Itoa(index)) 342 | }(i) 343 | } 344 | object1 := TestObj{} 345 | cache.SetWithTTL("object1", &object1, 20*time.Second) 346 | 347 | fmt.Printf("total number is %d\n", cache.Count()) 348 | assert(t, 14 > cache.Count(), "check the counts of total number of objects in cache") 349 | 350 | // make sure object4 won't expire 351 | time.Sleep(3 * time.Millisecond) 352 | fmt.Printf("2 total number is %d\n", cache.Count()) 353 | assert(t, 14 == cache.Count(), "some objects have expired") 354 | assert(t, !object1.isClosed, "object1 has not expired yet") 355 | } 356 | 357 | func TestStringToBoo(t *testing.T) { 358 | // true cases 359 | assert(t, StringToBool("true"), "string true yields boolean true") 360 | assert(t, StringToBool("True"), "string True yields boolean true") 361 | assert(t, StringToBool(" tRue"), "string tRue with space yields boolean true") 362 | assert(t, StringToBool("yes"), "string true yields boolean true") 363 | assert(t, StringToBool("1"), "string true yields boolean true") 364 | assert(t, StringToBool("enable"), "string true yields boolean true") 365 | assert(t, StringToBool(" Enabled "), "string Enabled with space yields boolean true") 366 | assert(t, StringToBool("ok"), "string ok yields boolean true") 367 | assert(t, StringToBool("Ok"), "string Ok yields boolean true") 368 | 369 | // false cases 370 | assert(t, !StringToBool(" "), "string space yields boolean false") 371 | assert(t, !StringToBool(""), "string empty string yields boolean false") 372 | assert(t, !StringToBool(" t rue"), "string t rue with space yields boolean false") 373 | assert(t, !StringToBool("no"), "string no yields boolean false") 374 | assert(t, !StringToBool("10"), "string 10 yields boolean false") 375 | assert(t, !StringToBool("0"), "string 0 yields boolean false") 376 | assert(t, !StringToBool("notok"), "string notok yields boolean false") 377 | assert(t, !StringToBool("disable"), "string disable yields boolean false") 378 | assert(t, !StringToBool("adsfasdf"), "string any string yields boolean false") 379 | } 380 | 381 | func TestTokenizeTopicFullName(t *testing.T) { 382 | isPersistent, tenant, ns, topic, err := TokenizeTopicFullName("persistent://public/default/test-topic") 383 | errNil(t, err) 384 | equals(t, isPersistent, true) 385 | equals(t, tenant, "public") 386 | equals(t, ns, "default") 387 | equals(t, topic, "test-topic") 388 | 389 | isPersistent, tenant, ns, topic, err = TokenizeTopicFullName("non-persistent://public/default/test-topic") 390 | errNil(t, err) 391 | equals(t, isPersistent, false) 392 | equals(t, tenant, "public") 393 | equals(t, ns, "default") 394 | equals(t, topic, "test-topic") 395 | 396 | _, _, _, _, err = TokenizeTopicFullName("persitent://public/default/test-topic") 397 | assertErr(t, "invalid persistent or non-persistent part", err) 398 | 399 | isPersistent, tenant, ns, topic, err = TokenizeTopicFullName("non-persistent://public/default") 400 | errNil(t, err) 401 | equals(t, isPersistent, false) 402 | equals(t, tenant, "public") 403 | equals(t, ns, "default") 404 | equals(t, topic, "") 405 | 406 | _, _, _, _, err = TokenizeTopicFullName("non-persistent://public") 407 | assertErr(t, "missing tenant, namespace, or topic name", err) 408 | 409 | _, _, _, _, err = TokenizeTopicFullName("non-persistent://public/default/to2/to3") 410 | assertErr(t, "missing tenant, namespace, or topic name", err) 411 | } 412 | 413 | func TestHTTPParams(t *testing.T) { 414 | params := url.Values{} 415 | params.Set("var1", "testme") 416 | params.Set("var2", "48") 417 | params.Set("var3", "7") 418 | equals(t, QueryParamInt(params, "var1", 5), 5) 419 | equals(t, QueryParamInt(params, "var2", 5), 48) 420 | equals(t, QueryParamInt(params, "var22", 5), 5) 421 | 422 | equals(t, QueryParamString(params, "var1", "48"), "testme") 423 | equals(t, QueryParamString(params, "var2", "test"), "48") 424 | equals(t, QueryParamString(params, "var22", "another"), "another") 425 | } 426 | -------------------------------------------------------------------------------- /src/util/cache-item.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | const ( 8 | // ItemNotExpire avoids the item being expired by TTL 9 | ItemNotExpire time.Duration = -1 10 | ) 11 | 12 | func newItem(key string, object interface{}, ttl time.Duration) *item { 13 | item := &item{ 14 | data: object, 15 | ttl: ttl, 16 | key: key, 17 | } 18 | // no mutex is required for the first time 19 | item.touch() 20 | return item 21 | } 22 | 23 | type item struct { 24 | key string 25 | data interface{} 26 | ttl time.Duration 27 | expireAt time.Time 28 | } 29 | 30 | // Reset the item expiration time 31 | func (item *item) touch() { 32 | if item.ttl > 0 { 33 | item.expireAt = time.Now().Add(item.ttl) 34 | } 35 | } 36 | 37 | // Verify if the item is expired 38 | func (item *item) expired() bool { 39 | if item.ttl <= 0 { 40 | return false 41 | } 42 | return item.expireAt.Before(time.Now()) 43 | } 44 | -------------------------------------------------------------------------------- /src/util/cert-loader.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // This is a TLS certificate loader that detects the change of key and cert. 4 | // One use case is to reload letsencrypt certificates 5 | // Example to generate test certificate and key files 6 | // openssl req -newkey rsa:2048 -nodes -keyout domain.key -x509 -days 365 -out domain.crt 7 | 8 | import ( 9 | "crypto/tls" 10 | "fmt" 11 | "log" 12 | "net/http" 13 | "os" 14 | "sync/atomic" 15 | "time" 16 | ) 17 | 18 | var cert atomic.Value 19 | 20 | type updatedChann struct{} 21 | 22 | func loadCert(certFile, keyFile string) error { 23 | c, err := tls.LoadX509KeyPair(certFile, keyFile) 24 | if err != nil { 25 | log.Printf("failed to LoadX509KeyPair %v", err) 26 | return err 27 | } 28 | 29 | log.Println("successfully load certs and keys") 30 | cert.Store(c) 31 | return nil 32 | } 33 | 34 | type fileState struct { 35 | info os.FileInfo 36 | err error 37 | } 38 | 39 | func watchFile(filePath string, updated chan *updatedChann) error { 40 | initialStat, err := os.Stat(filePath) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | ticker := time.NewTicker(1 * time.Second) 46 | defer ticker.Stop() 47 | for { 48 | select { 49 | case <-ticker.C: 50 | stat, err := os.Stat(filePath) 51 | if err == nil { 52 | if stat.Size() != initialStat.Size() || stat.ModTime() != initialStat.ModTime() { 53 | initialStat = stat 54 | updated <- &updatedChann{} 55 | } 56 | } 57 | } 58 | } 59 | } 60 | 61 | // ListenAndServeTLS listens HTTP with TLS option just like the default http.ListenAndServeTLS 62 | // in addition it also watches certificate and key file changes and reloads them if necessary 63 | func ListenAndServeTLS(address, certFile, keyFile string, handler http.Handler) error { 64 | if len(certFile) > 1 && len(keyFile) > 1 { 65 | return listenAndServeTLS(address, certFile, keyFile, handler) 66 | } 67 | return http.ListenAndServe(address, handler) 68 | } 69 | 70 | func listenAndServeTLS(address, certFile, keyFile string, handler http.Handler) error { 71 | log.Printf("load certs %s and key files %s\n", certFile, keyFile) 72 | if err := loadCert(certFile, keyFile); err != nil { 73 | return err 74 | } 75 | 76 | go func(cert, key string) { 77 | certMonitorChan := make(chan *updatedChann, 1) 78 | keyMonitorChan := make(chan *updatedChann, 1) 79 | 80 | go watchFile(certFile, certMonitorChan) 81 | go watchFile(keyFile, keyMonitorChan) 82 | 83 | var certUpdated, keyUpdated bool 84 | // only update X509 key pair when both cert and key files are updated 85 | for { 86 | select { 87 | case <-certMonitorChan: 88 | certUpdated = true 89 | if certUpdated == keyUpdated { 90 | certUpdated = false 91 | keyUpdated = false 92 | loadCert(certFile, keyFile) 93 | } 94 | case <-keyMonitorChan: 95 | keyUpdated = true 96 | if certUpdated == keyUpdated { 97 | certUpdated = false 98 | keyUpdated = false 99 | loadCert(certFile, keyFile) 100 | } 101 | } 102 | } 103 | }(certFile, keyFile) 104 | // Create tlsConfig that uses a custom GetCertificate method 105 | // Defined by GetCertificate func at https://golang.org/pkg/crypto/tls/ 106 | tlsConfig := tls.Config{ 107 | MinVersion: tls.VersionTLS12, 108 | GetCertificate: func(i *tls.ClientHelloInfo) (*tls.Certificate, error) { 109 | c, ok := cert.Load().(tls.Certificate) 110 | if !ok { 111 | return nil, fmt.Errorf("Unable to load cert: %+v", c) 112 | } 113 | 114 | return &c, nil 115 | }, 116 | } 117 | 118 | // listen on the port with TLS listener 119 | l, err := tls.Listen("tcp", address, &tlsConfig) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | return http.Serve(l, handler) 125 | } 126 | -------------------------------------------------------------------------------- /src/util/config.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "reflect" 10 | "strings" 11 | 12 | "unicode" 13 | 14 | "github.com/ghodss/yaml" 15 | "github.com/kafkaesque-io/pulsar-beam/src/icrypto" 16 | log "github.com/sirupsen/logrus" 17 | ) 18 | 19 | // DefaultConfigFile - default config file 20 | // it can be overwritten by env variable PULSAR_BEAM_CONFIG 21 | const DefaultConfigFile = "../config/pulsar_beam.yml" 22 | 23 | // Configuration has a set of parameters to configure the beam server. 24 | // The same name can be used in environment variable to override yml or json values. 25 | type Configuration struct { 26 | // PORT is the http port 27 | PORT string `json:"PORT"` 28 | 29 | // CLUSTER is the Plusar cluster 30 | CLUSTER string `json:"CLUSTER"` 31 | 32 | // LogLevel is used to set the application log level 33 | LogLevel string `json:"LogLevel"` 34 | 35 | // DbName is the database name in mongo or topic name when Pulsar is used as database 36 | DbName string `json:"DbName"` 37 | 38 | // DbPassword is either password or token when Pulsar is used as database 39 | DbPassword string `json:"DbPassword"` 40 | 41 | // DbConnectionStr can be mongo url or pulsar url 42 | DbConnectionStr string `json:"DbConnectionStr"` 43 | 44 | // PbDbType is the database type mongo or pulsar 45 | PbDbType string `json:"PbDbType"` 46 | 47 | // Pulsar public and private keys are used to encrypt and decrypt tokens 48 | // They are used by tokenServer end point and authorize Pulsar JWT subject 49 | PulsarPublicKey string `json:"PulsarPublicKey"` 50 | PulsarPrivateKey string `json:"PulsarPrivateKey"` 51 | 52 | // SuperRoles are Pulsar JWT superroles for authorization 53 | SuperRoles string `json:"SuperRoles"` 54 | 55 | // PulsarBrokerURL is the Pulsar Broker URL to allow direct connection to the broker 56 | PulsarBrokerURL string `json:"PulsarBrokerURL"` 57 | 58 | // Configure whether the Pulsar client accept untrusted TLS certificate from broker (default: false) 59 | // Set to `true` to enable 60 | PulsarTLSAllowInsecureConnection string `json:"PulsarTLSAllowInsecureConnection"` 61 | 62 | // Configure whether the Pulsar client verify the validity of the host name from broker (default: false) 63 | // Set to `true` to enable 64 | PulsarTLSValidateHostname string `json:"PulsarTLSValidateHostname"` 65 | 66 | // PbDbInterval is the interval the webhook brokers poll the database for updates in Mongo. 67 | // Pulsar as database has more realtime update feature. 68 | // default value 180s 69 | PbDbInterval string `json:"PbDbInterval"` 70 | 71 | // Pulsar CA certificate key store 72 | TrustStore string `json:"TrustStore"` 73 | 74 | // HTTPs certificate set up 75 | CertFile string `json:"CertFile"` 76 | KeyFile string `json:"KeyFile"` 77 | 78 | // PulsarClusters enforce Beam are only allowed to connect to the specified clusters 79 | // It is a comma separated pulsar URL string, so it can be a list of clusters 80 | PulsarClusters string `json:"PulsarClusters"` 81 | 82 | // HTTPAuthImpl specifies the jwt authen and authorization algorithm, `noauth` to skip JWT authentication 83 | HTTPAuthImpl string `json:"HTTPAuthImpl"` 84 | } 85 | 86 | var ( 87 | // AllowedPulsarURLs specifies a list of allowed pulsar URL/cluster 88 | AllowedPulsarURLs []string 89 | 90 | // SuperRoles are admin level users for jwt authorization 91 | SuperRoles []string 92 | 93 | // Config - this server's configuration instance 94 | Config Configuration 95 | 96 | // JWTAuth is the RSA key pair for sign and verify JWT 97 | JWTAuth *icrypto.RSAKeyPair 98 | 99 | // L is the logger 100 | L *log.Logger 101 | ) 102 | 103 | // Init initializes configuration 104 | func Init() *Configuration { 105 | configFile := AssignString(os.Getenv("PULSAR_BEAM_CONFIG"), DefaultConfigFile) 106 | config := ReadConfigFile(configFile) 107 | 108 | log.SetLevel(logLevel(Config.LogLevel)) 109 | 110 | log.Warnf("Configuration built from file - %s", configFile) 111 | JWTAuth = icrypto.NewRSAKeyPair(Config.PulsarPrivateKey, Config.PulsarPublicKey) 112 | return config 113 | } 114 | 115 | // ReadConfigFile reads configuration file. 116 | func ReadConfigFile(configFile string) *Configuration { 117 | fileBytes, err := ioutil.ReadFile(configFile) 118 | if err != nil { 119 | fmt.Printf("failed to load configuration file %s", configFile) 120 | panic(err) 121 | } 122 | 123 | if hasJSONPrefix(fileBytes) { 124 | err = json.Unmarshal(fileBytes, &Config) 125 | if err != nil { 126 | panic(err) 127 | } 128 | } else { 129 | err = yaml.Unmarshal(fileBytes, &Config) 130 | if err != nil { 131 | panic(err) 132 | } 133 | } 134 | 135 | // Next section allows env variable overwrites config file value 136 | fields := reflect.TypeOf(Config) 137 | // pointer to struct 138 | values := reflect.ValueOf(&Config) 139 | // struct 140 | st := values.Elem() 141 | for i := 0; i < fields.NumField(); i++ { 142 | field := fields.Field(i).Name 143 | f := st.FieldByName(field) 144 | 145 | if f.Kind() == reflect.String { 146 | envV := os.Getenv(field) 147 | if len(envV) > 0 && f.IsValid() && f.CanSet() { 148 | f.SetString(strings.TrimSuffix(envV, "\n")) // ensure no \n at the end of line that was introduced by loading k8s secrete file 149 | } 150 | os.Setenv(field, f.String()) 151 | } 152 | } 153 | 154 | clusterStr := AssignString(Config.PulsarClusters, "") 155 | AllowedPulsarURLs = strings.Split(clusterStr, ",") 156 | if Config.PulsarBrokerURL != "" { 157 | AllowedPulsarURLs = append([]string{Config.PulsarBrokerURL}, AllowedPulsarURLs...) 158 | } 159 | 160 | superRoleStr := AssignString(Config.SuperRoles, "superuser") 161 | SuperRoles = strings.Split(superRoleStr, ",") 162 | 163 | fmt.Printf("port %s, PbDbType %s, DbRefreshInterval %s, TrustStore %s, DbName %s, DbConnectString %s\n", 164 | Config.PORT, Config.PbDbType, Config.PbDbInterval, Config.TrustStore, Config.DbName, Config.DbConnectionStr) 165 | fmt.Printf("PublicKey %s, PrivateKey %s\n", 166 | Config.PulsarPublicKey, Config.PulsarPrivateKey) 167 | fmt.Printf("PulsarBrokerURL %s, AllowedPulsarURLs %v,PulsarTLSAllowInsecureConnection %s,PulsarTLSValidateHostname %s\n", 168 | Config.PulsarBrokerURL, AllowedPulsarURLs, Config.PulsarTLSAllowInsecureConnection, Config.PulsarTLSValidateHostname) 169 | return &Config 170 | } 171 | 172 | //GetConfig returns a reference to the Configuration 173 | func GetConfig() *Configuration { 174 | return &Config 175 | } 176 | 177 | func logLevel(level string) log.Level { 178 | switch strings.TrimSpace(strings.ToLower(level)) { 179 | case "debug": 180 | return log.DebugLevel 181 | case "warn": 182 | return log.WarnLevel 183 | case "error": 184 | return log.ErrorLevel 185 | case "fatal": 186 | return log.FatalLevel 187 | default: 188 | return log.InfoLevel 189 | } 190 | } 191 | 192 | var jsonPrefix = []byte("{") 193 | 194 | func hasJSONPrefix(buf []byte) bool { 195 | return hasPrefix(buf, jsonPrefix) 196 | } 197 | 198 | // Return true if the first non-whitespace bytes in buf is prefix. 199 | func hasPrefix(buf []byte, prefix []byte) bool { 200 | trim := bytes.TrimLeftFunc(buf, unicode.IsSpace) 201 | return bytes.HasPrefix(trim, prefix) 202 | } 203 | -------------------------------------------------------------------------------- /src/util/main_control.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | //it is control block to determine the main running mode 4 | 5 | // Broker acts Pulsar consumers to send message to webhook 6 | const Broker = "broker" 7 | 8 | // Receiver exposes endpoint to send events as Pulsar producer 9 | const Receiver = "receiver" 10 | 11 | // HTTPOnly exposes all http endpoints including receiver, token server, and rest api 12 | const HTTPOnly = "http" 13 | 14 | // Hybrid mode both broker and webserver 15 | const Hybrid = "hybrid" 16 | 17 | // TokenServer mode serves as a token server only 18 | const TokenServer = "tokenserver" 19 | 20 | // HTTPWithNoRest exposes all http endpoints except rest api 21 | const HTTPWithNoRest = "http2" 22 | 23 | // Rest mode provides a Rest API for webhook management 24 | const Rest = "rest" 25 | 26 | // IsBrokerRequired check if the broker is required 27 | func IsBrokerRequired(mode *string) bool { 28 | return *mode == Broker || *mode == Hybrid 29 | } 30 | 31 | // IsHTTPRouterRequired check whether to initialize http router 32 | func IsHTTPRouterRequired(mode *string) bool { 33 | modes := []string{Hybrid, Receiver, Rest, TokenServer, HTTPOnly, HTTPWithNoRest} 34 | return StrContains(modes, *mode) 35 | } 36 | 37 | // IsBroker check if the mode is broker 38 | func IsBroker(mode *string) bool { 39 | return *mode == Broker 40 | } 41 | 42 | // IsValidMode checks if the mode is supported 43 | func IsValidMode(mode *string) bool { 44 | modes := []string{Broker, Hybrid, Receiver, Rest, TokenServer, HTTPOnly, HTTPWithNoRest} 45 | return StrContains(modes, *mode) 46 | } 47 | -------------------------------------------------------------------------------- /src/util/ttlcache.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | // ExpireCallback is used as a callback on item expiration or when notifying of an item new to the cache 9 | type expireCallback func(key string, value interface{}) 10 | 11 | // Cache is a synchronized map of items that can auto-expire once stale 12 | type Cache struct { 13 | mutex sync.RWMutex 14 | opt CacheOption 15 | items map[string]*item 16 | shutdownSignal chan (chan struct{}) 17 | isShutDown bool 18 | } 19 | 20 | // CacheOption is the optional configuration for Cache 21 | type CacheOption struct { 22 | TTL time.Duration 23 | CleanInterval time.Duration 24 | ExpireCallback expireCallback 25 | } 26 | 27 | // Get gets an object from the cache 28 | func (c *Cache) Get(key string) (interface{}, bool) { 29 | c.mutex.RLock() 30 | item, exists := c.items[key] 31 | c.mutex.RUnlock() 32 | if !exists { 33 | return nil, false 34 | } 35 | 36 | if item.expired() { 37 | c.mutex.Lock() 38 | c.opt.ExpireCallback(key, item.data) 39 | delete(c.items, key) 40 | c.mutex.Unlock() 41 | return nil, false 42 | } 43 | 44 | // item has no expiration 45 | if item.ttl < 0 { 46 | return item.data, true 47 | } 48 | 49 | // update expiry time, puts back to the cache 50 | item.touch() 51 | c.mutex.Lock() 52 | c.items[key] = item 53 | c.mutex.Unlock() 54 | 55 | return item.data, true 56 | } 57 | 58 | // eventLoop name is a disguise. I should convert the lock/unlock to an event loop 59 | func (c *Cache) eventLoop() { 60 | ticker := time.NewTicker(c.opt.CleanInterval) 61 | defer ticker.Stop() 62 | for { 63 | select { 64 | case <-ticker.C: 65 | // RLock is faster than Lock, performant improve to get a slice of keys first 66 | c.mutex.RLock() 67 | keys := make([]string, 0, len(c.items)) 68 | for k := range c.items { 69 | keys = append(keys, k) 70 | } 71 | c.mutex.RUnlock() 72 | 73 | // Lock on individual item scan to reduce the lock section 74 | for _, keyValue := range keys { 75 | c.mutex.Lock() 76 | if item, ok := c.items[keyValue]; ok && item.expired() { 77 | c.opt.ExpireCallback(keyValue, item.data) 78 | delete(c.items, keyValue) 79 | } 80 | c.mutex.Unlock() 81 | } 82 | } 83 | } 84 | } 85 | 86 | // Close is not implmented yet 87 | func (c *Cache) Close() {} 88 | 89 | // Set adds a new item with a gobally set TTL by the cache 90 | func (c *Cache) Set(key string, data interface{}) { 91 | c.SetWithTTL(key, data, 0) 92 | } 93 | 94 | // SetWithTTL adds a new item with individual ttl 95 | func (c *Cache) SetWithTTL(key string, data interface{}, ttl time.Duration) { 96 | if ttl == 0 { 97 | ttl = c.opt.TTL 98 | } 99 | item := newItem(key, data, ttl) 100 | c.mutex.Lock() 101 | c.items[key] = item 102 | c.mutex.Unlock() 103 | } 104 | 105 | // Delete deletes an item with the key specified 106 | func (c *Cache) Delete(key string) { 107 | // deletion is an atomic operation therefore it must Write-mutex the entire seciton 108 | c.mutex.Lock() 109 | defer c.mutex.Unlock() 110 | 111 | if item, ok := c.items[key]; ok { 112 | c.opt.ExpireCallback(key, item.data) 113 | delete(c.items, key) 114 | } 115 | } 116 | 117 | // Count returns the number of items in the cache 118 | func (c *Cache) Count() int { 119 | c.mutex.RLock() 120 | defer c.mutex.RUnlock() 121 | return len(c.items) 122 | } 123 | 124 | // NewCache is a helper to create instance of the Cache struct 125 | func NewCache(option CacheOption) *Cache { 126 | 127 | shutdownChan := make(chan chan struct{}) 128 | 129 | cache := &Cache{ 130 | items: make(map[string]*item), 131 | opt: option, 132 | shutdownSignal: shutdownChan, 133 | isShutDown: false, 134 | } 135 | go cache.eventLoop() 136 | return cache 137 | } 138 | -------------------------------------------------------------------------------- /src/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | "strconv" 12 | "strings" 13 | 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | // ResponseErr - Error struct for Http response 18 | type ResponseErr struct { 19 | Error string `json:"error"` 20 | } 21 | 22 | // NewUUID generates a random UUID according to RFC 4122 23 | func NewUUID() (string, error) { 24 | uuid := make([]byte, 16) 25 | n, err := io.ReadFull(rand.Reader, uuid) 26 | if n != len(uuid) || err != nil { 27 | return "", err 28 | } 29 | // variant bits; see section 4.1.1 30 | uuid[8] = uuid[8]&^0xc0 | 0x80 31 | // version 4 (pseudo-random); see section 4.1.3 32 | uuid[6] = uuid[6]&^0xf0 | 0x40 33 | return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil 34 | } 35 | 36 | // JoinString joins multiple strings 37 | func JoinString(strs ...string) string { 38 | var sb strings.Builder 39 | for _, str := range strs { 40 | sb.WriteString(str) 41 | } 42 | return sb.String() 43 | } 44 | 45 | // ResponseErrorJSON builds a Http response. 46 | func ResponseErrorJSON(e error, w http.ResponseWriter, statusCode int) { 47 | response := ResponseErr{e.Error()} 48 | 49 | jsonResponse, err := json.Marshal(response) 50 | if err != nil { 51 | http.Error(w, err.Error(), http.StatusInternalServerError) 52 | return 53 | } 54 | 55 | w.Header().Set("Content-Type", "application/json") 56 | w.WriteHeader(statusCode) 57 | w.Write(jsonResponse) 58 | } 59 | 60 | // ReceiverHeader parses headers for Pulsar required configuration 61 | func ReceiverHeader(allowedClusters []string, h *http.Header) (token, topicFN, pulsarURL string, err error) { 62 | token = strings.TrimSpace(strings.Replace(h.Get("Authorization"), "Bearer", "", 1)) 63 | topicFN = h.Get("TopicFn") 64 | pulsarURL = h.Get("PulsarUrl") 65 | if len(allowedClusters) > 1 || (len(allowedClusters) == 1 && allowedClusters[0] != "") { 66 | if pulsarURL == "" { 67 | pulsarURL = allowedClusters[0] 68 | } else if !StrContains(allowedClusters, pulsarURL) { 69 | return "", "", "", fmt.Errorf("pulsar cluster %s is not allowed", pulsarURL) 70 | } 71 | } else if pulsarURL == "" { 72 | return "", "", "", fmt.Errorf("missing configured Pulsar URL") 73 | } 74 | return token, topicFN, pulsarURL, nil 75 | } 76 | 77 | // BuildTopicFn builds topic fullname. 78 | func BuildTopicFn(persistent, tenant, namespace, topic string) (string, error) { 79 | if persistent == "persistent" || persistent == "p" { 80 | return "persistent://" + tenant + "/" + namespace + "/" + topic, nil 81 | } else if persistent == "non-persistent" || persistent == "np" { 82 | return "non-persistent://" + tenant + "/" + namespace + "/" + topic, nil 83 | } else { 84 | return "", fmt.Errorf("supported persistent types are persistent, p, non-persistent, np") 85 | } 86 | } 87 | 88 | // AssignString returns the first non-empty string 89 | // It is equivalent the following in Javascript 90 | // var value = val0 || val1 || val2 || default 91 | func AssignString(values ...string) string { 92 | for _, value := range values { 93 | if value != "" { 94 | return value 95 | } 96 | } 97 | return "" 98 | } 99 | 100 | // ReportError logs error 101 | func ReportError(err error) error { 102 | log.Errorf("error %v", err) 103 | return err 104 | } 105 | 106 | // StrContains check if a string is contained in an array of string 107 | func StrContains(strs []string, str string) bool { 108 | for _, v := range strs { 109 | if strings.TrimSpace(v) == strings.TrimSpace(str) { 110 | return true 111 | } 112 | } 113 | return false 114 | } 115 | 116 | // GetEnvInt gets OS environment in integer format with a default if inproper value retrieved 117 | func GetEnvInt(env string, defaultNum int) int { 118 | if i, err := strconv.Atoi(os.Getenv(env)); err == nil { 119 | return i 120 | } 121 | return defaultNum 122 | } 123 | 124 | // StringToBool format various strings to boolean 125 | // strconv.ParseBool only covers `true` and `false` cases 126 | func StringToBool(str string) bool { 127 | s := strings.ToLower(strings.TrimSpace(str)) 128 | 129 | // `1` is true because the default Golang boolean is initialized as false 130 | if s == "true" || s == "yes" || s == "enable" || s == "enabled" || s == "1" || s == "ok" { 131 | return true 132 | } 133 | 134 | return false 135 | } 136 | 137 | // QueryParamString get URL query parameter with a default value 138 | func QueryParamString(params url.Values, name, defaultValue string) string { 139 | if str, ok := params[name]; ok { 140 | return str[0] 141 | } 142 | return defaultValue 143 | } 144 | 145 | // QueryParamInt returns a URL query parameter's value with a default value 146 | func QueryParamInt(params url.Values, name string, defaultValue int) int { 147 | if str, ok := params[name]; ok { 148 | if num, err := strconv.Atoi(str[0]); err == nil { 149 | return num 150 | } 151 | } 152 | return defaultValue 153 | } 154 | 155 | // TokenizeTopicFullName tokenizes a topic full name into persistent, tenant, namespace, and topic name. 156 | func TokenizeTopicFullName(topicFn string) (isPersistent bool, tenant, namespace, topic string, err error) { 157 | var topicRoute string 158 | if strings.HasPrefix(topicFn, "persistent://") { 159 | topicRoute = strings.Replace(topicFn, "persistent://", "", 1) 160 | isPersistent = true 161 | } else if strings.HasPrefix(topicFn, "non-persistent://") { 162 | topicRoute = strings.Replace(topicFn, "non-persistent://", "", 1) 163 | } else { 164 | return false, "", "", "", fmt.Errorf("invalid persistent or non-persistent part") 165 | } 166 | 167 | parts := strings.Split(topicRoute, "/") 168 | if len(parts) == 3 { 169 | return isPersistent, parts[0], parts[1], parts[2], nil 170 | } else if len(parts) == 2 { 171 | return isPersistent, parts[0], parts[1], "", nil 172 | } else { 173 | return false, "", "", "", fmt.Errorf("missing tenant, namespace, or topic name") 174 | } 175 | 176 | } 177 | --------------------------------------------------------------------------------