├── .dockerignore ├── .github ├── FUNDING.yml └── workflows │ ├── docker_edge.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── build_and_run.sh ├── client ├── 404.html ├── css │ └── main.css ├── favicon-16x16.png ├── favicon-32x32.png ├── favicon.ico ├── img │ ├── 404.svg │ ├── benthos_logo.svg │ ├── blobfish.svg │ ├── github_logo.svg │ ├── logo.svg │ └── prof.png ├── index.html ├── js │ ├── editor.js │ ├── js-cookie.js │ └── wasm_exec.js ├── vendor │ └── ace │ │ ├── ace.js │ │ ├── keybinding-emacs.js │ │ ├── keybinding-vim.js │ │ ├── mode-text.js │ │ ├── mode-yaml.js │ │ └── theme-monokai.js └── wasm │ └── benthos-lab.go ├── example ├── data │ └── .keep ├── docker-compose.yaml ├── grafana │ ├── config.monitoring │ └── provisioning │ │ ├── dashboards │ │ ├── dashboard.yml │ │ └── lab.json │ │ └── datasources │ │ └── datasource.yml └── prometheus │ └── prometheus.yml ├── go.mod ├── go.sum ├── lib ├── config │ ├── add.go │ └── normalise.go └── connectors │ ├── round_trip_reader.go │ └── round_trip_reader_test.go ├── logo.svg ├── scripts └── copy_wasm_exec.sh └── server └── benthos-lab └── main.go /.dockerignore: -------------------------------------------------------------------------------- 1 | client/wasm/*.wasm 2 | go.sum 3 | ./benthoslab 4 | ./benthos-lab -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: Jeffail 4 | -------------------------------------------------------------------------------- /.github/workflows/docker_edge.yml: -------------------------------------------------------------------------------- 1 | name: Docker Edge 2 | 3 | on: [ push, pull_request ] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Install Go 11 | uses: actions/setup-go@v2 12 | with: 13 | go-version: 1.16.x 14 | 15 | - name: Check Out Repo 16 | uses: actions/checkout@v2 17 | 18 | - uses: actions/cache@v2 19 | with: 20 | path: | 21 | ~/go/pkg/mod 22 | ~/.cache/go-build 23 | ~/Library/Caches/go-build 24 | %LocalAppData%\go-build 25 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 26 | restore-keys: | 27 | ${{ runner.os }}-go- 28 | 29 | - name: Cache Docker layers 30 | uses: actions/cache@v2 31 | with: 32 | path: /tmp/.buildx-cache 33 | key: ${{ runner.os }}-buildx-${{ github.sha }} 34 | restore-keys: | 35 | ${{ runner.os }}-buildx- 36 | 37 | - name: Deps 38 | run: go mod vendor 39 | 40 | - name: Login to Docker Hub 41 | uses: docker/login-action@v1 42 | with: 43 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 44 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 45 | 46 | - name: Install Buildx 47 | id: buildx 48 | uses: docker/setup-buildx-action@v1 49 | 50 | - name: Build and push 51 | uses: docker/build-push-action@v2 52 | with: 53 | context: ./ 54 | file: ./Dockerfile 55 | builder: ${{ steps.buildx.outputs.name }} 56 | platforms: linux/amd64,linux/arm64 57 | push: true 58 | cache-from: type=local,src=/tmp/.buildx-cache 59 | cache-to: type=local,dest=/tmp/.buildx-cache 60 | tags: jeffail/benthos-lab:edge 61 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | docker: 10 | runs-on: ubuntu-latest 11 | steps: 12 | 13 | - name: Install Go 14 | uses: actions/setup-go@v2 15 | with: 16 | go-version: 1.16.x 17 | 18 | - name: Checkout code 19 | uses: actions/checkout@v2 20 | 21 | - uses: actions/cache@v2 22 | with: 23 | path: | 24 | ~/go/pkg/mod 25 | ~/.cache/go-build 26 | ~/Library/Caches/go-build 27 | %LocalAppData%\go-build 28 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 29 | restore-keys: | 30 | ${{ runner.os }}-go- 31 | 32 | - name: Test 33 | run: go mod vendor 34 | 35 | - name: Cache Docker layers 36 | uses: actions/cache@v2 37 | with: 38 | path: /tmp/.buildx-cache 39 | key: ${{ runner.os }}-buildx-${{ github.sha }} 40 | restore-keys: | 41 | ${{ runner.os }}-buildx- 42 | 43 | - name: Login to Docker Hub 44 | uses: docker/login-action@v1 45 | with: 46 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 47 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 48 | 49 | - name: Install Buildx 50 | id: buildx 51 | uses: docker/setup-buildx-action@v1 52 | 53 | - name: Docker meta 54 | id: docker_meta 55 | uses: crazy-max/ghaction-docker-meta@v1 56 | with: 57 | images: jeffail/benthos-lab 58 | tag-semver: | 59 | {{version}} 60 | {{major}}.{{minor}} 61 | {{major}} 62 | 63 | - name: Build and push 64 | uses: docker/build-push-action@v2 65 | with: 66 | context: ./ 67 | file: ./Dockerfile 68 | builder: ${{ steps.buildx.outputs.name }} 69 | platforms: linux/amd64,linux/arm64 70 | push: true 71 | cache-from: type=local,src=/tmp/.buildx-cache 72 | cache-to: type=local,dest=/tmp/.buildx-cache 73 | tags: ${{ steps.docker_meta.outputs.tags }} 74 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [ push, pull_request ] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Install Go 11 | uses: actions/setup-go@v2 12 | with: 13 | go-version: 1.16.x 14 | 15 | - name: Checkout code 16 | uses: actions/checkout@v2 17 | 18 | - uses: actions/cache@v2 19 | with: 20 | path: | 21 | ~/go/pkg/mod 22 | ~/.cache/go-build 23 | ~/Library/Caches/go-build 24 | %LocalAppData%\go-build 25 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 26 | restore-keys: | 27 | ${{ runner.os }}-go- 28 | 29 | - name: Test 30 | run: go test ./lib/... ./server/... 31 | 32 | # lint: 33 | # runs-on: ubuntu-latest 34 | # steps: 35 | 36 | # - name: Checkout code 37 | # uses: actions/checkout@v2 38 | 39 | # - name: Lint 40 | # uses: golangci/golangci-lint-action@v2 41 | # with: 42 | # version: latest 43 | # args: --timeout 10m server/... lib/... 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | client/wasm/*.wasm 2 | /benthos-lab 3 | /vendor 4 | example/data/dump.rdb 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.16 AS build 2 | 3 | RUN useradd -u 10001 benthos 4 | 5 | WORKDIR /go/src/github.com/benthosdev/benthos-lab/ 6 | COPY . /go/src/github.com/benthosdev/benthos-lab/ 7 | 8 | ENV GO111MODULE on 9 | RUN CGO_ENABLED=0 GOOS=linux go build -mod=vendor -o ./benthos-lab ./server/benthos-lab 10 | RUN GOOS=js GOARCH=wasm go build -ldflags="-s -w" -mod=vendor -o ./client/wasm/benthos-lab.wasm ./client/wasm/benthos-lab.go 11 | 12 | FROM busybox AS package 13 | 14 | LABEL maintainer="Ashley Jeffs " 15 | 16 | WORKDIR / 17 | 18 | COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 19 | COPY --from=build /etc/passwd /etc/passwd 20 | COPY --from=build /go/src/github.com/benthosdev/benthos-lab/benthos-lab . 21 | COPY --from=build /go/src/github.com/benthosdev/benthos-lab/client /var/www 22 | 23 | USER benthos 24 | 25 | EXPOSE 8080 26 | EXPOSE 8443 27 | 28 | ENTRYPOINT ["/benthos-lab"] 29 | 30 | CMD ["--www", "/var/www"] 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Ashley Jeffs 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![benthos-lab](logo.svg "benthos lab") 2 | 3 | [![Build Status](https://cloud.drone.io/api/badges/benthosdev/benthos-lab/status.svg)](https://cloud.drone.io/benthosdev/benthos-lab) 4 | 5 | Benthos Lab is a web application for building, formatting, testing and sharing 6 | [Benthos](https://www.benthos.dev/) pipeline configurations. 7 | 8 | It contains a full version of the Benthos streaming engine compiled to Web 9 | Assembly. This allows it to run natively within the browser sandbox rather than 10 | on a hosted instance, this allows us to be relaxed in regards to allowing 11 | certain processors and connectors to execute. 12 | 13 | ### Install 14 | 15 | Pull a docker image with: 16 | 17 | ``` sh 18 | docker pull jeffail/benthos-lab 19 | ``` 20 | 21 | ### Build 22 | 23 | ``` sh 24 | # Build client 25 | GOOS=js GOARCH=wasm go build -ldflags='-s -w' -o ./client/wasm/benthos-lab.wasm ./client/wasm/benthos-lab.go 26 | 27 | # Install server 28 | go install ./server/benthos-lab 29 | ``` 30 | 31 | Docker: 32 | 33 | ``` sh 34 | go mod vendor 35 | docker build . -t jeffail/benthos-lab:latest 36 | ``` 37 | 38 | ### Run 39 | 40 | ``` sh 41 | cd ./client && benthos-lab 42 | ``` 43 | 44 | Docker: 45 | 46 | ``` sh 47 | docker run --rm -p 8080:8080 jeffail/benthos-lab 48 | ``` 49 | 50 | Then open your browser at `http://localhost:8080`. 51 | -------------------------------------------------------------------------------- /build_and_run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | GOOS=js GOARCH=wasm go build -ldflags="-s -w" -o ./client/wasm/benthos-lab.wasm ./client/wasm/benthos-lab.go 3 | go run ./server/benthos-lab --www ./client --news '[ 4 | {"content":"this is some example news"}, 5 | {"content":"and more content here"} 6 | ]' 7 | -------------------------------------------------------------------------------- /client/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Benthos Lab 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 25 | 26 |
27 |

404 Not Found

28 |

Sorry, looks like this experiment has

29 |

somehow escaped our lab

30 |
31 | Back home 32 |
33 |
34 | sorry, page not found 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /client/css/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: "open sans", sans-serif; 3 | background-color: #272822; 4 | color: #fff; 5 | padding: 0px; 6 | margin: 0px; 7 | font-size: 12pt; 8 | } 9 | 10 | h1, h2, h3, h4 { 11 | color: rgb(209, 252, 124); 12 | } 13 | 14 | h1 > a, h2 > a, h3 > a, h4 > a { 15 | color: rgb(209, 252, 124) !important; 16 | } 17 | 18 | hr { 19 | color: rgb(209, 252, 124, 0.5); 20 | max-width: 240px; 21 | } 22 | 23 | .normal-mode .ace_hidden-cursors .ace_cursor { 24 | background-color: transparent; 25 | border: 1px solid #F92672; 26 | opacity: 0.7; 27 | } 28 | 29 | .normal-mode .ace_cursor { 30 | background-color: rgba(249, 38, 115, 0.65); 31 | } 32 | 33 | #notFoundText { 34 | width: 100%; 35 | text-align: center; 36 | position: fixed; 37 | top: 100px; 38 | font-size: 1.5em; 39 | color: #a2ff91; 40 | } 41 | 42 | #notFoundText > h4 { 43 | padding: 0px; 44 | margin: 0px; 45 | } 46 | 47 | .undecorated { 48 | color: #fff; 49 | text-decoration: none; 50 | } 51 | 52 | .navbar { 53 | position: absolute; 54 | height: 35px; 55 | top: 0; 56 | left: 0; 57 | right: 0; 58 | padding: 5px 15px; 59 | background-color: #33352e; 60 | } 61 | 62 | .navbar > h3 { 63 | display: inline-block; 64 | margin: 0px; 65 | } 66 | 67 | .benthos-img { 68 | position: absolute; 69 | height: 28px; 70 | right: 56px; 71 | top: 8px; 72 | z-index: 10; 73 | } 74 | 75 | .github-img { 76 | position: absolute; 77 | height: 32px; 78 | right: 10px; 79 | top: 6px; 80 | z-index: 10; 81 | } 82 | 83 | #settings { 84 | padding: 10px; 85 | text-align: center; 86 | overflow: auto; 87 | } 88 | 89 | #settings > hr { 90 | margin: 30px auto 0px auto; 91 | } 92 | 93 | #settings > h2 { 94 | margin: 30px 0px 5px 0px; 95 | } 96 | 97 | #settings > h3 { 98 | font-weight: normal; 99 | font-size: 1em; 100 | margin: 0px 0px 20px 0px; 101 | } 102 | 103 | #settings > div { 104 | display: inline-block; 105 | text-align: center; 106 | } 107 | 108 | .setting { 109 | margin-bottom: 10px; 110 | text-align: left; 111 | } 112 | 113 | #editor, #settings { 114 | position: absolute; 115 | top: 50px; 116 | width: 55%; 117 | bottom: 0px; 118 | left: 0px; 119 | } 120 | 121 | #editorOutput { 122 | position: absolute; 123 | top: 50px; 124 | width: 45%; 125 | bottom: 0px; 126 | right: 0px; 127 | border-left: 2px solid #202020; 128 | overflow: auto; 129 | font-family: monospace; 130 | } 131 | 132 | #editorOutput > * { 133 | margin: 0px 10px; 134 | } 135 | 136 | a { 137 | color: #A6E22E; 138 | } 139 | 140 | .logMessage { 141 | color: #E6DB74; 142 | } 143 | 144 | .errorMessage { 145 | color: #F92672; 146 | } 147 | 148 | .lintMessage { 149 | color: #FD971F; 150 | } 151 | 152 | .infoMessage { 153 | color: #66D9EF; 154 | } 155 | 156 | #addComponentWindow { 157 | position: absolute; 158 | top: 50px; 159 | right: 45%; 160 | margin-right: 5px; 161 | text-align: right; 162 | background-color: #272822; 163 | border: 2px solid #5a5a5a; 164 | padding: 3px; 165 | border-radius: 5px; 166 | } 167 | 168 | #addComponentWindow > button { 169 | min-width: 2em; 170 | text-align: center; 171 | margin: 0px; 172 | padding: 2px 0px; 173 | border: 0px; 174 | border-radius: 3px; 175 | font-weight: bold; 176 | cursor: pointer; 177 | transition: background-color 0.5s, border-color 0.5s; 178 | } 179 | 180 | #addComponentSelects > select { 181 | min-width: 160px; 182 | display: block; 183 | margin-top: 5px; 184 | } 185 | 186 | select { 187 | cursor: pointer; 188 | background-color: #33352e; 189 | border: 1px #272822; 190 | color: #fff; 191 | border-radius: 4px; 192 | padding: 5px; 193 | } 194 | 195 | .hidden { 196 | display: none !important; 197 | } 198 | 199 | .tab { 200 | background-color: #464740; 201 | color: #fff; 202 | border: 2px outset #575851; 203 | border-bottom: 0px; 204 | padding: 8px 8px; 205 | border-radius: 6px; 206 | border-bottom-left-radius: 0px; 207 | border-bottom-right-radius: 0px; 208 | position: relative; 209 | bottom: -5px; 210 | cursor: pointer; 211 | } 212 | 213 | .tab:hover { 214 | background-color: #272822; 215 | } 216 | 217 | .openTab { 218 | background-color: #272822; 219 | } 220 | 221 | .btn { 222 | padding: 7px; 223 | border-radius: 3px; 224 | color: #fff; 225 | margin: 0px 2px; 226 | min-width: 80px; 227 | display: inline; 228 | text-decoration: none; 229 | cursor: pointer; 230 | transition: background-color 0.5s, border-color 0.5s; 231 | } 232 | 233 | .button-group { 234 | margin: 0 20px; 235 | display: inline; 236 | } 237 | 238 | .btn-primary { 239 | background-color: rgb(18, 185, 116); 240 | border-color: rgb(18, 185, 116); 241 | } 242 | 243 | .btn-primary:hover { 244 | background-color: rgb(18, 151, 96); 245 | border-color: rgb(18, 151, 96); 246 | } 247 | 248 | .btn-passive { 249 | background-color: rgb(226, 226, 226); 250 | border-color: rgb(226, 226, 226); 251 | } 252 | 253 | .btn-passive:hover { 254 | background-color: rgb(220, 220, 220); 255 | border-color: rgb(220, 220, 220); 256 | } 257 | 258 | .btn-secondary { 259 | background-color: rgb(18, 151, 185); 260 | border-color: rgb(18, 151, 185); 261 | } 262 | 263 | .btn-secondary:hover { 264 | background-color: rgb(17, 112, 136); 265 | border-color: rgb(17, 112, 136); 266 | } 267 | 268 | .btn-disabled { 269 | background-color: rgb(117, 117, 117); 270 | border-color: rgb(117, 117, 117); 271 | } 272 | 273 | .btn-disabled:hover { 274 | background-color: rgb(102, 102, 102); 275 | border-color: rgb(102, 102, 102); 276 | } 277 | 278 | .btn-warning { 279 | background-color: rgb(216, 80, 51); 280 | border-color: rgb(216, 80, 51); 281 | } 282 | 283 | .btn-warning:hover { 284 | background-color: rgb(177, 63, 37); 285 | border-color: rgb(177, 63, 37); 286 | } 287 | 288 | @media (max-width: 90rem) { 289 | #warningGroup { 290 | position: fixed; 291 | right: 0px; 292 | bottom: 30%; 293 | margin-bottom: 10px; 294 | margin-right: 10px; 295 | z-index: 50; 296 | } 297 | 298 | #warningGroup > button { 299 | width: 120px; 300 | display: block; 301 | margin-top: 5px; 302 | } 303 | 304 | #editor, #settings { 305 | width: 100%; 306 | top: 50px; 307 | bottom: 30%; 308 | left: 0px; 309 | right: 0px; 310 | } 311 | 312 | #editorOutput { 313 | top: 70%; 314 | width: 100%; 315 | left: 0px; 316 | bottom: 0px; 317 | right: 0px; 318 | border-top: 2px solid #202020; 319 | border-left: none; 320 | } 321 | 322 | #addComponentWindow { 323 | top: 50px; 324 | right: 0px; 325 | margin-right: 5px; 326 | } 327 | 328 | #profBanner { 329 | display: none; 330 | } 331 | } 332 | 333 | @media (max-width: 60rem) { 334 | #shareGroup { 335 | position: fixed; 336 | right: 0px; 337 | top: 70%; 338 | margin-top: 5px; 339 | margin-right: 10px; 340 | z-index: 50; 341 | } 342 | 343 | #shareGroup > button { 344 | width: 120px; 345 | display: block; 346 | margin-top: 5px; 347 | } 348 | } 349 | 350 | @media (max-width: 42rem) { 351 | #addComponentWindow > * { 352 | min-width: 140px; 353 | } 354 | 355 | #happyGroup { 356 | position: fixed; 357 | right: 0px; 358 | top: 0px; 359 | margin-top: 7px; 360 | margin-right: 10px; 361 | z-index: 50; 362 | } 363 | 364 | .benthos-img { 365 | display: none; 366 | } 367 | .github-img { 368 | display: none; 369 | } 370 | } -------------------------------------------------------------------------------- /client/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benthosdev/benthos-lab/341431ff360f95365f03fa8c7c9ad344dba2cbee/client/favicon-16x16.png -------------------------------------------------------------------------------- /client/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benthosdev/benthos-lab/341431ff360f95365f03fa8c7c9ad344dba2cbee/client/favicon-32x32.png -------------------------------------------------------------------------------- /client/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benthosdev/benthos-lab/341431ff360f95365f03fa8c7c9ad344dba2cbee/client/favicon.ico -------------------------------------------------------------------------------- /client/img/benthos_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xml -------------------------------------------------------------------------------- /client/img/blobfish.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 20 | 22 | 25 | 29 | 33 | 34 | 45 | 48 | 52 | 56 | 57 | 68 | 79 | 90 | 98 | 102 | 103 | 114 | 115 | 134 | 136 | 137 | 139 | image/svg+xml 140 | 142 | 143 | 144 | 145 | 146 | 151 | 158 | 161 | 170 | 179 | 188 | 191 | 197 | 202 | 211 | 217 | 225 | 230 | 236 | 241 | 250 | 256 | 264 | 265 | 266 | 272 | 278 | 284 | 290 | 291 | 292 | -------------------------------------------------------------------------------- /client/img/github_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xml -------------------------------------------------------------------------------- /client/img/prof.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benthosdev/benthos-lab/341431ff360f95365f03fa8c7c9ad344dba2cbee/client/img/prof.png -------------------------------------------------------------------------------- /client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | Benthos Lab 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 64 | 65 |
66 |
67 |
68 | 69 |
70 |
71 |

Attempting to initialise Benthos engine...

72 |

If this message does not disappear then something went wrong, but you can still edit and share your config as 73 | usual.

74 |
75 |
76 | 103 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 172 | 173 | 174 | -------------------------------------------------------------------------------- /client/js/editor.js: -------------------------------------------------------------------------------- 1 | var benthosLab; 2 | 3 | function useSetting(id, onchange) { 4 | var currentVal = window.Cookies.get(id); 5 | 6 | var settingField = document.getElementById(id); 7 | if (typeof (currentVal) === "string" && currentVal.length > 0) { 8 | settingField.value = currentVal; 9 | } 10 | 11 | settingField.onchange = function (e) { 12 | window.Cookies.set(id, e.target.value, { expires: 30 }); 13 | onchange(e.target); 14 | }; 15 | 16 | window.Cookies.set(id, settingField.value, { expires: 30 }); 17 | onchange(settingField); 18 | } 19 | 20 | (function () { 21 | 22 | "use strict"; 23 | 24 | var aboutContent = document.createElement("div"); 25 | aboutContent.innerHTML = `

26 | Welcome to the Benthos Lab, a place where you can experiment with Benthos 27 | pipeline configurations and share them with others. 28 |

`; 29 | 30 | var aboutContent2 = document.createElement("div"); 31 | aboutContent2.innerHTML = `

32 | Edit your pipeline configuration as well as the input data on the left by 33 | changing tabs. When you're ready to try your pipeline click 'Compile'. 34 |

35 | 36 |

37 | If your config compiled successfully you can then execute it with your test data 38 | by clicking 'Execute'. Each line of your input data will be read as a message of 39 | a batch, in order to test with multiple batches add a blank line between each 40 | batch. The output of your pipeline will be printed in this window. 41 |

42 | 43 |

44 | Is your config ugly or incomplete? Click 'Normalise' to have Benthos format it. 45 |

46 | 47 |

48 | Some components might not work within the sandbox of your browser, but you can 49 | still write and share configs that use them. 50 |

`; 51 | 52 | var aboutContent3 = document.createElement("div"); 53 | aboutContent3.innerHTML = `

54 | For more information about Benthos check out the website at 55 | https://www.benthos.dev/. 56 |

`; 57 | 58 | var sessionSettings = {}; 59 | 60 | function useSessionSetting(id, defaultVal, onchange) { 61 | var currentVal = sessionSettings[id]; 62 | 63 | var settingField = document.getElementById(id); 64 | if (typeof (currentVal) === "string" && currentVal.length > 0) { 65 | settingField.value = currentVal; 66 | } else { 67 | settingField.value = defaultVal; 68 | } 69 | 70 | settingField.onchange = function (e) { 71 | sessionSettings[id] = e.target.value; 72 | onchange(e.target); 73 | }; 74 | 75 | sessionSettings[id] = settingField.value; 76 | onchange(settingField); 77 | } 78 | 79 | var configTab, inputTab, settingsTab; 80 | 81 | var openConfig = function () { 82 | if (benthosLab.addProcessor !== undefined) { 83 | document.getElementById("addComponentWindow").classList.remove("hidden"); 84 | } 85 | document.getElementById("editor").classList.remove("hidden"); 86 | document.getElementById("settings").classList.add("hidden"); 87 | configTab.classList.add("openTab"); 88 | inputTab.classList.remove("openTab"); 89 | settingsTab.classList.remove("openTab"); 90 | editor.setSession(configSession); 91 | }; 92 | 93 | var openInput = function () { 94 | document.getElementById("addComponentWindow").classList.add("hidden"); 95 | document.getElementById("editor").classList.remove("hidden"); 96 | document.getElementById("settings").classList.add("hidden"); 97 | configTab.classList.remove("openTab"); 98 | inputTab.classList.add("openTab"); 99 | settingsTab.classList.remove("openTab"); 100 | editor.setSession(inputSession); 101 | }; 102 | 103 | var openSettings = function () { 104 | document.getElementById("addComponentWindow").classList.add("hidden"); 105 | document.getElementById("editor").classList.add("hidden"); 106 | document.getElementById("settings").classList.remove("hidden"); 107 | configTab.classList.remove("openTab"); 108 | inputTab.classList.remove("openTab"); 109 | settingsTab.classList.add("openTab"); 110 | }; 111 | 112 | var initTabs = function () { 113 | configTab = document.getElementById("configTab"); 114 | inputTab = document.getElementById("inputTab"); 115 | settingsTab = document.getElementById("settingsTab"); 116 | configTab.classList.add("openTab"); 117 | 118 | configTab.onclick = openConfig; 119 | inputTab.onclick = openInput; 120 | settingsTab.onclick = openSettings; 121 | 122 | if (window.location.hash === "#input") { 123 | openInput(); 124 | } 125 | }; 126 | 127 | let inputMethod = "batches"; 128 | 129 | window.onload = function () { 130 | if (typeof (model.settings) === "object" && model.settings !== null) { 131 | sessionSettings = model.settings; 132 | } 133 | 134 | initTabs(); 135 | 136 | useSessionSetting("inputMethodSelect", inputMethod, function (e) { 137 | inputMethod = e.value; 138 | }); 139 | 140 | let setWelcomeText = function () { 141 | writeOutputElement(aboutContent); 142 | writeOutputElement(aboutContent2); 143 | writeOutputElement(aboutContent3); 144 | }; 145 | 146 | document.getElementById("aboutBtn").onclick = setWelcomeText; 147 | document.getElementById("clearOutputBtn").onclick = clearOutput; 148 | document.getElementById("shareBtn").onclick = function () { 149 | share(getInput(), getConfig(), setShareURL); 150 | }; 151 | 152 | document.getElementById("normaliseBtn").onclick = function () { 153 | benthosLab.normalise(getConfig(), function (conf) { 154 | setConfig(conf); 155 | }); 156 | }; 157 | 158 | var expandAddCompBtn = document.getElementById("expandAddComponentSelects"); 159 | var collapseAddCompBtn = document.getElementById("collapseAddComponentSelects"); 160 | var selects = document.getElementById("addComponentSelects"); 161 | 162 | expandAddCompBtn.onclick = function () { 163 | expandAddCompBtn.classList.add("hidden"); 164 | collapseAddCompBtn.classList.remove("hidden"); 165 | selects.classList.remove("hidden"); 166 | window.Cookies.set("collapseAddComponents", "false", { expires: 30 }); 167 | }; 168 | collapseAddCompBtn.onclick = function () { 169 | expandAddCompBtn.classList.remove("hidden"); 170 | collapseAddCompBtn.classList.add("hidden"); 171 | selects.classList.add("hidden"); 172 | window.Cookies.set("collapseAddComponents", "true", { expires: 30 }); 173 | }; 174 | 175 | var collapseAddComps = window.Cookies.get("collapseAddComponents"); 176 | if (typeof (collapseAddComps) === "string" && collapseAddComps === "true") { 177 | collapseAddCompBtn.click(); 178 | } else { 179 | expandAddCompBtn.click(); 180 | } 181 | 182 | writeOutputElement(aboutContent); 183 | writeOutputElement(aboutContent3); 184 | }; 185 | 186 | var writeOutput = function (value, style) { 187 | var pre = document.createElement("div"); 188 | if (style) { 189 | pre.classList.add(style); 190 | } 191 | pre.innerText = value; 192 | writeOutputElement(pre); 193 | }; 194 | 195 | var writeOutputElement = function (element) { 196 | var outputDiv = document.getElementById("editorOutput"); 197 | outputDiv.appendChild(element); 198 | outputDiv.scrollTo(0, outputDiv.scrollHeight); 199 | }; 200 | 201 | var setShareURL = function (url) { 202 | var span = document.createElement("span"); 203 | span.innerText = "Session saved at: "; 204 | span.classList.add("infoMessage"); 205 | var a = document.createElement("a"); 206 | a.href = url; 207 | a.innerText = url; 208 | a.target = "_blank"; 209 | var div = document.createElement("div"); 210 | div.appendChild(span); 211 | div.appendChild(a); 212 | writeOutputElement(div); 213 | } 214 | 215 | var share = function (input, config, success) { 216 | var xhr = new XMLHttpRequest(); 217 | xhr.open('POST', '/share'); 218 | xhr.setRequestHeader('Content-Type', 'application/json'); 219 | xhr.onload = function () { 220 | if (xhr.status === 200) { 221 | let shareURL = new URL(window.location.href); 222 | shareURL.pathname = "/l/" + xhr.responseText; 223 | success(shareURL.href); 224 | } else { 225 | writeOutput("Error: Request failed with status: " + xhr.status + "\n", "errorMessage"); 226 | } 227 | }; 228 | xhr.send(JSON.stringify({ 229 | input: input, 230 | config: config, 231 | settings: sessionSettings 232 | })); 233 | }; 234 | 235 | var getNews = function (success) { 236 | var xhr = new XMLHttpRequest(); 237 | xhr.open('GET', '/news'); 238 | xhr.onload = function () { 239 | if (xhr.status === 200) { 240 | success(xhr.responseText); 241 | } 242 | }; 243 | xhr.send(); 244 | }; 245 | 246 | var normaliseViaAPI = function (config, success) { 247 | var xhr = new XMLHttpRequest(); 248 | xhr.open('POST', '/normalise'); 249 | xhr.setRequestHeader('Content-Type', 'text/yaml'); 250 | xhr.onload = function () { 251 | if (xhr.status === 200) { 252 | success(xhr.responseText); 253 | } else { 254 | writeOutput("Error: Normalise request failed with status: " + xhr.status + "\n", "errorMessage"); 255 | } 256 | }; 257 | xhr.send(config); 258 | } 259 | 260 | var setConfig = function (value) { 261 | var session = configSession; 262 | var length = session.getLength(); 263 | var lastLineLength = session.getRowLength(length - 1); 264 | session.remove({ start: { row: 0, column: 0 }, end: { row: length, column: lastLineLength } }); 265 | session.insert({ row: 0, column: 0 }, value); 266 | openConfig(); 267 | }; 268 | 269 | var getConfig = function () { 270 | return configSession.getValue() 271 | }; 272 | 273 | var getInput = function () { 274 | return inputSession.getValue() 275 | }; 276 | 277 | var clearOutput = function () { 278 | var outputDiv = document.getElementById("editorOutput"); 279 | outputDiv.innerText = ""; 280 | }; 281 | 282 | var populateInsertSelect = function (list, addFunc, selectId) { 283 | let s = document.getElementById(selectId); 284 | s.addEventListener("change", function () { 285 | let newConfig = addFunc(this.value, getConfig()) 286 | if (typeof (newConfig) === "string") { 287 | setConfig(newConfig); 288 | } 289 | this.value = ""; 290 | }); 291 | list.forEach(function (v) { 292 | let opt = document.createElement("option"); 293 | opt.text = v; 294 | opt.value = v; 295 | s.add(opt); 296 | }); 297 | }; 298 | 299 | let initLabControls = function () { 300 | document.getElementById("failedText").classList.add("hidden"); 301 | if (configTab == null || configTab.classList.contains("openTab")) { 302 | document.getElementById("addComponentWindow").classList.remove("hidden"); 303 | } 304 | document.getElementById("happyGroup").classList.remove("hidden"); 305 | 306 | populateInsertSelect(benthosLab.getInputs(), benthosLab.addInput, "inputSelect"); 307 | populateInsertSelect(benthosLab.getProcessors(), benthosLab.addProcessor, "procSelect"); 308 | populateInsertSelect(benthosLab.getOutputs(), benthosLab.addOutput, "outputSelect"); 309 | populateInsertSelect(benthosLab.getCaches(), benthosLab.addCache, "cacheSelect"); 310 | populateInsertSelect(benthosLab.getRatelimits(), benthosLab.addRatelimit, "ratelimitSelect"); 311 | 312 | document.getElementById("normaliseBtn").onclick = function () { 313 | benthosLab.normalise(getConfig(), function (result) { 314 | setConfig(result); 315 | }); 316 | }; 317 | 318 | var hasCompiled = false; 319 | var compile = function (onSuccess) { 320 | benthosLab.compile(getConfig(), function () { 321 | hasCompiled = true; 322 | compileBtn.classList.add("btn-disabled"); 323 | compileBtn.classList.remove("btn-primary"); 324 | compileBtn.disabled = true; 325 | if (typeof (onSuccess) === "function") { 326 | onSuccess(); 327 | } 328 | }); 329 | }; 330 | 331 | let executeBtn = document.getElementById("executeBtn"); 332 | executeBtn.onclick = function () { 333 | if (!hasCompiled) { 334 | compile(function () { 335 | benthosLab.execute(inputMethod, getInput()); 336 | }); 337 | } else { 338 | benthosLab.execute(inputMethod, getInput()); 339 | } 340 | }; 341 | 342 | let compileBtn = document.getElementById("compileBtn"); 343 | compileBtn.onclick = compile; 344 | 345 | configSession.on("change", function () { 346 | compileBtn.classList.remove("btn-disabled"); 347 | compileBtn.classList.add("btn-primary"); 348 | compileBtn.disabled = false; 349 | hasCompiled = false; 350 | }) 351 | 352 | writeOutput("Running Benthos version: " + benthosLab.version + "\n", "infoMessage"); 353 | }; 354 | 355 | getNews(function (news) { 356 | var newsContent = document.createElement("div"); 357 | newsContent.style = "margin-bottom: 20px; border-bottom: 1px solid rgb(209, 252, 124); border-top: 1px solid rgb(209, 252, 124);" 358 | 359 | var title = document.createElement("h2"); 360 | title.innerText = "News"; 361 | newsContent.appendChild(title); 362 | 363 | try { 364 | var newsArray = JSON.parse(news); 365 | } catch (e) { 366 | console.error("Failed to parse news: " + e); 367 | return; 368 | } 369 | 370 | var nItems = newsArray.length; 371 | for (var i = 0; i < nItems; i++) { 372 | var p = document.createElement("p"); 373 | p.innerText = newsArray[i].content; 374 | newsContent.appendChild(p); 375 | } 376 | 377 | writeOutputElement(newsContent); 378 | }); 379 | 380 | benthosLab = { 381 | onLoad: initLabControls, 382 | print: writeOutput, 383 | normalise: normaliseViaAPI, 384 | }; 385 | 386 | })(); 387 | 388 | if (!WebAssembly.instantiateStreaming) { 389 | // polyfill 390 | WebAssembly.instantiateStreaming = async (resp, importObject) => { 391 | const source = await (await resp).arrayBuffer(); 392 | return await WebAssembly.instantiate(source, importObject); 393 | }; 394 | } 395 | 396 | const go = new Go(); 397 | 398 | WebAssembly.instantiateStreaming(fetch("/wasm/benthos-lab.wasm"), go.importObject).then((result) => { 399 | go.run(result.instance); 400 | }); 401 | -------------------------------------------------------------------------------- /client/js/js-cookie.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * JavaScript Cookie v2.2.0 3 | * https://github.com/js-cookie/js-cookie 4 | * 5 | * Copyright 2006, 2015 Klaus Hartl & Fagner Brack 6 | * Released under the MIT license 7 | */ 8 | ;(function (factory) { 9 | var registeredInModuleLoader; 10 | if (typeof define === 'function' && define.amd) { 11 | define(factory); 12 | registeredInModuleLoader = true; 13 | } 14 | if (typeof exports === 'object') { 15 | module.exports = factory(); 16 | registeredInModuleLoader = true; 17 | } 18 | if (!registeredInModuleLoader) { 19 | var OldCookies = window.Cookies; 20 | var api = window.Cookies = factory(); 21 | api.noConflict = function () { 22 | window.Cookies = OldCookies; 23 | return api; 24 | }; 25 | } 26 | }(function () { 27 | function extend () { 28 | var i = 0; 29 | var result = {}; 30 | for (; i < arguments.length; i++) { 31 | var attributes = arguments[ i ]; 32 | for (var key in attributes) { 33 | result[key] = attributes[key]; 34 | } 35 | } 36 | return result; 37 | } 38 | 39 | function decode (s) { 40 | return s.replace(/(%[0-9A-Z]{2})+/g, decodeURIComponent); 41 | } 42 | 43 | function init (converter) { 44 | function api() {} 45 | 46 | function set (key, value, attributes) { 47 | if (typeof document === 'undefined') { 48 | return; 49 | } 50 | 51 | attributes = extend({ 52 | path: '/' 53 | }, api.defaults, attributes); 54 | 55 | if (typeof attributes.expires === 'number') { 56 | attributes.expires = new Date(new Date() * 1 + attributes.expires * 864e+5); 57 | } 58 | 59 | // We're using "expires" because "max-age" is not supported by IE 60 | attributes.expires = attributes.expires ? attributes.expires.toUTCString() : ''; 61 | 62 | try { 63 | var result = JSON.stringify(value); 64 | if (/^[\{\[]/.test(result)) { 65 | value = result; 66 | } 67 | } catch (e) {} 68 | 69 | value = converter.write ? 70 | converter.write(value, key) : 71 | encodeURIComponent(String(value)) 72 | .replace(/%(23|24|26|2B|3A|3C|3E|3D|2F|3F|40|5B|5D|5E|60|7B|7D|7C)/g, decodeURIComponent); 73 | 74 | key = encodeURIComponent(String(key)) 75 | .replace(/%(23|24|26|2B|5E|60|7C)/g, decodeURIComponent) 76 | .replace(/[\(\)]/g, escape); 77 | 78 | var stringifiedAttributes = ''; 79 | for (var attributeName in attributes) { 80 | if (!attributes[attributeName]) { 81 | continue; 82 | } 83 | stringifiedAttributes += '; ' + attributeName; 84 | if (attributes[attributeName] === true) { 85 | continue; 86 | } 87 | 88 | // Considers RFC 6265 section 5.2: 89 | // ... 90 | // 3. If the remaining unparsed-attributes contains a %x3B (";") 91 | // character: 92 | // Consume the characters of the unparsed-attributes up to, 93 | // not including, the first %x3B (";") character. 94 | // ... 95 | stringifiedAttributes += '=' + attributes[attributeName].split(';')[0]; 96 | } 97 | 98 | return (document.cookie = key + '=' + value + stringifiedAttributes); 99 | } 100 | 101 | function get (key, json) { 102 | if (typeof document === 'undefined') { 103 | return; 104 | } 105 | 106 | var jar = {}; 107 | // To prevent the for loop in the first place assign an empty array 108 | // in case there are no cookies at all. 109 | var cookies = document.cookie ? document.cookie.split('; ') : []; 110 | var i = 0; 111 | 112 | for (; i < cookies.length; i++) { 113 | var parts = cookies[i].split('='); 114 | var cookie = parts.slice(1).join('='); 115 | 116 | if (!json && cookie.charAt(0) === '"') { 117 | cookie = cookie.slice(1, -1); 118 | } 119 | 120 | try { 121 | var name = decode(parts[0]); 122 | cookie = (converter.read || converter)(cookie, name) || 123 | decode(cookie); 124 | 125 | if (json) { 126 | try { 127 | cookie = JSON.parse(cookie); 128 | } catch (e) {} 129 | } 130 | 131 | jar[name] = cookie; 132 | 133 | if (key === name) { 134 | break; 135 | } 136 | } catch (e) {} 137 | } 138 | 139 | return key ? jar[key] : jar; 140 | } 141 | 142 | api.set = set; 143 | api.get = function (key) { 144 | return get(key, false /* read as raw */); 145 | }; 146 | api.getJSON = function (key) { 147 | return get(key, true /* read as json */); 148 | }; 149 | api.remove = function (key, attributes) { 150 | set(key, '', extend(attributes, { 151 | expires: -1 152 | })); 153 | }; 154 | 155 | api.defaults = {}; 156 | 157 | api.withConverter = init; 158 | 159 | return api; 160 | } 161 | 162 | return init(function () {}); 163 | })); -------------------------------------------------------------------------------- /client/js/wasm_exec.js: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | (() => { 6 | // Map multiple JavaScript environments to a single common API, 7 | // preferring web standards over Node.js API. 8 | // 9 | // Environments considered: 10 | // - Browsers 11 | // - Node.js 12 | // - Electron 13 | // - Parcel 14 | 15 | if (typeof global !== "undefined") { 16 | // global already exists 17 | } else if (typeof window !== "undefined") { 18 | window.global = window; 19 | } else if (typeof self !== "undefined") { 20 | self.global = self; 21 | } else { 22 | throw new Error("cannot export Go (neither global, window nor self is defined)"); 23 | } 24 | 25 | if (!global.require && typeof require !== "undefined") { 26 | global.require = require; 27 | } 28 | 29 | if (!global.fs && global.require) { 30 | global.fs = require("fs"); 31 | } 32 | 33 | const enosys = () => { 34 | const err = new Error("not implemented"); 35 | err.code = "ENOSYS"; 36 | return err; 37 | }; 38 | 39 | if (!global.fs) { 40 | let outputBuf = ""; 41 | global.fs = { 42 | constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused 43 | writeSync(fd, buf) { 44 | outputBuf += decoder.decode(buf); 45 | const nl = outputBuf.lastIndexOf("\n"); 46 | if (nl != -1) { 47 | console.log(outputBuf.substr(0, nl)); 48 | outputBuf = outputBuf.substr(nl + 1); 49 | } 50 | return buf.length; 51 | }, 52 | write(fd, buf, offset, length, position, callback) { 53 | if (offset !== 0 || length !== buf.length || position !== null) { 54 | callback(enosys()); 55 | return; 56 | } 57 | const n = this.writeSync(fd, buf); 58 | callback(null, n); 59 | }, 60 | chmod(path, mode, callback) { callback(enosys()); }, 61 | chown(path, uid, gid, callback) { callback(enosys()); }, 62 | close(fd, callback) { callback(enosys()); }, 63 | fchmod(fd, mode, callback) { callback(enosys()); }, 64 | fchown(fd, uid, gid, callback) { callback(enosys()); }, 65 | fstat(fd, callback) { callback(enosys()); }, 66 | fsync(fd, callback) { callback(null); }, 67 | ftruncate(fd, length, callback) { callback(enosys()); }, 68 | lchown(path, uid, gid, callback) { callback(enosys()); }, 69 | link(path, link, callback) { callback(enosys()); }, 70 | lstat(path, callback) { callback(enosys()); }, 71 | mkdir(path, perm, callback) { callback(enosys()); }, 72 | open(path, flags, mode, callback) { callback(enosys()); }, 73 | read(fd, buffer, offset, length, position, callback) { callback(enosys()); }, 74 | readdir(path, callback) { callback(enosys()); }, 75 | readlink(path, callback) { callback(enosys()); }, 76 | rename(from, to, callback) { callback(enosys()); }, 77 | rmdir(path, callback) { callback(enosys()); }, 78 | stat(path, callback) { callback(enosys()); }, 79 | symlink(path, link, callback) { callback(enosys()); }, 80 | truncate(path, length, callback) { callback(enosys()); }, 81 | unlink(path, callback) { callback(enosys()); }, 82 | utimes(path, atime, mtime, callback) { callback(enosys()); }, 83 | }; 84 | } 85 | 86 | if (!global.process) { 87 | global.process = { 88 | getuid() { return -1; }, 89 | getgid() { return -1; }, 90 | geteuid() { return -1; }, 91 | getegid() { return -1; }, 92 | getgroups() { throw enosys(); }, 93 | pid: -1, 94 | ppid: -1, 95 | umask() { throw enosys(); }, 96 | cwd() { throw enosys(); }, 97 | chdir() { throw enosys(); }, 98 | } 99 | } 100 | 101 | if (!global.crypto) { 102 | const nodeCrypto = require("crypto"); 103 | global.crypto = { 104 | getRandomValues(b) { 105 | nodeCrypto.randomFillSync(b); 106 | }, 107 | }; 108 | } 109 | 110 | if (!global.performance) { 111 | global.performance = { 112 | now() { 113 | const [sec, nsec] = process.hrtime(); 114 | return sec * 1000 + nsec / 1000000; 115 | }, 116 | }; 117 | } 118 | 119 | if (!global.TextEncoder) { 120 | global.TextEncoder = require("util").TextEncoder; 121 | } 122 | 123 | if (!global.TextDecoder) { 124 | global.TextDecoder = require("util").TextDecoder; 125 | } 126 | 127 | // End of polyfills for common API. 128 | 129 | const encoder = new TextEncoder("utf-8"); 130 | const decoder = new TextDecoder("utf-8"); 131 | 132 | global.Go = class { 133 | constructor() { 134 | this.argv = ["js"]; 135 | this.env = {}; 136 | this.exit = (code) => { 137 | if (code !== 0) { 138 | console.warn("exit code:", code); 139 | } 140 | }; 141 | this._exitPromise = new Promise((resolve) => { 142 | this._resolveExitPromise = resolve; 143 | }); 144 | this._pendingEvent = null; 145 | this._scheduledTimeouts = new Map(); 146 | this._nextCallbackTimeoutID = 1; 147 | 148 | const setInt64 = (addr, v) => { 149 | this.mem.setUint32(addr + 0, v, true); 150 | this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true); 151 | } 152 | 153 | const getInt64 = (addr) => { 154 | const low = this.mem.getUint32(addr + 0, true); 155 | const high = this.mem.getInt32(addr + 4, true); 156 | return low + high * 4294967296; 157 | } 158 | 159 | const loadValue = (addr) => { 160 | const f = this.mem.getFloat64(addr, true); 161 | if (f === 0) { 162 | return undefined; 163 | } 164 | if (!isNaN(f)) { 165 | return f; 166 | } 167 | 168 | const id = this.mem.getUint32(addr, true); 169 | return this._values[id]; 170 | } 171 | 172 | const storeValue = (addr, v) => { 173 | const nanHead = 0x7FF80000; 174 | 175 | if (typeof v === "number") { 176 | if (isNaN(v)) { 177 | this.mem.setUint32(addr + 4, nanHead, true); 178 | this.mem.setUint32(addr, 0, true); 179 | return; 180 | } 181 | if (v === 0) { 182 | this.mem.setUint32(addr + 4, nanHead, true); 183 | this.mem.setUint32(addr, 1, true); 184 | return; 185 | } 186 | this.mem.setFloat64(addr, v, true); 187 | return; 188 | } 189 | 190 | switch (v) { 191 | case undefined: 192 | this.mem.setFloat64(addr, 0, true); 193 | return; 194 | case null: 195 | this.mem.setUint32(addr + 4, nanHead, true); 196 | this.mem.setUint32(addr, 2, true); 197 | return; 198 | case true: 199 | this.mem.setUint32(addr + 4, nanHead, true); 200 | this.mem.setUint32(addr, 3, true); 201 | return; 202 | case false: 203 | this.mem.setUint32(addr + 4, nanHead, true); 204 | this.mem.setUint32(addr, 4, true); 205 | return; 206 | } 207 | 208 | let id = this._ids.get(v); 209 | if (id === undefined) { 210 | id = this._idPool.pop(); 211 | if (id === undefined) { 212 | id = this._values.length; 213 | } 214 | this._values[id] = v; 215 | this._goRefCounts[id] = 0; 216 | this._ids.set(v, id); 217 | } 218 | this._goRefCounts[id]++; 219 | let typeFlag = 1; 220 | switch (typeof v) { 221 | case "string": 222 | typeFlag = 2; 223 | break; 224 | case "symbol": 225 | typeFlag = 3; 226 | break; 227 | case "function": 228 | typeFlag = 4; 229 | break; 230 | } 231 | this.mem.setUint32(addr + 4, nanHead | typeFlag, true); 232 | this.mem.setUint32(addr, id, true); 233 | } 234 | 235 | const loadSlice = (addr) => { 236 | const array = getInt64(addr + 0); 237 | const len = getInt64(addr + 8); 238 | return new Uint8Array(this._inst.exports.mem.buffer, array, len); 239 | } 240 | 241 | const loadSliceOfValues = (addr) => { 242 | const array = getInt64(addr + 0); 243 | const len = getInt64(addr + 8); 244 | const a = new Array(len); 245 | for (let i = 0; i < len; i++) { 246 | a[i] = loadValue(array + i * 8); 247 | } 248 | return a; 249 | } 250 | 251 | const loadString = (addr) => { 252 | const saddr = getInt64(addr + 0); 253 | const len = getInt64(addr + 8); 254 | return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len)); 255 | } 256 | 257 | const timeOrigin = Date.now() - performance.now(); 258 | this.importObject = { 259 | go: { 260 | // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) 261 | // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported 262 | // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function). 263 | // This changes the SP, thus we have to update the SP used by the imported function. 264 | 265 | // func wasmExit(code int32) 266 | "runtime.wasmExit": (sp) => { 267 | const code = this.mem.getInt32(sp + 8, true); 268 | this.exited = true; 269 | delete this._inst; 270 | delete this._values; 271 | delete this._goRefCounts; 272 | delete this._ids; 273 | delete this._idPool; 274 | this.exit(code); 275 | }, 276 | 277 | // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) 278 | "runtime.wasmWrite": (sp) => { 279 | const fd = getInt64(sp + 8); 280 | const p = getInt64(sp + 16); 281 | const n = this.mem.getInt32(sp + 24, true); 282 | fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n)); 283 | }, 284 | 285 | // func resetMemoryDataView() 286 | "runtime.resetMemoryDataView": (sp) => { 287 | this.mem = new DataView(this._inst.exports.mem.buffer); 288 | }, 289 | 290 | // func nanotime1() int64 291 | "runtime.nanotime1": (sp) => { 292 | setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000); 293 | }, 294 | 295 | // func walltime1() (sec int64, nsec int32) 296 | "runtime.walltime1": (sp) => { 297 | const msec = (new Date).getTime(); 298 | setInt64(sp + 8, msec / 1000); 299 | this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true); 300 | }, 301 | 302 | // func scheduleTimeoutEvent(delay int64) int32 303 | "runtime.scheduleTimeoutEvent": (sp) => { 304 | const id = this._nextCallbackTimeoutID; 305 | this._nextCallbackTimeoutID++; 306 | this._scheduledTimeouts.set(id, setTimeout( 307 | () => { 308 | this._resume(); 309 | while (this._scheduledTimeouts.has(id)) { 310 | // for some reason Go failed to register the timeout event, log and try again 311 | // (temporary workaround for https://github.com/golang/go/issues/28975) 312 | console.warn("scheduleTimeoutEvent: missed timeout event"); 313 | this._resume(); 314 | } 315 | }, 316 | getInt64(sp + 8) + 1, // setTimeout has been seen to fire up to 1 millisecond early 317 | )); 318 | this.mem.setInt32(sp + 16, id, true); 319 | }, 320 | 321 | // func clearTimeoutEvent(id int32) 322 | "runtime.clearTimeoutEvent": (sp) => { 323 | const id = this.mem.getInt32(sp + 8, true); 324 | clearTimeout(this._scheduledTimeouts.get(id)); 325 | this._scheduledTimeouts.delete(id); 326 | }, 327 | 328 | // func getRandomData(r []byte) 329 | "runtime.getRandomData": (sp) => { 330 | crypto.getRandomValues(loadSlice(sp + 8)); 331 | }, 332 | 333 | // func finalizeRef(v ref) 334 | "syscall/js.finalizeRef": (sp) => { 335 | const id = this.mem.getUint32(sp + 8, true); 336 | this._goRefCounts[id]--; 337 | if (this._goRefCounts[id] === 0) { 338 | const v = this._values[id]; 339 | this._values[id] = null; 340 | this._ids.delete(v); 341 | this._idPool.push(id); 342 | } 343 | }, 344 | 345 | // func stringVal(value string) ref 346 | "syscall/js.stringVal": (sp) => { 347 | storeValue(sp + 24, loadString(sp + 8)); 348 | }, 349 | 350 | // func valueGet(v ref, p string) ref 351 | "syscall/js.valueGet": (sp) => { 352 | const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16)); 353 | sp = this._inst.exports.getsp(); // see comment above 354 | storeValue(sp + 32, result); 355 | }, 356 | 357 | // func valueSet(v ref, p string, x ref) 358 | "syscall/js.valueSet": (sp) => { 359 | Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32)); 360 | }, 361 | 362 | // func valueDelete(v ref, p string) 363 | "syscall/js.valueDelete": (sp) => { 364 | Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16)); 365 | }, 366 | 367 | // func valueIndex(v ref, i int) ref 368 | "syscall/js.valueIndex": (sp) => { 369 | storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16))); 370 | }, 371 | 372 | // valueSetIndex(v ref, i int, x ref) 373 | "syscall/js.valueSetIndex": (sp) => { 374 | Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24)); 375 | }, 376 | 377 | // func valueCall(v ref, m string, args []ref) (ref, bool) 378 | "syscall/js.valueCall": (sp) => { 379 | try { 380 | const v = loadValue(sp + 8); 381 | const m = Reflect.get(v, loadString(sp + 16)); 382 | const args = loadSliceOfValues(sp + 32); 383 | const result = Reflect.apply(m, v, args); 384 | sp = this._inst.exports.getsp(); // see comment above 385 | storeValue(sp + 56, result); 386 | this.mem.setUint8(sp + 64, 1); 387 | } catch (err) { 388 | storeValue(sp + 56, err); 389 | this.mem.setUint8(sp + 64, 0); 390 | } 391 | }, 392 | 393 | // func valueInvoke(v ref, args []ref) (ref, bool) 394 | "syscall/js.valueInvoke": (sp) => { 395 | try { 396 | const v = loadValue(sp + 8); 397 | const args = loadSliceOfValues(sp + 16); 398 | const result = Reflect.apply(v, undefined, args); 399 | sp = this._inst.exports.getsp(); // see comment above 400 | storeValue(sp + 40, result); 401 | this.mem.setUint8(sp + 48, 1); 402 | } catch (err) { 403 | storeValue(sp + 40, err); 404 | this.mem.setUint8(sp + 48, 0); 405 | } 406 | }, 407 | 408 | // func valueNew(v ref, args []ref) (ref, bool) 409 | "syscall/js.valueNew": (sp) => { 410 | try { 411 | const v = loadValue(sp + 8); 412 | const args = loadSliceOfValues(sp + 16); 413 | const result = Reflect.construct(v, args); 414 | sp = this._inst.exports.getsp(); // see comment above 415 | storeValue(sp + 40, result); 416 | this.mem.setUint8(sp + 48, 1); 417 | } catch (err) { 418 | storeValue(sp + 40, err); 419 | this.mem.setUint8(sp + 48, 0); 420 | } 421 | }, 422 | 423 | // func valueLength(v ref) int 424 | "syscall/js.valueLength": (sp) => { 425 | setInt64(sp + 16, parseInt(loadValue(sp + 8).length)); 426 | }, 427 | 428 | // valuePrepareString(v ref) (ref, int) 429 | "syscall/js.valuePrepareString": (sp) => { 430 | const str = encoder.encode(String(loadValue(sp + 8))); 431 | storeValue(sp + 16, str); 432 | setInt64(sp + 24, str.length); 433 | }, 434 | 435 | // valueLoadString(v ref, b []byte) 436 | "syscall/js.valueLoadString": (sp) => { 437 | const str = loadValue(sp + 8); 438 | loadSlice(sp + 16).set(str); 439 | }, 440 | 441 | // func valueInstanceOf(v ref, t ref) bool 442 | "syscall/js.valueInstanceOf": (sp) => { 443 | this.mem.setUint8(sp + 24, loadValue(sp + 8) instanceof loadValue(sp + 16)); 444 | }, 445 | 446 | // func copyBytesToGo(dst []byte, src ref) (int, bool) 447 | "syscall/js.copyBytesToGo": (sp) => { 448 | const dst = loadSlice(sp + 8); 449 | const src = loadValue(sp + 32); 450 | if (!(src instanceof Uint8Array)) { 451 | this.mem.setUint8(sp + 48, 0); 452 | return; 453 | } 454 | const toCopy = src.subarray(0, dst.length); 455 | dst.set(toCopy); 456 | setInt64(sp + 40, toCopy.length); 457 | this.mem.setUint8(sp + 48, 1); 458 | }, 459 | 460 | // func copyBytesToJS(dst ref, src []byte) (int, bool) 461 | "syscall/js.copyBytesToJS": (sp) => { 462 | const dst = loadValue(sp + 8); 463 | const src = loadSlice(sp + 16); 464 | if (!(dst instanceof Uint8Array)) { 465 | this.mem.setUint8(sp + 48, 0); 466 | return; 467 | } 468 | const toCopy = src.subarray(0, dst.length); 469 | dst.set(toCopy); 470 | setInt64(sp + 40, toCopy.length); 471 | this.mem.setUint8(sp + 48, 1); 472 | }, 473 | 474 | "debug": (value) => { 475 | console.log(value); 476 | }, 477 | } 478 | }; 479 | } 480 | 481 | async run(instance) { 482 | this._inst = instance; 483 | this.mem = new DataView(this._inst.exports.mem.buffer); 484 | this._values = [ // JS values that Go currently has references to, indexed by reference id 485 | NaN, 486 | 0, 487 | null, 488 | true, 489 | false, 490 | global, 491 | this, 492 | ]; 493 | this._goRefCounts = []; // number of references that Go has to a JS value, indexed by reference id 494 | this._ids = new Map(); // mapping from JS values to reference ids 495 | this._idPool = []; // unused ids that have been garbage collected 496 | this.exited = false; // whether the Go program has exited 497 | 498 | // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory. 499 | let offset = 4096; 500 | 501 | const strPtr = (str) => { 502 | const ptr = offset; 503 | const bytes = encoder.encode(str + "\0"); 504 | new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes); 505 | offset += bytes.length; 506 | if (offset % 8 !== 0) { 507 | offset += 8 - (offset % 8); 508 | } 509 | return ptr; 510 | }; 511 | 512 | const argc = this.argv.length; 513 | 514 | const argvPtrs = []; 515 | this.argv.forEach((arg) => { 516 | argvPtrs.push(strPtr(arg)); 517 | }); 518 | argvPtrs.push(0); 519 | 520 | const keys = Object.keys(this.env).sort(); 521 | keys.forEach((key) => { 522 | argvPtrs.push(strPtr(`${key}=${this.env[key]}`)); 523 | }); 524 | argvPtrs.push(0); 525 | 526 | const argv = offset; 527 | argvPtrs.forEach((ptr) => { 528 | this.mem.setUint32(offset, ptr, true); 529 | this.mem.setUint32(offset + 4, 0, true); 530 | offset += 8; 531 | }); 532 | 533 | this._inst.exports.run(argc, argv); 534 | if (this.exited) { 535 | this._resolveExitPromise(); 536 | } 537 | await this._exitPromise; 538 | } 539 | 540 | _resume() { 541 | if (this.exited) { 542 | throw new Error("Go program has already exited"); 543 | } 544 | this._inst.exports.resume(); 545 | if (this.exited) { 546 | this._resolveExitPromise(); 547 | } 548 | } 549 | 550 | _makeFuncWrapper(id) { 551 | const go = this; 552 | return function () { 553 | const event = { id: id, this: this, args: arguments }; 554 | go._pendingEvent = event; 555 | go._resume(); 556 | return event.result; 557 | }; 558 | } 559 | } 560 | 561 | if ( 562 | global.require && 563 | global.require.main === module && 564 | global.process && 565 | global.process.versions && 566 | !global.process.versions.electron 567 | ) { 568 | if (process.argv.length < 3) { 569 | console.error("usage: go_js_wasm_exec [wasm binary] [arguments]"); 570 | process.exit(1); 571 | } 572 | 573 | const go = new Go(); 574 | go.argv = process.argv.slice(2); 575 | go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env); 576 | go.exit = process.exit; 577 | WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => { 578 | process.on("exit", (code) => { // Node.js exits if no event handler is pending 579 | if (code === 0 && !go.exited) { 580 | // deadlock, make Go print error and stack traces 581 | go._pendingEvent = { id: 0 }; 582 | go._resume(); 583 | } 584 | }); 585 | return go.run(result.instance); 586 | }).catch((err) => { 587 | console.error(err); 588 | process.exit(1); 589 | }); 590 | } 591 | })(); 592 | -------------------------------------------------------------------------------- /client/vendor/ace/keybinding-emacs.js: -------------------------------------------------------------------------------- 1 | ace.define("ace/occur",["require","exports","module","ace/lib/oop","ace/range","ace/search","ace/edit_session","ace/search_highlight","ace/lib/dom"],function(e,t,n){"use strict";function a(){}var r=e("./lib/oop"),i=e("./range").Range,s=e("./search").Search,o=e("./edit_session").EditSession,u=e("./search_highlight").SearchHighlight;r.inherits(a,s),function(){this.enter=function(e,t){if(!t.needle)return!1;var n=e.getCursorPosition();this.displayOccurContent(e,t);var r=this.originalToOccurPosition(e.session,n);return e.moveCursorToPosition(r),!0},this.exit=function(e,t){var n=t.translatePosition&&e.getCursorPosition(),r=n&&this.occurToOriginalPosition(e.session,n);return this.displayOriginalContent(e),r&&e.moveCursorToPosition(r),!0},this.highlight=function(e,t){var n=e.$occurHighlight=e.$occurHighlight||e.addDynamicMarker(new u(null,"ace_occur-highlight","text"));n.setRegexp(t),e._emit("changeBackMarker")},this.displayOccurContent=function(e,t){this.$originalSession=e.session;var n=this.matchingLines(e.session,t),r=n.map(function(e){return e.content}),i=new o(r.join("\n"));i.$occur=this,i.$occurMatchingLines=n,e.setSession(i),this.$useEmacsStyleLineStart=this.$originalSession.$useEmacsStyleLineStart,i.$useEmacsStyleLineStart=this.$useEmacsStyleLineStart,this.highlight(i,t.re),i._emit("changeBackMarker")},this.displayOriginalContent=function(e){e.setSession(this.$originalSession),this.$originalSession.$useEmacsStyleLineStart=this.$useEmacsStyleLineStart},this.originalToOccurPosition=function(e,t){var n=e.$occurMatchingLines,r={row:0,column:0};if(!n)return r;for(var i=0;i30&&this.$data.shift()},append:function(e){var t=this.$data.length-1,n=this.$data[t]||"";e&&(n+=e),n&&(this.$data[t]=n)},get:function(e){return e=e||1,this.$data.slice(this.$data.length-e,this.$data.length).reverse().join("\n")},pop:function(){return this.$data.length>1&&this.$data.pop(),this.get()},rotate:function(){return this.$data.unshift(this.$data.pop()),this.get()}}}); (function() { 2 | ace.require(["ace/keyboard/emacs"], function(m) { 3 | if (typeof module == "object" && typeof exports == "object" && module) { 4 | module.exports = m; 5 | } 6 | }); 7 | })(); 8 | -------------------------------------------------------------------------------- /client/vendor/ace/mode-text.js: -------------------------------------------------------------------------------- 1 | ; (function() { 2 | ace.require(["ace/mode/text"], function(m) { 3 | if (typeof module == "object" && typeof exports == "object" && module) { 4 | module.exports = m; 5 | } 6 | }); 7 | })(); 8 | -------------------------------------------------------------------------------- /client/vendor/ace/mode-yaml.js: -------------------------------------------------------------------------------- 1 | ace.define("ace/mode/yaml_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"],function(e,t,n){"use strict";var r=e("../lib/oop"),i=e("./text_highlight_rules").TextHighlightRules,s=function(){this.$rules={start:[{token:"comment",regex:"#.*$"},{token:"list.markup",regex:/^(?:-{3}|\.{3})\s*(?=#|$)/},{token:"list.markup",regex:/^\s*[\-?](?:$|\s)/},{token:"constant",regex:"!![\\w//]+"},{token:"constant.language",regex:"[&\\*][a-zA-Z0-9-_]+"},{token:["meta.tag","keyword"],regex:/^(\s*\w.*?)(:(?=\s|$))/},{token:["meta.tag","keyword"],regex:/(\w+?)(\s*:(?=\s|$))/},{token:"keyword.operator",regex:"<<\\w*:\\w*"},{token:"keyword.operator",regex:"-\\s*(?=[{])"},{token:"string",regex:'["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'},{token:"string",regex:/[|>][-+\d]*(?:$|\s+(?:$|#))/,onMatch:function(e,t,n,r){r=r.replace(/ #.*/,"");var i=/^ *((:\s*)?-(\s*[^|>])?)?/.exec(r)[0].replace(/\S\s*$/,"").length,s=parseInt(/\d+[\s+-]*$/.exec(r));return s?(i+=s-1,this.next="mlString"):this.next="mlStringPre",n.length?(n[0]=this.next,n[1]=i):(n.push(this.next),n.push(i)),this.token},next:"mlString"},{token:"string",regex:"['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"},{token:"constant.numeric",regex:/(\b|[+\-\.])[\d_]+(?:(?:\.[\d_]*)?(?:[eE][+\-]?[\d_]+)?)(?=[^\d-\w]|$)/},{token:"constant.numeric",regex:/[+\-]?\.inf\b|NaN\b|0x[\dA-Fa-f_]+|0b[10_]+/},{token:"constant.language.boolean",regex:"\\b(?:true|false|TRUE|FALSE|True|False|yes|no)\\b"},{token:"paren.lparen",regex:"[[({]"},{token:"paren.rparen",regex:"[\\])}]"},{token:"text",regex:/[^\s,:\[\]\{\}]+/}],mlStringPre:[{token:"indent",regex:/^ *$/},{token:"indent",regex:/^ */,onMatch:function(e,t,n){var r=n[1];return r>=e.length?(this.next="start",n.shift(),n.shift()):(n[1]=e.length-1,this.next=n[0]="mlString"),this.token},next:"mlString"},{defaultToken:"string"}],mlString:[{token:"indent",regex:/^ *$/},{token:"indent",regex:/^ */,onMatch:function(e,t,n){var r=n[1];return r>=e.length?(this.next="start",n.splice(0)):this.next="mlString",this.token},next:"mlString"},{token:"string",regex:".+"}]},this.normalizeRules()};r.inherits(s,i),t.YamlHighlightRules=s}),ace.define("ace/mode/matching_brace_outdent",["require","exports","module","ace/range"],function(e,t,n){"use strict";var r=e("../range").Range,i=function(){};(function(){this.checkOutdent=function(e,t){return/^\s+$/.test(e)?/^\s*\}/.test(t):!1},this.autoOutdent=function(e,t){var n=e.getLine(t),i=n.match(/^(\s*\})/);if(!i)return 0;var s=i[1].length,o=e.findMatchingBracket({row:t,column:s});if(!o||o.row==t)return 0;var u=this.$getIndent(e.getLine(o.row));e.replace(new r(t,0,t,s-1),u)},this.$getIndent=function(e){return e.match(/^\s*/)[0]}}).call(i.prototype),t.MatchingBraceOutdent=i}),ace.define("ace/mode/folding/coffee",["require","exports","module","ace/lib/oop","ace/mode/folding/fold_mode","ace/range"],function(e,t,n){"use strict";var r=e("../../lib/oop"),i=e("./fold_mode").FoldMode,s=e("../../range").Range,o=t.FoldMode=function(){};r.inherits(o,i),function(){this.getFoldWidgetRange=function(e,t,n){var r=this.indentationBlock(e,n);if(r)return r;var i=/\S/,o=e.getLine(n),u=o.search(i);if(u==-1||o[u]!="#")return;var a=o.length,f=e.getLength(),l=n,c=n;while(++nl){var p=e.getLine(c).length;return new s(l,a,c,p)}},this.getFoldWidget=function(e,t,n){var r=e.getLine(n),i=r.search(/\S/),s=e.getLine(n+1),o=e.getLine(n-1),u=o.search(/\S/),a=s.search(/\S/);if(i==-1)return e.foldWidgets[n-1]=u!=-1&&u 0 { 213 | reportLints(lints) 214 | } 215 | 216 | go reportUsage("compile/success") 217 | writeOutput("Compiled successfully.\n", "infoMessage") 218 | if successFunc.Type() == js.TypeFunction { 219 | successFunc.Invoke() 220 | } 221 | }() 222 | return nil 223 | } 224 | 225 | func execute(this js.Value, args []js.Value) interface{} { 226 | inputMethod := args[0].String() 227 | inputContent := args[1].String() 228 | 229 | inputMsgs := []types.Message{} 230 | 231 | switch inputMethod { 232 | case "batches": 233 | lines := strings.Split(inputContent, "\n") 234 | 235 | inputMsgs = append(inputMsgs, message.New(nil)) 236 | for _, line := range lines { 237 | if len(line) == 0 { 238 | if inputMsgs[len(inputMsgs)-1].Len() > 0 { 239 | inputMsgs = append(inputMsgs, message.New(nil)) 240 | } 241 | continue 242 | } 243 | inputMsgs[len(inputMsgs)-1].Append(message.NewPart([]byte(line))) 244 | } 245 | case "messages": 246 | lines := strings.Split(inputContent, "\n") 247 | for _, line := range lines { 248 | inputMsgs = append(inputMsgs, message.New([][]byte{[]byte(line)})) 249 | } 250 | case "message": 251 | inputMsgs = append(inputMsgs, message.New([][]byte{[]byte(inputContent)})) 252 | default: 253 | reportErr("failed to dispatch message: %v\n", fmt.Errorf("unrecognised input method: %v", inputMethod)) 254 | go reportUsage("execute/failed") 255 | return nil 256 | } 257 | 258 | go reportUsage("execute/success") 259 | go state.SendAll(inputMsgs) 260 | return nil 261 | } 262 | 263 | //------------------------------------------------------------------------------ 264 | 265 | type logWriter struct{} 266 | 267 | func (l logWriter) Printf(format string, v ...interface{}) { 268 | writeOutput("Log: "+fmt.Sprintf(format, v...), "logMessage") 269 | } 270 | 271 | func (l logWriter) Println(v ...interface{}) { 272 | if str, ok := v[0].(string); ok { 273 | writeOutput("Log: "+fmt.Sprintf(str, v[1:]...)+"\n", "logMessage") 274 | } else { 275 | writeOutput("Log: "+fmt.Sprintf("%v\n", v), "logMessage") 276 | } 277 | } 278 | 279 | //------------------------------------------------------------------------------ 280 | 281 | func normalise(this js.Value, args []js.Value) interface{} { 282 | contents := args[0].String() 283 | conf, err := labConfig.Unmarshal(contents) 284 | if err != nil { 285 | reportErr("failed to create pipeline: %v\n", err) 286 | go reportUsage("normalise/failed") 287 | return nil 288 | } 289 | 290 | sanitBytes, err := labConfig.Marshal(conf) 291 | if err != nil { 292 | reportErr("failed to normalise config: %v\n", err) 293 | go reportUsage("normalise/failed") 294 | return nil 295 | } 296 | 297 | go reportUsage("normalise/success") 298 | if args[1].Type() == js.TypeFunction { 299 | args[1].Invoke(string(sanitBytes)) 300 | } 301 | return nil 302 | } 303 | 304 | //------------------------------------------------------------------------------ 305 | 306 | func addInput(this js.Value, args []js.Value) interface{} { 307 | inputType, contents := args[0].String(), args[1].String() 308 | conf, err := labConfig.Unmarshal(contents) 309 | if err != nil { 310 | reportErr("Failed to unmarshal current config: %v\n", err) 311 | return nil 312 | } 313 | 314 | if err := labConfig.AddInput(inputType, &conf); err != nil { 315 | reportErr("Failed to add input: %v\n", err) 316 | return nil 317 | } 318 | 319 | resultBytes, err := labConfig.Marshal(conf) 320 | if err != nil { 321 | reportErr("failed to normalise config: %v\n", err) 322 | return nil 323 | } 324 | return string(resultBytes) 325 | } 326 | 327 | func addProcessor(this js.Value, args []js.Value) interface{} { 328 | procType, contents := args[0].String(), args[1].String() 329 | conf, err := labConfig.Unmarshal(contents) 330 | if err != nil { 331 | reportErr("Failed to unmarshal current config: %v\n", err) 332 | return nil 333 | } 334 | 335 | if err := labConfig.AddProcessor(procType, &conf); err != nil { 336 | reportErr("Failed to add processor: %v\n", err) 337 | return nil 338 | } 339 | 340 | resultBytes, err := labConfig.Marshal(conf) 341 | if err != nil { 342 | reportErr("failed to normalise config: %v\n", err) 343 | return nil 344 | } 345 | return string(resultBytes) 346 | } 347 | 348 | func addOutput(this js.Value, args []js.Value) interface{} { 349 | outputType, contents := args[0].String(), args[1].String() 350 | conf, err := labConfig.Unmarshal(contents) 351 | if err != nil { 352 | reportErr("Failed to unmarshal current config: %v\n", err) 353 | return nil 354 | } 355 | 356 | if err := labConfig.AddOutput(outputType, &conf); err != nil { 357 | reportErr("Failed to add output: %v\n", err) 358 | return nil 359 | } 360 | 361 | resultBytes, err := labConfig.Marshal(conf) 362 | if err != nil { 363 | reportErr("failed to normalise config: %v\n", err) 364 | return nil 365 | } 366 | return string(resultBytes) 367 | } 368 | 369 | func addCache(this js.Value, args []js.Value) interface{} { 370 | procType, contents := args[0].String(), args[1].String() 371 | conf, err := labConfig.Unmarshal(contents) 372 | if err != nil { 373 | reportErr("Failed to unmarshal current config: %v\n", err) 374 | return nil 375 | } 376 | 377 | if err := labConfig.AddCache(procType, &conf); err != nil { 378 | reportErr("Failed to add cache: %v\n", err) 379 | return nil 380 | } 381 | 382 | resultBytes, err := labConfig.Marshal(conf) 383 | if err != nil { 384 | reportErr("failed to normalise config: %v\n", err) 385 | return nil 386 | } 387 | 388 | return string(resultBytes) 389 | } 390 | 391 | func addRatelimit(this js.Value, args []js.Value) interface{} { 392 | procType, contents := args[0].String(), args[1].String() 393 | conf, err := labConfig.Unmarshal(contents) 394 | if err != nil { 395 | reportErr("Failed to unmarshal current config: %v\n", err) 396 | return nil 397 | } 398 | 399 | if err := labConfig.AddRatelimit(procType, &conf); err != nil { 400 | reportErr("Failed to add cache: %v\n", err) 401 | return nil 402 | } 403 | 404 | resultBytes, err := labConfig.Marshal(conf) 405 | if err != nil { 406 | reportErr("failed to normalise config: %v\n", err) 407 | return nil 408 | } 409 | 410 | return string(resultBytes) 411 | } 412 | 413 | //------------------------------------------------------------------------------ 414 | 415 | var statusDeprecated = "deprecated" 416 | 417 | func getInputs(this js.Value, args []js.Value) interface{} { 418 | inputs := []string{"benthos_lab"} 419 | for k, v := range input.Constructors { 420 | if string(v.Status) != statusDeprecated { 421 | inputs = append(inputs, k) 422 | } 423 | } 424 | sort.Strings(inputs) 425 | generic := make([]interface{}, len(inputs)) 426 | for i, v := range inputs { 427 | generic[i] = v 428 | } 429 | return generic 430 | } 431 | 432 | func getProcessors(this js.Value, args []js.Value) interface{} { 433 | procs := []string{} 434 | for k, v := range processor.Constructors { 435 | if string(v.Status) != statusDeprecated { 436 | procs = append(procs, k) 437 | } 438 | } 439 | sort.Strings(procs) 440 | generic := make([]interface{}, len(procs)) 441 | for i, v := range procs { 442 | generic[i] = v 443 | } 444 | return generic 445 | } 446 | 447 | func getOutputs(this js.Value, args []js.Value) interface{} { 448 | outputs := []string{"benthos_lab"} 449 | for k, v := range output.Constructors { 450 | if string(v.Status) != statusDeprecated { 451 | outputs = append(outputs, k) 452 | } 453 | } 454 | sort.Strings(outputs) 455 | generic := make([]interface{}, len(outputs)) 456 | for i, v := range outputs { 457 | generic[i] = v 458 | } 459 | return generic 460 | } 461 | 462 | func getCaches(this js.Value, args []js.Value) interface{} { 463 | caches := []string{} 464 | for k := range cache.Constructors { 465 | caches = append(caches, k) 466 | } 467 | sort.Strings(caches) 468 | generic := make([]interface{}, len(caches)) 469 | for i, v := range caches { 470 | generic[i] = v 471 | } 472 | return generic 473 | } 474 | 475 | func getRatelimits(this js.Value, args []js.Value) interface{} { 476 | ratelimits := []string{} 477 | for k := range ratelimit.Constructors { 478 | ratelimits = append(ratelimits, k) 479 | } 480 | sort.Strings(ratelimits) 481 | generic := make([]interface{}, len(ratelimits)) 482 | for i, v := range ratelimits { 483 | generic[i] = v 484 | } 485 | return generic 486 | } 487 | 488 | //------------------------------------------------------------------------------ 489 | 490 | var onLoad func() 491 | 492 | func registerFunctions() func() { 493 | benthosLab := js.Global().Get("benthosLab") 494 | if benthosLab.Type() == js.TypeUndefined { 495 | benthosLab = js.ValueOf(map[string]interface{}{}) 496 | js.Global().Set("benthosLab", benthosLab) 497 | } 498 | 499 | version := "Unknown" 500 | info, ok := debug.ReadBuildInfo() 501 | if ok { 502 | for _, mod := range info.Deps { 503 | if mod.Path == "github.com/Jeffail/benthos/v3" { 504 | version = mod.Version 505 | } 506 | } 507 | } 508 | benthosLab.Set("version", version) 509 | 510 | if print := benthosLab.Get("print"); print.Type() == js.TypeFunction { 511 | writeFunc = print 512 | } else { 513 | writeFunc = js.Global().Get("console").Get("log") 514 | benthosLab.Set("print", writeFunc) 515 | } 516 | 517 | if onLoadVal := benthosLab.Get("onLoad"); onLoadVal.Type() == js.TypeFunction { 518 | onLoad = func() { 519 | onLoadVal.Invoke() 520 | } 521 | } else { 522 | onLoad = func() {} 523 | } 524 | 525 | var fields []string 526 | var funcs []js.Func 527 | addLabFunction := func(name string, fn js.Func) { 528 | funcs = append(funcs, fn) 529 | fields = append(fields, name) 530 | benthosLab.Set(name, fn) 531 | } 532 | 533 | addLabFunction("getInputs", js.FuncOf(getInputs)) 534 | addLabFunction("getProcessors", js.FuncOf(getProcessors)) 535 | addLabFunction("getOutputs", js.FuncOf(getOutputs)) 536 | addLabFunction("getCaches", js.FuncOf(getCaches)) 537 | addLabFunction("getRatelimits", js.FuncOf(getRatelimits)) 538 | addLabFunction("addInput", js.FuncOf(addInput)) 539 | addLabFunction("addProcessor", js.FuncOf(addProcessor)) 540 | addLabFunction("addOutput", js.FuncOf(addOutput)) 541 | addLabFunction("addCache", js.FuncOf(addCache)) 542 | addLabFunction("addRatelimit", js.FuncOf(addRatelimit)) 543 | addLabFunction("normalise", js.FuncOf(normalise)) 544 | addLabFunction("compile", js.FuncOf(compile)) 545 | addLabFunction("execute", js.FuncOf(execute)) 546 | 547 | return func() { 548 | for _, field := range fields { 549 | benthosLab.Set(field, js.Undefined()) 550 | } 551 | for _, fn := range funcs { 552 | fn.Release() 553 | } 554 | } 555 | } 556 | 557 | func main() { 558 | c := make(chan struct{}, 0) 559 | 560 | defer registerConnectors()() 561 | defer registerFunctions()() 562 | 563 | println("WASM Benthos Initialized") 564 | onLoad() 565 | 566 | js.Global().Call("addEventListener", "beforeunload", js.FuncOf(func(this js.Value, args []js.Value) interface{} { 567 | c <- struct{}{} 568 | return nil 569 | })) 570 | 571 | <-c 572 | } 573 | 574 | //------------------------------------------------------------------------------ 575 | -------------------------------------------------------------------------------- /example/data/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benthosdev/benthos-lab/341431ff360f95365f03fa8c7c9ad344dba2cbee/example/data/.keep -------------------------------------------------------------------------------- /example/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | volumes: 4 | prometheus_data: {} 5 | grafana_data: {} 6 | 7 | services: 8 | prometheus: 9 | image: prom/prometheus 10 | volumes: 11 | - ./prometheus/:/etc/prometheus/ 12 | - prometheus_data:/prometheus 13 | command: 14 | - '--config.file=/etc/prometheus/prometheus.yml' 15 | - '--storage.tsdb.path=/prometheus' 16 | - '--web.console.libraries=/usr/share/prometheus/console_libraries' 17 | - '--web.console.templates=/usr/share/prometheus/consoles' 18 | ports: 19 | - 9090:9090 20 | 21 | grafana: 22 | image: grafana/grafana 23 | depends_on: 24 | - prometheus 25 | ports: 26 | - 3000:3000 27 | volumes: 28 | - grafana_data:/var/lib/grafana 29 | - ./grafana/provisioning/:/etc/grafana/provisioning/ 30 | env_file: 31 | - ./grafana/config.monitoring 32 | 33 | redis: 34 | image: redis:5 35 | ports: 36 | - "6379:6379" 37 | command: 38 | - '--save 900 1' 39 | - '--save 30 2' 40 | volumes: 41 | - ./data:/data 42 | 43 | benthos-lab: 44 | image: jeffail/benthos-lab 45 | command: 46 | - '--www=/var/www' 47 | - '--redis-url=tcp://redis:6379' 48 | - '--redis-ttl=0s' 49 | ports: 50 | - 8080:8080 51 | - 8081:8081 52 | -------------------------------------------------------------------------------- /example/grafana/config.monitoring: -------------------------------------------------------------------------------- 1 | GF_SECURITY_ADMIN_PASSWORD=admin 2 | GF_USERS_ALLOW_SIGN_UP=false 3 | -------------------------------------------------------------------------------- /example/grafana/provisioning/dashboards/dashboard.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'Prometheus' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | editable: true 10 | options: 11 | path: /etc/grafana/provisioning/dashboards 12 | -------------------------------------------------------------------------------- /example/grafana/provisioning/dashboards/lab.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "links": [], 19 | "panels": [ 20 | { 21 | "aliasColors": {}, 22 | "bars": false, 23 | "dashLength": 10, 24 | "dashes": false, 25 | "datasource": "Prometheus", 26 | "fill": 10, 27 | "gridPos": { 28 | "h": 9, 29 | "w": 12, 30 | "x": 0, 31 | "y": 0 32 | }, 33 | "id": 2, 34 | "legend": { 35 | "avg": false, 36 | "current": false, 37 | "max": false, 38 | "min": false, 39 | "show": true, 40 | "total": false, 41 | "values": false 42 | }, 43 | "lines": true, 44 | "linewidth": 1, 45 | "links": [], 46 | "nullPointMode": "null as zero", 47 | "percentage": false, 48 | "pointradius": 5, 49 | "points": false, 50 | "renderer": "flot", 51 | "seriesOverrides": [], 52 | "spaceLength": 10, 53 | "stack": true, 54 | "steppedLine": false, 55 | "targets": [ 56 | { 57 | "expr": "sum(delta(benthoslab_http_wasm_get_200[1h]))", 58 | "format": "time_series", 59 | "intervalFactor": 1, 60 | "legendFormat": "WASM 200", 61 | "refId": "A" 62 | }, 63 | { 64 | "expr": "sum(delta(benthoslab_http_wasm_get_304[1h]))", 65 | "format": "time_series", 66 | "intervalFactor": 1, 67 | "legendFormat": "WASM 304", 68 | "refId": "B" 69 | }, 70 | { 71 | "expr": "sum(delta(benthoslab_http_wasm_no__gzip[1h]))", 72 | "format": "time_series", 73 | "intervalFactor": 1, 74 | "legendFormat": "WASM No GZIP", 75 | "refId": "C" 76 | }, 77 | { 78 | "expr": "sum(delta(benthoslab_cache_add_success[1h]))", 79 | "format": "time_series", 80 | "intervalFactor": 1, 81 | "legendFormat": "Share", 82 | "refId": "D" 83 | }, 84 | { 85 | "expr": "sum(delta(benthoslab_cache_get_success[1h]))", 86 | "format": "time_series", 87 | "intervalFactor": 1, 88 | "legendFormat": "Get", 89 | "refId": "E" 90 | } 91 | ], 92 | "thresholds": [], 93 | "timeFrom": null, 94 | "timeShift": null, 95 | "title": "Requests", 96 | "tooltip": { 97 | "shared": true, 98 | "sort": 0, 99 | "value_type": "individual" 100 | }, 101 | "type": "graph", 102 | "xaxis": { 103 | "buckets": null, 104 | "mode": "time", 105 | "name": null, 106 | "show": true, 107 | "values": [] 108 | }, 109 | "yaxes": [ 110 | { 111 | "format": "short", 112 | "label": null, 113 | "logBase": 1, 114 | "max": null, 115 | "min": null, 116 | "show": true 117 | }, 118 | { 119 | "format": "short", 120 | "label": null, 121 | "logBase": 1, 122 | "max": null, 123 | "min": null, 124 | "show": true 125 | } 126 | ], 127 | "yaxis": { 128 | "align": false, 129 | "alignLevel": null 130 | } 131 | }, 132 | { 133 | "aliasColors": {}, 134 | "bars": false, 135 | "dashLength": 10, 136 | "dashes": false, 137 | "datasource": "Prometheus", 138 | "fill": 10, 139 | "gridPos": { 140 | "h": 9, 141 | "w": 12, 142 | "x": 12, 143 | "y": 0 144 | }, 145 | "id": 3, 146 | "legend": { 147 | "avg": false, 148 | "current": false, 149 | "max": false, 150 | "min": false, 151 | "show": true, 152 | "total": false, 153 | "values": false 154 | }, 155 | "lines": true, 156 | "linewidth": 1, 157 | "links": [], 158 | "nullPointMode": "null as zero", 159 | "percentage": false, 160 | "pointradius": 5, 161 | "points": false, 162 | "renderer": "flot", 163 | "seriesOverrides": [], 164 | "spaceLength": 10, 165 | "stack": true, 166 | "steppedLine": false, 167 | "targets": [ 168 | { 169 | "expr": "sum(delta(benthoslab_usage_activity[1h]))", 170 | "format": "time_series", 171 | "intervalFactor": 1, 172 | "legendFormat": "Total", 173 | "refId": "A" 174 | }, 175 | { 176 | "expr": "sum(delta(benthoslab_usage_compile_success[1h]))", 177 | "format": "time_series", 178 | "intervalFactor": 1, 179 | "legendFormat": "Compile Success", 180 | "refId": "B" 181 | }, 182 | { 183 | "expr": "sum(delta(benthoslab_usage_compile_failed[1h]))", 184 | "format": "time_series", 185 | "intervalFactor": 1, 186 | "legendFormat": "Compile Failed", 187 | "refId": "C" 188 | }, 189 | { 190 | "expr": "sum(delta(benthoslab_usage_normalise_success[1h]))", 191 | "format": "time_series", 192 | "intervalFactor": 1, 193 | "legendFormat": "Normalise Success", 194 | "refId": "D" 195 | }, 196 | { 197 | "expr": "sum(delta(benthoslab_usage_normalise_failed[1h]))", 198 | "format": "time_series", 199 | "intervalFactor": 1, 200 | "legendFormat": "Normalise Failed", 201 | "refId": "E" 202 | }, 203 | { 204 | "expr": "sum(delta(benthoslab_usage_execute_success[1h]))", 205 | "format": "time_series", 206 | "intervalFactor": 1, 207 | "legendFormat": "Execute Success", 208 | "refId": "F" 209 | }, 210 | { 211 | "expr": "sum(delta(benthoslab_usage_execute_failed[1h]))", 212 | "format": "time_series", 213 | "intervalFactor": 1, 214 | "legendFormat": "Execute Failed", 215 | "refId": "G" 216 | } 217 | ], 218 | "thresholds": [], 219 | "timeFrom": null, 220 | "timeShift": null, 221 | "title": "Usage", 222 | "tooltip": { 223 | "shared": true, 224 | "sort": 0, 225 | "value_type": "individual" 226 | }, 227 | "type": "graph", 228 | "xaxis": { 229 | "buckets": null, 230 | "mode": "time", 231 | "name": null, 232 | "show": true, 233 | "values": [] 234 | }, 235 | "yaxes": [ 236 | { 237 | "format": "short", 238 | "label": null, 239 | "logBase": 1, 240 | "max": null, 241 | "min": null, 242 | "show": true 243 | }, 244 | { 245 | "format": "short", 246 | "label": null, 247 | "logBase": 1, 248 | "max": null, 249 | "min": null, 250 | "show": true 251 | } 252 | ], 253 | "yaxis": { 254 | "align": false, 255 | "alignLevel": null 256 | } 257 | } 258 | ], 259 | "schemaVersion": 16, 260 | "style": "dark", 261 | "tags": [], 262 | "templating": { 263 | "list": [] 264 | }, 265 | "time": { 266 | "from": "now-6h", 267 | "to": "now" 268 | }, 269 | "timepicker": { 270 | "refresh_intervals": [ 271 | "5s", 272 | "10s", 273 | "30s", 274 | "1m", 275 | "5m", 276 | "15m", 277 | "30m", 278 | "1h", 279 | "2h", 280 | "1d" 281 | ], 282 | "time_options": [ 283 | "5m", 284 | "15m", 285 | "1h", 286 | "6h", 287 | "12h", 288 | "24h", 289 | "2d", 290 | "7d", 291 | "30d" 292 | ] 293 | }, 294 | "timezone": "", 295 | "title": "Lab", 296 | "uid": "vuF5UYZWk", 297 | "version": 1 298 | } -------------------------------------------------------------------------------- /example/grafana/provisioning/datasources/datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | # list of datasources that should be deleted from the database 4 | deleteDatasources: 5 | - name: Prometheus 6 | orgId: 1 7 | 8 | datasources: 9 | - name: Prometheus 10 | type: prometheus 11 | access: proxy 12 | orgId: 1 13 | url: http://prometheus:9090 14 | version: 1 15 | editable: true 16 | -------------------------------------------------------------------------------- /example/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | external_labels: 5 | monitor: 'benthos-lab-monitor' 6 | 7 | scrape_configs: 8 | - job_name: 'prometheus' 9 | scrape_interval: 5s 10 | static_configs: 11 | - targets: ['localhost:9090'] 12 | 13 | - job_name: 'benthos-lab' 14 | scrape_interval: 5s 15 | static_configs: 16 | - targets: ['benthos-lab:8081'] -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/benthosdev/benthos-lab 2 | 3 | require ( 4 | github.com/Jeffail/benthos/v3 v3.46.0 5 | golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e 6 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b 7 | ) 8 | 9 | go 1.14 10 | -------------------------------------------------------------------------------- /lib/config/add.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Ashley Jeffs 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package config 22 | 23 | import ( 24 | "fmt" 25 | 26 | "github.com/Jeffail/benthos/v3/lib/cache" 27 | "github.com/Jeffail/benthos/v3/lib/condition" 28 | "github.com/Jeffail/benthos/v3/lib/config" 29 | "github.com/Jeffail/benthos/v3/lib/input" 30 | "github.com/Jeffail/benthos/v3/lib/output" 31 | "github.com/Jeffail/benthos/v3/lib/processor" 32 | "github.com/Jeffail/benthos/v3/lib/ratelimit" 33 | ) 34 | 35 | //------------------------------------------------------------------------------ 36 | 37 | // AddInput inserts a default input of a type to an existing config. 38 | func AddInput(cType string, conf *config.Type) error { 39 | if cType != "benthos_lab" { 40 | if _, ok := input.Constructors[cType]; !ok { 41 | return fmt.Errorf("input type '%v' not recognised", cType) 42 | } 43 | } 44 | inputConf := input.NewConfig() 45 | inputConf.Type = cType 46 | 47 | if conf.Input.Type != input.TypeBroker { 48 | currentInput := conf.Input 49 | brokerInput := input.NewConfig() 50 | brokerInput.Type = input.TypeBroker 51 | brokerInput.Broker.Inputs = append(brokerInput.Broker.Inputs, currentInput) 52 | conf.Input = brokerInput 53 | if cType == input.TypeBroker { 54 | return nil 55 | } 56 | } 57 | conf.Input.Broker.Inputs = append(conf.Input.Broker.Inputs, inputConf) 58 | return nil 59 | } 60 | 61 | // AddProcessor inserts a default processor of a type to an existing config. 62 | func AddProcessor(cType string, conf *config.Type) error { 63 | if _, ok := processor.Constructors[cType]; !ok { 64 | return fmt.Errorf("processor type '%v' not recognised", cType) 65 | } 66 | procConf := processor.NewConfig() 67 | procConf.Type = cType 68 | 69 | conf.Pipeline.Processors = append(conf.Pipeline.Processors, procConf) 70 | return nil 71 | } 72 | 73 | // AddCondition inserts a filter_parts processor with a default condition of a 74 | // type to an existing config. 75 | func AddCondition(cType string, conf *config.Type) error { 76 | if _, ok := condition.Constructors[cType]; !ok { 77 | return fmt.Errorf("condition type '%v' not recognised", cType) 78 | } 79 | condConf := condition.NewConfig() 80 | condConf.Type = cType 81 | 82 | procConf := processor.NewConfig() 83 | procConf.Type = processor.TypeFilterParts 84 | procConf.FilterParts.Config = condConf 85 | 86 | conf.Pipeline.Processors = append(conf.Pipeline.Processors, procConf) 87 | return nil 88 | } 89 | 90 | // AddOutput inserts a default output of a type to an existing config. 91 | func AddOutput(cType string, conf *config.Type) error { 92 | if cType != "benthos_lab" { 93 | if _, ok := output.Constructors[cType]; !ok { 94 | return fmt.Errorf("output type '%v' not recognised", cType) 95 | } 96 | } 97 | outputConf := output.NewConfig() 98 | outputConf.Type = cType 99 | 100 | if conf.Output.Type != output.TypeBroker { 101 | currentOutput := conf.Output 102 | brokerOutput := output.NewConfig() 103 | brokerOutput.Type = output.TypeBroker 104 | brokerOutput.Broker.Outputs = append(brokerOutput.Broker.Outputs, currentOutput) 105 | conf.Output = brokerOutput 106 | if cType == output.TypeBroker { 107 | return nil 108 | } 109 | } 110 | conf.Output.Broker.Outputs = append(conf.Output.Broker.Outputs, outputConf) 111 | return nil 112 | } 113 | 114 | // AddCache inserts a default cache of a type to an existing config. 115 | func AddCache(cType string, conf *config.Type) error { 116 | if _, ok := cache.Constructors[cType]; !ok { 117 | return fmt.Errorf("cache type '%v' not recognised", cType) 118 | } 119 | cacheConf := cache.NewConfig() 120 | cacheConf.Type = cType 121 | 122 | cacheNum := len(conf.Manager.Caches) + len(conf.ResourceCaches) 123 | cacheConf.Label = fmt.Sprintf("example%v", cacheNum) 124 | 125 | conf.ResourceCaches = append(conf.ResourceCaches, cacheConf) 126 | return nil 127 | } 128 | 129 | // AddRatelimit inserts a default rate limit of a type to an existing config. 130 | func AddRatelimit(cType string, conf *config.Type) error { 131 | if _, ok := ratelimit.Constructors[cType]; !ok { 132 | return fmt.Errorf("ratelimit type '%v' not recognised", cType) 133 | } 134 | ratelimitConf := ratelimit.NewConfig() 135 | ratelimitConf.Type = cType 136 | 137 | ratelimitNum := len(conf.Manager.RateLimits) + len(conf.ResourceRateLimits) 138 | ratelimitConf.Label = fmt.Sprintf("example%v", ratelimitNum) 139 | 140 | conf.ResourceRateLimits = append(conf.ResourceRateLimits, ratelimitConf) 141 | return nil 142 | } 143 | 144 | //------------------------------------------------------------------------------ 145 | -------------------------------------------------------------------------------- /lib/config/normalise.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Ashley Jeffs 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package config 22 | 23 | import ( 24 | "github.com/Jeffail/benthos/v3/lib/config" 25 | uconf "github.com/Jeffail/benthos/v3/lib/util/config" 26 | "gopkg.in/yaml.v3" 27 | ) 28 | 29 | //------------------------------------------------------------------------------ 30 | 31 | // New creates a fresh Benthos config with defaults that suit the lab 32 | // environment. 33 | func New() config.Type { 34 | conf := config.New() 35 | conf.Input.Type = "benthos_lab" 36 | conf.Output.Type = "benthos_lab" 37 | return conf 38 | } 39 | 40 | // Unmarshal a config string into a config struct with defaults that suit the 41 | // lab environment. 42 | func Unmarshal(confStr string) (config.Type, error) { 43 | conf := New() 44 | if err := yaml.Unmarshal([]byte(confStr), &conf); err != nil { 45 | return conf, err 46 | } 47 | return conf, nil 48 | } 49 | 50 | type normalisedLabConfig struct { 51 | Input yaml.Node `yaml:"input,omitempty"` 52 | Buffer yaml.Node `yaml:"buffer,omitempty"` 53 | Pipeline yaml.Node `yaml:"pipeline"` 54 | Output yaml.Node `yaml:"output,omitempty"` 55 | Resources yaml.Node `yaml:"resources,omitempty"` 56 | CacheResources yaml.Node `yaml:"cache_resources,omitempty"` 57 | InputResources yaml.Node `yaml:"input_resources,omitempty"` 58 | OutputResources yaml.Node `yaml:"output_resources,omitempty"` 59 | ProcessorResources yaml.Node `yaml:"processor_resources,omitempty"` 60 | RateLimitResources yaml.Node `yaml:"rate_limit_resources,omitempty"` 61 | } 62 | 63 | // Marshal a config struct into a subset of fields relevant to the lab 64 | // environment. 65 | func Marshal(conf config.Type) ([]byte, error) { 66 | node, err := conf.SanitisedV2(config.SanitisedV2Config{ 67 | RemoveTypeField: true, 68 | RemoveDeprecatedFields: false, 69 | }) 70 | if err != nil { 71 | return nil, err 72 | } 73 | nConf := normalisedLabConfig{} 74 | if err := node.Decode(&nConf); err != nil { 75 | return nil, err 76 | } 77 | if conf.Input.Type == "benthos_lab" { 78 | nConf.Input = yaml.Node{} 79 | } 80 | if conf.Output.Type == "benthos_lab" { 81 | nConf.Output = yaml.Node{} 82 | } 83 | if conf.Buffer.Type == "none" { 84 | nConf.Buffer = yaml.Node{} 85 | } 86 | return uconf.MarshalYAML(nConf) 87 | } 88 | 89 | //------------------------------------------------------------------------------ 90 | -------------------------------------------------------------------------------- /lib/connectors/round_trip_reader.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Ashley Jeffs 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package connectors 22 | 23 | import ( 24 | "time" 25 | 26 | "github.com/Jeffail/benthos/v3/lib/message/roundtrip" 27 | "github.com/Jeffail/benthos/v3/lib/types" 28 | ) 29 | 30 | //------------------------------------------------------------------------------ 31 | 32 | // RoundTripReader is a reader implementation that allows you to define a 33 | // closure for producing messages and another for processing the result. 34 | // 35 | // The mechanism for receiving results is implemented using a ResultStore, and 36 | // therefore is subject to pipelines that preserve context and one or more 37 | // outputs destinations storing their results. 38 | type RoundTripReader struct { 39 | read func() (types.Message, error) 40 | processResults func([]types.Message, error) 41 | 42 | store roundtrip.ResultStore 43 | } 44 | 45 | // NewRoundTripReader returns a RoundTripReader. 46 | func NewRoundTripReader( 47 | read func() (types.Message, error), 48 | processResults func([]types.Message, error), 49 | ) *RoundTripReader { 50 | return &RoundTripReader{ 51 | read: read, 52 | processResults: processResults, 53 | store: roundtrip.NewResultStore(), 54 | } 55 | } 56 | 57 | // Connect is a noop. 58 | func (f *RoundTripReader) Connect() error { 59 | return nil 60 | } 61 | 62 | // Acknowledge is a noop. 63 | func (f *RoundTripReader) Acknowledge(err error) error { 64 | msgs := f.store.Get() 65 | f.processResults(msgs, err) 66 | f.store.Clear() 67 | return nil 68 | } 69 | 70 | // Read returns a message result from the provided closure. 71 | func (f *RoundTripReader) Read() (types.Message, error) { 72 | msg, err := f.read() 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | roundtrip.AddResultStore(msg, f.store) 78 | return msg, nil 79 | } 80 | 81 | // CloseAsync is a noop. 82 | func (f *RoundTripReader) CloseAsync() {} 83 | 84 | // WaitForClose is a noop. 85 | func (f *RoundTripReader) WaitForClose(time.Duration) error { 86 | return nil 87 | } 88 | 89 | //------------------------------------------------------------------------------ 90 | -------------------------------------------------------------------------------- /lib/connectors/round_trip_reader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Ashley Jeffs 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package connectors 22 | 23 | import ( 24 | "errors" 25 | "reflect" 26 | "testing" 27 | "time" 28 | 29 | "github.com/Jeffail/benthos/v3/lib/message" 30 | "github.com/Jeffail/benthos/v3/lib/message/roundtrip" 31 | "github.com/Jeffail/benthos/v3/lib/types" 32 | ) 33 | 34 | func TestRoundTripReader(t *testing.T) { 35 | r := NewRoundTripReader(func() (types.Message, error) { 36 | return message.New([][]byte{[]byte("foo"), []byte("bar")}), nil 37 | }, func(msgs []types.Message, err error) { 38 | if err != nil { 39 | t.Error(err) 40 | } 41 | if len(msgs) > 0 { 42 | t.Error("didnt expect a round trip") 43 | } 44 | }) 45 | 46 | if err := r.Connect(); err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | msg, err := r.Read() 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | 55 | if msg.Len() != 2 { 56 | t.Fatalf("Wrong count of messages: %v", msg.Len()) 57 | } 58 | if exp, act := "foo", string(msg.Get(0).Get()); exp != act { 59 | t.Errorf("Wrong message contents: %v != %v", act, exp) 60 | } 61 | if exp, act := "bar", string(msg.Get(1).Get()); exp != act { 62 | t.Errorf("Wrong message contents: %v != %v", act, exp) 63 | } 64 | 65 | if err = r.Acknowledge(nil); err != nil { 66 | t.Error(err) 67 | } 68 | 69 | r.CloseAsync() 70 | if err = r.WaitForClose(time.Second); err != nil { 71 | t.Error(err) 72 | } 73 | } 74 | 75 | func TestRoundTripReaderError(t *testing.T) { 76 | errTest := errors.New("test err") 77 | r := NewRoundTripReader(func() (types.Message, error) { 78 | return nil, errTest 79 | }, func(msgs []types.Message, err error) { 80 | if err != nil { 81 | t.Error(err) 82 | } 83 | if len(msgs) > 0 { 84 | t.Error("didnt expect a round trip") 85 | } 86 | }) 87 | 88 | if err := r.Connect(); err != nil { 89 | t.Fatal(err) 90 | } 91 | 92 | _, err := r.Read() 93 | if err != errTest { 94 | t.Errorf("Expected test error, received: %v", err) 95 | } 96 | 97 | if err = r.Acknowledge(nil); err != nil { 98 | t.Error(err) 99 | } 100 | 101 | r.CloseAsync() 102 | if err = r.WaitForClose(time.Second); err != nil { 103 | t.Error(err) 104 | } 105 | } 106 | 107 | func TestRoundTripReaderResponseError(t *testing.T) { 108 | errTest := errors.New("test err") 109 | var errRes error 110 | r := NewRoundTripReader(func() (types.Message, error) { 111 | return message.New([][]byte{[]byte("foo")}), nil 112 | }, func(msgs []types.Message, err error) { 113 | errRes = err 114 | }) 115 | 116 | if err := r.Connect(); err != nil { 117 | t.Fatal(err) 118 | } 119 | 120 | _, err := r.Read() 121 | if err != nil { 122 | t.Error(err) 123 | } 124 | 125 | if err = r.Acknowledge(errTest); err != nil { 126 | t.Error(err) 127 | } 128 | 129 | if errTest != errRes { 130 | t.Errorf("Wrong error returned: %v != %v", errRes, errTest) 131 | } 132 | 133 | r.CloseAsync() 134 | if err = r.WaitForClose(time.Second); err != nil { 135 | t.Error(err) 136 | } 137 | } 138 | 139 | func TestRoundTripReaderResponse(t *testing.T) { 140 | var results []types.Message 141 | 142 | r := NewRoundTripReader(func() (types.Message, error) { 143 | return message.New([][]byte{[]byte("foo"), []byte("bar")}), nil 144 | }, func(msgs []types.Message, err error) { 145 | if err != nil { 146 | t.Error(err) 147 | } 148 | results = msgs 149 | }) 150 | 151 | msg, err := r.Read() 152 | if err != nil { 153 | t.Fatal(err) 154 | } 155 | 156 | if msg.Len() != 2 { 157 | t.Fatalf("Wrong count of messages: %v", msg.Len()) 158 | } 159 | if exp, act := "foo", string(msg.Get(0).Get()); exp != act { 160 | t.Errorf("Wrong message contents: %v != %v", act, exp) 161 | } 162 | if exp, act := "bar", string(msg.Get(1).Get()); exp != act { 163 | t.Errorf("Wrong message contents: %v != %v", act, exp) 164 | } 165 | 166 | ctx := message.GetContext(msg.Get(0)) 167 | store, ok := ctx.Value(roundtrip.ResultStoreKey).(roundtrip.ResultStore) 168 | if !ok { 169 | t.Fatalf("Wrong type returned from context: %T", store) 170 | } 171 | 172 | store.Add(message.New([][]byte{[]byte("baz")})) 173 | store.Add(message.New([][]byte{[]byte("qux")})) 174 | 175 | if len(results) > 0 { 176 | t.Error("Received premature results") 177 | } 178 | 179 | if err = r.Acknowledge(nil); err != nil { 180 | t.Error(err) 181 | } 182 | 183 | if exp, act := 2, len(results); exp != act { 184 | t.Fatalf("Wrong count of results: %v != %v", act, exp) 185 | } 186 | if exp, act := [][]byte{[]byte("baz")}, message.GetAllBytes(results[0]); !reflect.DeepEqual(exp, act) { 187 | t.Errorf("Wrong result: %v != %v", act, exp) 188 | } 189 | if exp, act := [][]byte{[]byte("qux")}, message.GetAllBytes(results[1]); !reflect.DeepEqual(exp, act) { 190 | t.Errorf("Wrong result: %v != %v", act, exp) 191 | } 192 | 193 | r.CloseAsync() 194 | if err = r.WaitForClose(time.Second); err != nil { 195 | t.Error(err) 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /scripts/copy_wasm_exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" ./client/js/wasm_exec.js -------------------------------------------------------------------------------- /server/benthos-lab/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "crypto/sha256" 7 | "crypto/tls" 8 | "encoding/base64" 9 | "encoding/json" 10 | "flag" 11 | "io" 12 | "io/ioutil" 13 | "net/http" 14 | "os" 15 | "path/filepath" 16 | "regexp" 17 | "strings" 18 | "sync" 19 | "time" 20 | 21 | "github.com/Jeffail/benthos/v3/lib/cache" 22 | "github.com/Jeffail/benthos/v3/lib/config" 23 | "github.com/Jeffail/benthos/v3/lib/log" 24 | "github.com/Jeffail/benthos/v3/lib/metrics" 25 | "github.com/Jeffail/benthos/v3/lib/ratelimit" 26 | "github.com/Jeffail/benthos/v3/lib/types" 27 | labConfig "github.com/benthosdev/benthos-lab/lib/config" 28 | "golang.org/x/crypto/acme" 29 | "golang.org/x/crypto/acme/autocert" 30 | ) 31 | 32 | //------------------------------------------------------------------------------ 33 | 34 | func hijackCode(code int, w http.ResponseWriter, r *http.Request, hijacker http.HandlerFunc) http.ResponseWriter { 35 | return &codeHijacker{ 36 | w: w, 37 | r: r, 38 | code: code, 39 | hijacker: hijacker, 40 | } 41 | } 42 | 43 | type codeHijacker struct { 44 | w http.ResponseWriter 45 | r *http.Request 46 | 47 | code int 48 | hijacked bool 49 | 50 | hijacker http.HandlerFunc 51 | } 52 | 53 | func (c *codeHijacker) Header() http.Header { 54 | return c.w.Header() 55 | } 56 | 57 | func (c *codeHijacker) Write(msg []byte) (int, error) { 58 | if c.hijacked { 59 | c.hijacker(c.w, c.r) 60 | return len(msg), nil 61 | } 62 | return c.w.Write(msg) 63 | } 64 | 65 | func (c *codeHijacker) WriteHeader(statusCode int) { 66 | if statusCode == c.code { 67 | c.hijacked = true 68 | } else { 69 | c.w.WriteHeader(statusCode) 70 | } 71 | } 72 | 73 | //------------------------------------------------------------------------------ 74 | 75 | type benthosLabCache struct { 76 | path string 77 | log log.Modular 78 | 79 | cache []byte 80 | cachedAt time.Time 81 | 82 | sync.RWMutex 83 | } 84 | 85 | func newBenthosLabCache(path string, log log.Modular) *benthosLabCache { 86 | c := benthosLabCache{ 87 | path: path, 88 | log: log, 89 | } 90 | c.read() 91 | go c.loop() 92 | return &c 93 | } 94 | 95 | func (c *benthosLabCache) Get() []byte { 96 | c.RLock() 97 | cache := c.cache 98 | c.RUnlock() 99 | return cache 100 | } 101 | 102 | func (c *benthosLabCache) read() { 103 | finfo, err := os.Stat(c.path) 104 | if err != nil { 105 | c.log.Errorf("Failed to stat benthos-lab.wasm: %v\n", err) 106 | return 107 | } 108 | if finfo.ModTime().After(c.cachedAt) { 109 | c.log.Debugln("Reading modified benthos-lab.wasm") 110 | file, err := os.Open(c.path) 111 | if err != nil { 112 | c.log.Errorf("Failed to open benthos-lab.wasm: %v\n", err) 113 | return 114 | } 115 | var gzipBuf bytes.Buffer 116 | gzipWriter := gzip.NewWriter(&gzipBuf) 117 | if _, err = io.Copy(gzipWriter, file); err != nil { 118 | c.log.Errorf("Failed to compress benthos-lab.wasm: %v\n", err) 119 | return 120 | } 121 | gzipWriter.Close() 122 | c.Lock() 123 | c.cache = gzipBuf.Bytes() 124 | c.cachedAt = finfo.ModTime() 125 | c.Unlock() 126 | } 127 | } 128 | 129 | func (c *benthosLabCache) loop() { 130 | for { 131 | <-time.After(time.Second) 132 | c.read() 133 | } 134 | } 135 | 136 | //------------------------------------------------------------------------------ 137 | 138 | func shareHash(content []byte) []byte { 139 | var buf bytes.Buffer 140 | 141 | hasher := sha256.New() 142 | hasher.Write(content) 143 | 144 | encoder := base64.NewEncoder(base64.URLEncoding, &buf) 145 | encoder.Write(hasher.Sum(nil)) 146 | encoder.Close() 147 | 148 | hashBytes := buf.Bytes() 149 | 150 | hashLen := 11 151 | for hashLen <= len(hashBytes) && hashBytes[hashLen-1] == '_' { 152 | hashLen++ 153 | } 154 | return hashBytes[:hashLen] 155 | } 156 | 157 | func main() { 158 | cacheConf := cache.NewConfig() 159 | ratelimitConf := ratelimit.NewConfig() 160 | ratelimitConf.Type = ratelimit.TypeLocal 161 | ratelimitConf.Local.Count = 100 162 | ratelimitConf.Local.Interval = "1s" 163 | 164 | tlsHost := flag.String( 165 | "auto-tls", "", "Enable automatic HTTPS for a specified host using ACME.", 166 | ) 167 | tlsCache := flag.String( 168 | "cert-dir", "", "An optional directory to cache tls certificates.", 169 | ) 170 | tlsStaging := flag.Bool( 171 | "cert-staging", false, "Whether to use a staging ACME URL instead of a production one when obtaining TLS certificates.", 172 | ) 173 | wwwPath := flag.String( 174 | "www", ".", "Path to the directory of client files to serve", 175 | ) 176 | news := flag.String( 177 | "news", "", `An optional JSON array of news items of the form [{"content":"this is news"}].`, 178 | ) 179 | flag.StringVar( 180 | &cacheConf.Redis.URL, "redis-url", "", "Optional: Redis URL to use for caching", 181 | ) 182 | flag.StringVar( 183 | &cacheConf.Redis.Expiration, "redis-ttl", cacheConf.Redis.Expiration, "Optional: Redis TTL to use for caching", 184 | ) 185 | flag.StringVar( 186 | &cacheConf.DynamoDB.Table, "dynamodb-table", "", "Optional: A DynamoDB table to use for caching", 187 | ) 188 | flag.StringVar( 189 | &cacheConf.DynamoDB.TTL, "dynamodb-ttl", "", "Optional: TTL to use for caching", 190 | ) 191 | flag.StringVar( 192 | &cacheConf.DynamoDB.Region, "dynamodb-region", "eu-west-1", "The AWS region to use when caching with DynamoDB", 193 | ) 194 | metricsTarget := flag.String( 195 | "metrics-target", metrics.TypePrometheus, "How metrics should be exported", 196 | ) 197 | flag.IntVar( 198 | &ratelimitConf.Local.Count, "rate-limit-count", 199 | ratelimitConf.Local.Count, "The count for session access rate limiting", 200 | ) 201 | flag.StringVar( 202 | &ratelimitConf.Local.Interval, "rate-limit-interval", 203 | ratelimitConf.Local.Interval, "The interval for session access rate limiting", 204 | ) 205 | flag.Parse() 206 | 207 | if len(cacheConf.DynamoDB.Table) > 0 { 208 | cacheConf.Type = cache.TypeDynamoDB 209 | cacheConf.DynamoDB.HashKey = "Id" 210 | cacheConf.DynamoDB.DataKey = "Content" 211 | cacheConf.DynamoDB.TTLKey = "TTL" 212 | } else if len(cacheConf.Redis.URL) > 0 { 213 | cacheConf.Type = cache.TypeRedis 214 | } 215 | 216 | logConf := log.NewConfig() 217 | logConf.Prefix = "benthos-lab" 218 | log := log.New(os.Stdout, logConf) 219 | 220 | metricsConf := metrics.NewConfig() 221 | metricsConf.Prometheus.Prefix = "benthoslab" 222 | metricsConf.CloudWatch.Region = cacheConf.DynamoDB.Region 223 | metricsConf.CloudWatch.Namespace = "benthoslab" 224 | metricsConf.CloudWatch.FlushPeriod = "30s" 225 | metricsConf.Type = *metricsTarget 226 | stats, err := metrics.New(metricsConf) 227 | if err != nil { 228 | panic(err) 229 | } 230 | defer stats.Close() 231 | 232 | cacheConf.Memory.TTL = 259200 233 | 234 | var componentMetrics metrics.Type = metrics.Noop() 235 | if *metricsTarget != metrics.TypeCloudWatch { 236 | // Avoid flooding CW with metrics. 237 | componentMetrics = stats 238 | } 239 | cache, err := cache.New(cacheConf, types.DudMgr{}, log.NewModule(".cache"), metrics.Namespaced(componentMetrics, "cache")) 240 | if err != nil { 241 | panic(err) 242 | } 243 | 244 | rlimit, err := ratelimit.New(ratelimitConf, types.DudMgr{}, log.NewModule(".ratelimit"), metrics.Namespaced(componentMetrics, "ratelimit")) 245 | if err != nil { 246 | panic(err) 247 | } 248 | 249 | labCache := newBenthosLabCache(filepath.Join(*wwwPath, "/wasm/benthos-lab.wasm"), log) 250 | 251 | mux := http.NewServeMux() 252 | fileServe := http.FileServer(http.Dir(*wwwPath)) 253 | 254 | httpStats := metrics.Namespaced(stats, "http") 255 | mWASMGet200 := httpStats.GetCounter("wasm.get.200") 256 | mWASMGet304 := httpStats.GetCounter("wasm.get.304") 257 | mWASMGetNoGZIP := httpStats.GetCounter("wasm.no_gzip") 258 | mHTTPNormaliseSucc := stats.GetCounter("usage.normalise_http.success") 259 | mHTTPNormaliseFail := stats.GetCounter("usage.normalise_http.failed") 260 | mShareSucc := stats.GetCounter("usage.share.success") 261 | mShareFail := stats.GetCounter("usage.share.failed") 262 | mActivity := stats.GetCounter("usage.activity") 263 | 264 | makeMetricHandler := func(path string) http.HandlerFunc { 265 | counter := stats.GetCounter(path) 266 | return func(w http.ResponseWriter, r *http.Request) { 267 | counter.Incr(1) 268 | mActivity.Incr(1) 269 | } 270 | } 271 | 272 | notFoundHandler := func(w http.ResponseWriter, r *http.Request) { 273 | w.Header().Del("Content-Type") 274 | w.Header().Set("Content-Type", "text/html") 275 | w.WriteHeader(http.StatusNotFound) 276 | notFoundFile, err := os.Open(filepath.Join(*wwwPath, "/404.html")) 277 | if err != nil { 278 | log.Errorf("Failed to open 404.html: %v\n", err) 279 | w.Write([]byte("Not found")) 280 | return 281 | } 282 | defer notFoundFile.Close() 283 | if _, err = io.Copy(w, notFoundFile); err != nil { 284 | log.Errorf("Failed to write 404.html: %v\n", err) 285 | w.Write([]byte("Not found")) 286 | } 287 | } 288 | 289 | mux.HandleFunc("/usage/compile/success", makeMetricHandler("usage.compile.success")) 290 | mux.HandleFunc("/usage/compile/failed", makeMetricHandler("usage.compile.failed")) 291 | mux.HandleFunc("/usage/execute/success", makeMetricHandler("usage.execute.success")) 292 | mux.HandleFunc("/usage/execute/failed", makeMetricHandler("usage.execute.failed")) 293 | mux.HandleFunc("/usage/normalise/success", makeMetricHandler("usage.normalise.success")) 294 | mux.HandleFunc("/usage/normalise/failed", makeMetricHandler("usage.normalise.failed")) 295 | 296 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 297 | fileServe.ServeHTTP(hijackCode(http.StatusNotFound, w, r, notFoundHandler), r) 298 | }) 299 | 300 | mux.HandleFunc("/news", func(w http.ResponseWriter, r *http.Request) { 301 | if len(*news) == 0 { 302 | w.WriteHeader(http.StatusNoContent) 303 | return 304 | } 305 | w.Write([]byte(*news)) 306 | }) 307 | 308 | mux.HandleFunc("/wasm/benthos-lab.wasm", func(w http.ResponseWriter, r *http.Request) { 309 | if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { 310 | mWASMGetNoGZIP.Incr(1) 311 | fileServe.ServeHTTP(w, r) 312 | return 313 | } 314 | if since := r.Header.Get("If-Modified-Since"); len(since) > 0 { 315 | tSince, err := time.Parse(time.RFC1123, since) 316 | if err != nil { 317 | log.Errorf("Failed to parse time: %v\n", err) 318 | } 319 | if err == nil && labCache.cachedAt.Sub(tSince) < time.Second { 320 | mWASMGet304.Incr(1) 321 | w.WriteHeader(http.StatusNotModified) 322 | return 323 | } 324 | } 325 | mWASMGet200.Incr(1) 326 | w.Header().Set("Content-Encoding", "gzip") 327 | w.Header().Set("Content-Type", "application/wasm") 328 | w.Header().Set("Last-Modified", labCache.cachedAt.UTC().Format(time.RFC1123)) 329 | w.Write(labCache.Get()) 330 | }) 331 | 332 | templateRegexp := regexp.MustCompile(`// BENTHOS LAB START([\n]|.)*// BENTHOS LAB END`) 333 | indexPath := filepath.Join(*wwwPath, "/index.html") 334 | 335 | mux.HandleFunc("/l/", func(w http.ResponseWriter, r *http.Request) { 336 | path := strings.TrimPrefix(r.URL.Path, "/l/") 337 | if len(path) == 0 { 338 | http.Error(w, "Path required", http.StatusBadRequest) 339 | log.Warnf("Bad path: %v\n", path) 340 | return 341 | } 342 | 343 | for { 344 | tout, err := rlimit.Access() 345 | if err != nil { 346 | http.Error(w, "Server failed", http.StatusBadGateway) 347 | log.Errorf("Failed to access rate limit: %v\n", err) 348 | return 349 | } 350 | if tout == 0 { 351 | break 352 | } 353 | select { 354 | case <-time.After(tout): 355 | case <-r.Context().Done(): 356 | http.Error(w, "Timed out", http.StatusRequestTimeout) 357 | return 358 | } 359 | } 360 | 361 | stateBody, err := cache.Get(path) 362 | if err != nil { 363 | if err == types.ErrKeyNotFound { 364 | notFoundHandler(w, r) 365 | } else { 366 | http.Error(w, "Server failed", http.StatusBadGateway) 367 | log.Errorf("Failed to read state: %v\n", err) 368 | } 369 | return 370 | } 371 | 372 | index, err := ioutil.ReadFile(indexPath) 373 | if err != nil { 374 | http.Error(w, "Server failed", http.StatusBadGateway) 375 | log.Errorf("Failed to read index: %v\n", path) 376 | return 377 | } 378 | 379 | index = templateRegexp.ReplaceAllLiteral(index, stateBody) 380 | 381 | w.Header().Set("Content-Type", "text/html") 382 | w.Write(index) 383 | }) 384 | 385 | mux.HandleFunc("/normalise", func(w http.ResponseWriter, r *http.Request) { 386 | mActivity.Incr(1) 387 | if r.Method != "POST" { 388 | http.Error(w, "Method not supported", http.StatusBadRequest) 389 | log.Warnf("Bad method: %v\n", r.Method) 390 | mHTTPNormaliseFail.Incr(1) 391 | return 392 | } 393 | reqBody, err := ioutil.ReadAll(r.Body) 394 | if err != nil { 395 | http.Error(w, "Failed to read body", http.StatusBadRequest) 396 | log.Errorf("Failed to read request body: %v\n", err) 397 | mHTTPNormaliseFail.Incr(1) 398 | return 399 | } 400 | defer r.Body.Close() 401 | 402 | var conf config.Type 403 | if conf, err = labConfig.Unmarshal(string(reqBody)); err != nil { 404 | http.Error(w, "Failed to parse body", http.StatusBadRequest) 405 | log.Errorf("Failed to parse request body: %v\n", err) 406 | mHTTPNormaliseFail.Incr(1) 407 | return 408 | } 409 | 410 | var resBytes []byte 411 | if resBytes, err = labConfig.Marshal(conf); err != nil { 412 | http.Error(w, "Failed to marshal response", http.StatusInternalServerError) 413 | log.Errorf("Failed to marshal response body: %v\n", err) 414 | mHTTPNormaliseFail.Incr(1) 415 | return 416 | } 417 | 418 | mHTTPNormaliseSucc.Incr(1) 419 | w.Write(resBytes) 420 | }) 421 | 422 | mux.HandleFunc("/share", func(w http.ResponseWriter, r *http.Request) { 423 | mActivity.Incr(1) 424 | if r.Method != "POST" { 425 | http.Error(w, "Method not supported", http.StatusBadRequest) 426 | log.Warnf("Bad method: %v\n", r.Method) 427 | mShareFail.Incr(1) 428 | return 429 | } 430 | reqBody, err := ioutil.ReadAll(r.Body) 431 | if err != nil { 432 | http.Error(w, "Failed to read body", http.StatusBadRequest) 433 | log.Errorf("Failed to read request body: %v\n", err) 434 | mShareFail.Incr(1) 435 | return 436 | } 437 | defer r.Body.Close() 438 | 439 | state := struct { 440 | Config string `json:"config"` 441 | Input string `json:"input"` 442 | Settings map[string]string `json:"settings"` 443 | }{ 444 | Settings: map[string]string{}, 445 | } 446 | 447 | if err = json.Unmarshal(reqBody, &state); err != nil { 448 | http.Error(w, "Failed to parse body", http.StatusBadRequest) 449 | log.Errorf("Failed to parse request body: %v\n", err) 450 | mShareFail.Incr(1) 451 | return 452 | } 453 | 454 | if reqBody, err = json.Marshal(state); err != nil { 455 | http.Error(w, "Failed to parse body", http.StatusBadRequest) 456 | log.Errorf("Failed to normalise request body: %v\n", err) 457 | mShareFail.Incr(1) 458 | return 459 | } 460 | 461 | for { 462 | tout, err := rlimit.Access() 463 | if err != nil { 464 | http.Error(w, "Server failed", http.StatusBadGateway) 465 | log.Errorf("Failed to access rate limit: %v\n", err) 466 | mShareFail.Incr(1) 467 | return 468 | } 469 | if tout == 0 { 470 | break 471 | } 472 | select { 473 | case <-time.After(tout): 474 | case <-r.Context().Done(): 475 | http.Error(w, "Timed out", http.StatusRequestTimeout) 476 | mShareFail.Incr(1) 477 | return 478 | } 479 | } 480 | 481 | hashBytes := shareHash(reqBody) 482 | if err = cache.Add(string(hashBytes), reqBody); err != nil && err != types.ErrKeyAlreadyExists { 483 | http.Error(w, "Save failed", http.StatusBadGateway) 484 | log.Errorf("Failed to store request body: %v\n", err) 485 | mShareFail.Incr(1) 486 | return 487 | } 488 | 489 | mShareSucc.Incr(1) 490 | w.Write(hashBytes) 491 | }) 492 | 493 | adminMux := http.NewServeMux() 494 | adminMux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { 495 | w.Write([]byte("pong")) 496 | }) 497 | 498 | if wHandlerFunc, ok := stats.(metrics.WithHandlerFunc); ok { 499 | adminMux.HandleFunc("/metrics", wHandlerFunc.HandlerFunc()) 500 | adminMux.HandleFunc("/stats", wHandlerFunc.HandlerFunc()) 501 | } 502 | 503 | go func() { 504 | log.Infoln("Listening for admin HTTP requests at :8081") 505 | if herr := http.ListenAndServe(":8081", adminMux); herr != nil { 506 | panic(herr) 507 | } 508 | }() 509 | 510 | if len(*tlsHost) == 0 { 511 | log.Infoln("Listening for HTTP requests at :8080") 512 | if herr := http.ListenAndServe(":8080", mux); herr != nil { 513 | panic(herr) 514 | } 515 | return 516 | } 517 | 518 | log.Infoln("Listening for HTTPS requests at :8443") 519 | 520 | var certCache autocert.Cache 521 | if len(*tlsCache) > 0 { 522 | certCache = autocert.DirCache(*tlsCache) 523 | } 524 | 525 | var acmeClient *acme.Client 526 | if *tlsStaging { 527 | acmeClient = &acme.Client{DirectoryURL: "https://acme-staging-v02.api.letsencrypt.org/directory"} 528 | } 529 | 530 | certManager := autocert.Manager{ 531 | Client: acmeClient, 532 | Prompt: autocert.AcceptTOS, 533 | HostPolicy: autocert.HostWhitelist(*tlsHost), 534 | Cache: certCache, 535 | } 536 | 537 | server := &http.Server{ 538 | Addr: ":8443", 539 | Handler: mux, 540 | TLSConfig: &tls.Config{ 541 | GetCertificate: certManager.GetCertificate, 542 | }, 543 | } 544 | 545 | go http.ListenAndServe(":8080", certManager.HTTPHandler(nil)) 546 | 547 | if herr := server.ListenAndServeTLS("", ""); herr != nil { 548 | panic(herr) 549 | } 550 | } 551 | 552 | //------------------------------------------------------------------------------ 553 | --------------------------------------------------------------------------------