├── .circleci ├── bin │ └── entrypoint.sh └── config.yml ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ ├── documentation.yml │ └── feature.yml ├── codecov.yml ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── check.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── Dockerfile.ci ├── Dockerfile.ci.dockerignore ├── Dockerfile.dockerignore ├── LICENSE ├── README.md ├── assets └── kiwi-bird.png ├── doc ├── CONFIGURATION.md ├── PLUGIN.md └── PROTOCOL.md ├── docker-compose.yml ├── examples ├── README.md ├── authenticate-http │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ ├── README.md │ ├── kiwi.yml │ └── src │ │ └── lib.rs └── intercept-simple │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ ├── README.md │ ├── kiwi.yml │ └── src │ └── lib.rs ├── rustfmt.toml └── src ├── kiwi-sdk ├── Cargo.toml ├── macro │ ├── Cargo.toml │ ├── src │ │ └── lib.rs │ └── wit ├── src │ ├── hook │ │ ├── authenticate.rs │ │ ├── intercept.rs │ │ └── mod.rs │ ├── http.rs │ └── lib.rs └── wit ├── kiwi ├── Cargo.toml ├── src │ ├── config.rs │ ├── connection.rs │ ├── hook │ │ ├── authenticate │ │ │ ├── mod.rs │ │ │ ├── types.rs │ │ │ └── wasm │ │ │ │ ├── bindgen.rs │ │ │ │ ├── bridge.rs │ │ │ │ └── mod.rs │ │ ├── intercept │ │ │ ├── mod.rs │ │ │ ├── types.rs │ │ │ └── wasm │ │ │ │ ├── bindgen.rs │ │ │ │ ├── bridge.rs │ │ │ │ └── mod.rs │ │ ├── mod.rs │ │ └── wasm │ │ │ └── mod.rs │ ├── lib.rs │ ├── main.rs │ ├── protocol.rs │ ├── source │ │ ├── counter.rs │ │ ├── kafka.rs │ │ └── mod.rs │ ├── subscription.rs │ ├── tls.rs │ ├── util │ │ ├── macros.rs │ │ ├── mod.rs │ │ ├── serde.rs │ │ └── stream.rs │ └── ws.rs └── tests │ ├── common │ ├── healthcheck.rs │ ├── kafka.rs │ ├── kiwi.rs │ ├── mod.rs │ └── ws.rs │ ├── hook.rs │ ├── kafka.rs │ ├── lifecycle.rs │ └── wasm │ ├── authenticate-api-key.wasm │ └── kafka-even-numbers-intercept.wasm └── wit ├── authenticate-types.wit ├── deps ├── cli │ ├── command.wit │ ├── environment.wit │ ├── exit.wit │ ├── imports.wit │ ├── run.wit │ ├── stdio.wit │ └── terminal.wit ├── clocks │ ├── monotonic-clock.wit │ ├── wall-clock.wit │ └── world.wit ├── filesystem │ ├── preopens.wit │ ├── types.wit │ └── world.wit ├── http │ ├── handler.wit │ ├── proxy.wit │ └── types.wit ├── io │ ├── error.wit │ ├── poll.wit │ ├── streams.wit │ └── world.wit ├── random │ ├── insecure-seed.wit │ ├── insecure.wit │ ├── random.wit │ └── world.wit └── sockets │ ├── instance-network.wit │ ├── ip-name-lookup.wit │ ├── network.wit │ ├── tcp-create-socket.wit │ ├── tcp.wit │ ├── udp-create-socket.wit │ ├── udp.wit │ └── world.wit ├── intercept-types.wit └── world.wit /.circleci/bin/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export PATH=$PATH:/app/target/release 3 | exec "$@" 4 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | rust: circleci/rust@1.0.0 5 | 6 | jobs: 7 | integration-tests: 8 | parameters: 9 | module: 10 | type: string 11 | machine: 12 | image: ubuntu-2004:current 13 | docker_layer_caching: true 14 | working_directory: ~/kiwi 15 | steps: 16 | - checkout 17 | - when: 18 | condition: 19 | equal: [<< parameters.module >>, kafka] 20 | steps: 21 | - run: 22 | name: Install Docker Compose 23 | environment: 24 | COMPOSE_VERSION: 'v2.24.7' 25 | command: | 26 | curl -L "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o ~/docker-compose 27 | chmod +x ~/docker-compose 28 | sudo mv ~/docker-compose /usr/local/bin/docker-compose 29 | - when: 30 | condition: 31 | equal: [<< parameters.module >>, kafka] 32 | steps: 33 | - run: 34 | name: Start all services declared in docker-compose.yml 35 | command: docker-compose up --wait 36 | - run: 37 | name: Build Test Docker Image 38 | command: | 39 | DOCKER_BUILDKIT=1 docker build -t kiwi:latest -f Dockerfile.ci . 40 | - run: 41 | name: Run Test 42 | command: | 43 | NETWORK_NAME=$(docker network ls --filter name='kiwi' -q) 44 | [[ -n $NETWORK_NAME ]] && NETWORK_ARG="--network $NETWORK_NAME" || NETWORK_ARG="" 45 | docker run $NETWORK_ARG \ 46 | --env 'BOOTSTRAP_SERVERS=kafka:19092' \ 47 | --name kiwi-tests \ 48 | kiwi:latest \ 49 | cargo test --test '<< parameters.module >>' -- --nocapture --test-threads=1 50 | 51 | workflows: 52 | integration-tests: 53 | jobs: 54 | - integration-tests: 55 | matrix: 56 | parameters: 57 | module: 58 | - kafka 59 | - lifecycle 60 | - hook 61 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | end_of_line = lf 3 | insert_final_newline = true 4 | indent_style = space 5 | indent_size = 4 6 | 7 | [*.js] 8 | indent_size = 2 9 | 10 | [*.jsx] 11 | indent_size = 2 12 | 13 | [*.td] 14 | indent_size = 2 15 | 16 | [*.ts] 17 | indent_size = 2 18 | 19 | [*.json] 20 | indent_size = 2 21 | 22 | [*.yml] 23 | indent_size = 2 24 | 25 | [*.html] 26 | indent_size = 2 27 | 28 | [Makefile] 29 | indent_style = tab 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: A problem with the existing functionality 3 | title: "[Bug] " 4 | labels: ["bug"] 5 | 6 | body: 7 | - type: markdown 8 | attributes: 9 | value: "Thank you for taking the time to report an issue! 🙏" 10 | - type: input 11 | id: version 12 | attributes: 13 | label: What version of kiwi are you using? 14 | placeholder: "If building & running from source, please provide the commit hash. Otherwise, provide the release version." 15 | validations: 16 | required: true 17 | - type: textarea 18 | id: describe-bug 19 | attributes: 20 | label: Describe the bug 21 | description: A clear and concise description of what the bug is. 22 | validations: 23 | required: true 24 | - type: textarea 25 | id: expected-behavior 26 | attributes: 27 | label: Expected behavior 28 | description: A clear and concise description of what you expected to happen. 29 | placeholder: "I expected that..." 30 | - type: textarea 31 | id: reproduce-steps 32 | attributes: 33 | label: Steps to Reproduce 34 | description: Steps to reproduce the behavior 35 | placeholder: | 36 | 1. Run the command '....' 37 | 2. Subscribe to source '....' 38 | 3. Request '...' messages 39 | validations: 40 | required: true 41 | - type: textarea 42 | id: additional-context 43 | attributes: 44 | label: Additional context 45 | description: Add any other context about the problem here. 46 | placeholder: "Any additional information (e.g. configuration, logs, etc.)" 47 | validations: 48 | required: false 49 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.yml: -------------------------------------------------------------------------------- 1 | name: Documentation Update 2 | description: Suggest improvements or request additions to the documentation 3 | title: "[Docs] " 4 | labels: ["documentation"] 5 | 6 | body: 7 | - type: markdown 8 | attributes: 9 | value: "Help us improve our documentation by submitting a request! 📚" 10 | 11 | - type: input 12 | id: doc-title 13 | attributes: 14 | label: Documentation Title 15 | description: The title or topic of the documentation page needing improvement or addition. 16 | placeholder: "Title or topic of the documentation" 17 | validations: 18 | required: true 19 | 20 | - type: textarea 21 | id: problem-description 22 | attributes: 23 | label: Problem Description 24 | description: Describe the problem with the current documentation or what's missing. 25 | placeholder: "What issue have you encountered with the current documentation or what is missing?" 26 | validations: 27 | required: true 28 | 29 | - type: textarea 30 | id: proposed-change 31 | attributes: 32 | label: Proposed Change 33 | description: Detail the improvements or additions you're proposing. 34 | placeholder: "Please describe the changes you propose..." 35 | validations: 36 | required: true 37 | 38 | - type: textarea 39 | id: additional-context 40 | attributes: 41 | label: Additional Context 42 | description: Provide any additional context that could be helpful. 43 | placeholder: "Any other context or notes?" 44 | validations: 45 | required: false 46 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature.yml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: Suggest an idea for kiwi 3 | title: "[Feature] " 4 | labels: ["enhancement"] 5 | 6 | body: 7 | - type: markdown 8 | attributes: 9 | value: "Thanks for taking the time to suggest a feature! 🌟" 10 | - type: textarea 11 | id: feature-request 12 | attributes: 13 | label: Feature Request 14 | description: Detailed description of the feature you're proposing. 15 | placeholder: "Please describe the feature you would like to see added..." 16 | validations: 17 | required: true 18 | - type: textarea 19 | id: motivation 20 | attributes: 21 | label: Motivation and Goals 22 | description: Explain why this feature is important for the project and the goals it will achieve. 23 | placeholder: "Explain why this feature is important..." 24 | validations: 25 | required: true 26 | -------------------------------------------------------------------------------- /.github/codecov.yml: -------------------------------------------------------------------------------- 1 | # ref: https://docs.codecov.com/docs/codecovyml-reference 2 | coverage: 3 | # Hold ourselves to a high bar 4 | range: 85..100 5 | round: down 6 | precision: 1 7 | status: 8 | # ref: https://docs.codecov.com/docs/commit-status 9 | project: 10 | default: 11 | # Avoid false negatives 12 | threshold: 1% 13 | 14 | # Test files aren't important for coverage 15 | ignore: 16 | - "tests" 17 | 18 | # Make comments less noisy 19 | comment: 20 | layout: "files" 21 | require_changes: true 22 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | - package-ecosystem: cargo 8 | directory: / 9 | schedule: 10 | interval: weekly 11 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Please include a clear and concise description of what the pull request does. Explain the problem it solves or the feature it adds to the project. If this PR fixes an open issue, please link to the issue here, using the format "Fixes #issue_number". 4 | 5 | ## Type of change 6 | 7 | Please delete options that are not relevant. 8 | 9 | - [ ] Bug fix (non-breaking change which fixes an issue) 10 | - [ ] New feature (non-breaking change which adds functionality) 11 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 12 | - [ ] Documentation update 13 | 14 | ## How Has This Been Tested? 15 | 16 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration. 17 | 18 | - [ ] Test A 19 | - [ ] Test B 20 | 21 | ## Checklist: 22 | 23 | Before submitting your PR, please review and confirm the following: 24 | 25 | - [ ] I have performed a self-review of my own code and ensured it adheres to the project guidelines and coding standards. 26 | - [ ] I have commented my code, particularly in hard-to-understand areas. 27 | - [ ] I have made corresponding changes to the documentation, if applicable. 28 | - [ ] My changes generate no new warnings or errors. 29 | - [ ] I have added tests that prove my fix is effective or that my feature works. 30 | - [ ] Integration tests have been added for relevant changes. 31 | - [ ] Any dependent changes have been merged and published in downstream modules. 32 | 33 | ## Additional context 34 | 35 | Add any other context about the pull request here. This can include challenges faced, decisions made, or any other relevant information that would help the reviewers understand your changes better. 36 | 37 | -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | permissions: 2 | contents: read 3 | 4 | on: 5 | push: 6 | branches: [main] 7 | pull_request: 8 | 9 | concurrency: 10 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 11 | cancel-in-progress: true 12 | 13 | name: check 14 | 15 | jobs: 16 | fmt: 17 | runs-on: ubuntu-latest 18 | name: fmt 19 | steps: 20 | - uses: actions/checkout@v4 21 | with: 22 | submodules: true 23 | - name: Install stable 24 | uses: dtolnay/rust-toolchain@stable 25 | with: 26 | components: rustfmt 27 | - name: cargo fmt --check 28 | run: cargo fmt --check 29 | clippy: 30 | runs-on: ubuntu-latest 31 | name: clippy 32 | permissions: 33 | contents: read 34 | checks: write 35 | steps: 36 | - uses: actions/checkout@v4 37 | with: 38 | submodules: true 39 | - name: Install stable 40 | uses: dtolnay/rust-toolchain@stable 41 | with: 42 | components: clippy 43 | - name: cargo clippy 44 | run: cargo clippy -- -Dwarnings 45 | doc: 46 | runs-on: ubuntu-latest 47 | name: docs 48 | steps: 49 | - uses: actions/checkout@v4 50 | with: 51 | submodules: true 52 | - name: Install stable 53 | uses: dtolnay/rust-toolchain@stable 54 | - name: cargo doc 55 | run: cargo doc --no-deps --all-features 56 | env: 57 | RUSTDOCFLAGS: --cfg docsrs 58 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | tags: 8 | - 'kiwi-v*' 9 | 10 | env: 11 | REGISTRY: ghcr.io 12 | IMAGE_NAME: ${{ github.repository }} 13 | 14 | jobs: 15 | docker: 16 | runs-on: ubuntu-latest 17 | permissions: 18 | contents: read 19 | packages: write 20 | steps: 21 | - name: Checkout 22 | uses: actions/checkout@v4 23 | 24 | - name: Docker Meta 25 | id: meta 26 | uses: docker/metadata-action@v5 27 | with: 28 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 29 | tags: | 30 | type=ref,event=branch 31 | type=match,pattern=kiwi-v(.*),group=1 32 | 33 | - name: Login to Container Registry 34 | uses: docker/login-action@v3 35 | with: 36 | registry: ${{ env.REGISTRY }} 37 | username: ${{ github.actor }} 38 | password: ${{ secrets.GITHUB_TOKEN }} 39 | 40 | - name: Build and Push 41 | uses: docker/build-push-action@v5 42 | with: 43 | context: . 44 | push: true 45 | tags: ${{ steps.meta.outputs.tags }} 46 | labels: ${{ steps.meta.outputs.labels }} 47 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | permissions: 2 | contents: read 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | 8 | concurrency: 9 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 10 | cancel-in-progress: true 11 | 12 | name: test 13 | 14 | jobs: 15 | required: 16 | runs-on: ubuntu-latest 17 | name: ubuntu / stable 18 | steps: 19 | - uses: actions/checkout@v4 20 | with: 21 | submodules: true 22 | - name: Install stable 23 | uses: dtolnay/rust-toolchain@stable 24 | - name: cargo test --locked 25 | run: cargo test --locked --all-features --workspace --lib 26 | # https://github.com/rust-lang/cargo/issues/6669 27 | - name: cargo test --doc 28 | run: cargo test --locked --all-features --doc 29 | coverage: 30 | # use llvm-cov to build and collect coverage and outputs in a format that is compatible with 31 | # codecov.io 32 | runs-on: ubuntu-latest 33 | name: ubuntu / stable / coverage 34 | steps: 35 | - uses: actions/checkout@v4 36 | with: 37 | submodules: true 38 | - name: Install stable 39 | uses: dtolnay/rust-toolchain@stable 40 | with: 41 | components: llvm-tools-preview 42 | - name: cargo install cargo-llvm-cov 43 | uses: taiki-e/install-action@cargo-llvm-cov 44 | - name: cargo llvm-cov 45 | run: cargo llvm-cov --locked --workspace --lib --all-features --lcov --output-path lcov.info 46 | - name: Upload to codecov.io 47 | uses: codecov/codecov-action@v3 48 | env: 49 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 50 | with: 51 | files: lcov.info 52 | fail_ci_if_error: true 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /*.wasm 3 | /*kiwi.yml 4 | /*.crt 5 | /*.key 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["src/kiwi", "src/kiwi-sdk"] 3 | resolver = "2" 4 | 5 | [workspace.package] 6 | edition = "2021" 7 | rust-version = "1.80.0" 8 | authors = ["Rohan Krishnaswamy "] 9 | description = "Kiwi is a fast and extensible WebSocket gateway for real-time event sources" 10 | repository = "https://github.com/rkrishn7/kiwi" 11 | license = "MIT" 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM lukemathwalker/cargo-chef:latest-rust-slim-bookworm AS chef 2 | WORKDIR /app 3 | 4 | FROM chef AS planner 5 | 6 | COPY . . 7 | 8 | RUN cargo chef prepare --recipe-path recipe.json 9 | 10 | FROM chef AS builder 11 | 12 | COPY --from=planner /app/recipe.json recipe.json 13 | 14 | # Install cmake (For building librdkafka) 15 | RUN apt-get update && \ 16 | apt-get install -y cmake curl g++ && \ 17 | apt-get clean 18 | 19 | # Build dependencies - this is the caching Docker layer! 20 | RUN cargo chef cook --release --recipe-path recipe.json 21 | 22 | # Build application 23 | COPY . . 24 | RUN cargo build --release --locked --bin kiwi 25 | 26 | FROM debian:bookworm-slim 27 | WORKDIR /app 28 | 29 | # Install libssl (Rust links against this library) 30 | RUN apt-get update && \ 31 | apt-get install -y libssl-dev ca-certificates && \ 32 | apt-get clean 33 | 34 | COPY --from=builder /app/target/release/kiwi /usr/local/bin 35 | 36 | ENTRYPOINT ["/usr/local/bin/kiwi"] 37 | -------------------------------------------------------------------------------- /Dockerfile.ci: -------------------------------------------------------------------------------- 1 | FROM rust:1.80-bookworm as builder 2 | WORKDIR /app 3 | 4 | # Install cmake (Required to build librdkafka) 5 | RUN apt-get update && \ 6 | apt-get install -y cmake && \ 7 | apt-get clean 8 | 9 | COPY . . 10 | 11 | RUN cargo build --release --locked --bin kiwi 12 | 13 | ENTRYPOINT ["./.circleci/bin/entrypoint.sh"] 14 | -------------------------------------------------------------------------------- /Dockerfile.ci.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | 3 | !/src 4 | !Cargo.toml 5 | !Cargo.lock 6 | !/.circleci/bin 7 | -------------------------------------------------------------------------------- /Dockerfile.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | 3 | !/src 4 | !Cargo.toml 5 | !Cargo.lock 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Rohan Krishnaswamy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | drawing 3 |

Kiwi - Extensible Real-Time Data Streaming

4 | 5 | [![test](https://github.com/rkrishn7/kiwi/actions/workflows/test.yml/badge.svg)](https://github.com/rkrishn7/kiwi/actions/workflows/test.yml) [![check](https://github.com/rkrishn7/kiwi/actions/workflows/check.yml/badge.svg)](https://github.com/rkrishn7/kiwi/actions/workflows/check.yml) [![CircleCI](https://dl.circleci.com/status-badge/img/gh/rkrishn7/kiwi/tree/main.svg?style=shield)](https://dl.circleci.com/status-badge/redirect/gh/rkrishn7/kiwi/tree/main) ![contributions](https://img.shields.io/badge/contributions-welcome-green) 6 |
7 | 8 | Kiwi is a WebSocket adapter for real-time data sources. It implements a simple protocol for clients to subscribe to configured sources, while allowing operators to maintain control over the flow of data via [WebAssembly](https://webassembly.org/) (WASM) plugins. Kiwi is designed to be a lightweight, extensible, and secure solution for delivering real-time data to clients, ensuring that they stay reactive and up-to-date with the latest data. 9 | 10 | - [Features](#features) 11 | - [Motivation](#motivation) 12 | - [Getting Started](#getting-started) 13 | - [Plugins](#plugins) 14 | - [Sources](#sources) 15 | - [Kafka](#kafka) 16 | - [Counter](#counter) 17 | - [Protocol](#protocol) 18 | - [Configuration](#configuration) 19 | - [Considerations](#considerations) 20 | 21 | ## Features 22 | 23 | - **Subscribe with Ease**: Set up subscriptions to various sources with a simple command. Kiwi efficiently routes event data to connected WebSocket clients based on these subscriptions. 24 | - **Extensible**: Kiwi supports WebAssembly (WASM) plugins to enrich and control the flow of data. Plugins are called with context about the current connection and event, and can be used to control how/when events are forwarded to downstream clients. 25 | - **Backpressure Management**: Kiwi draws from flow-control concepts used by Reactive Streams. Specifically, clients can emit a `request(n)` signal to control the rate at which they receive events. 26 | - **Secure**: Kiwi supports TLS encryption and custom client authentication via WASM plugins. 27 | - **Configuration Reloads**: Kiwi can reload a subset of its configuration at runtime, allowing for dynamic updates to sources and plugin code without restarting the server. 28 | 29 | ## Motivation 30 | 31 | The digital era has increasingly moved towards real-time data and event-driven architectures. Tools like Apache Kafka have set the standard for robust, high-throughput messaging and streaming, enabling applications to process and react to data as it arrives. Kafka, and the ecosystem built around it, excel at ingesting streams of events and providing the backbone for enterprise-level data processing and analytics. However, there's often a disconnect when trying to extend this real-time data paradigm directly to end-users in a web or mobile environment. 32 | 33 | Enter **Kiwi**. 34 | 35 | While Kafka and technologies that build upon it serve as powerful platforms for data aggregation and processing, Kiwi aims to complement these tools by acting as the last mile for delivering real-time data to users. Serving as a "general-purpose" gateway, a major component of Kiwi is its plugin interface, empowering developers to define the behavior of their plugins according to the unique requirements of their applications. This approach allows Kiwi to focus on its primary objective of efficiently routing data to clients and managing subscriptions. 36 | 37 | ## Getting Started 38 | 39 | The easiest way to get started with Kiwi is with Docker! First, create a simple configuration file for Kiwi named `kiwi.yml`: 40 | 41 | ```yaml 42 | sources: 43 | # Counter sources are primarily used for testing and demonstration purposes 44 | - id: "counter" 45 | type: "counter" 46 | interval_ms: 1000 47 | lazy: true 48 | min: 0 49 | 50 | server: 51 | address: '0.0.0.0:8000' 52 | ``` 53 | 54 | Next, in the same directory as the `kiwi.yml` file, run the following command to start Kiwi: 55 | 56 | ```sh 57 | docker run -p 8000:8000 -v $(pwd)/kiwi.yml:/etc/kiwi/config/kiwi.yml ghcr.io/rkrishn7/kiwi:main 58 | ``` 59 | 60 | Success! Kiwi is now running and ready to accept WebSocket connections on port 8000. You can start interacting with the server by using a WebSocket client utility of your choice (e.g. [wscat](https://www.npmjs.com/package/wscat)). Refer to the [protocol documentation](./doc/PROTOCOL.md) for details on how to interact with the Kiwi server. 61 | 62 | For more examples, please see the [examples](./examples) directory. 63 | 64 | ## Plugins 65 | 66 | Kiwi supports WebAssembly (WASM) plugins which allows developers to define the behavior of event delivery and authorization according to the unique requirements of their applications. A [Rust SDK](https://docs.rs/kiwi-sdk/latest/kiwi_sdk/) is provided to simplify the process of writing plugins in Rust. 67 | 68 | There are two types of plugins that Kiwi supports: 69 | 70 | - **Intercept**: Intercept plugins are invoked before an event is sent to a client. They are called with context about the current connection and event, and can be used to control how/when events are forwarded to downstream clients. 71 | - For example, imagine you are writing a chat application and only want users to receive messages they are authorized to see. While the chat message source may emit messages for all conversations, an intercept plugin can be used to filter out messages that the user is not authorized to see. 72 | 73 | - **Authentication**: Authentication plugins are invoked when a client connects to the server. They are called with context about the current connection and can be used to authenticate the client, potentially rejecting the connection if the client is not authorized to connect. 74 | - Authentication plugins allow users of Kiwi to enforce custom authentication logic, such as verifying JWT tokens or checking for specific user roles. Additionally, the plugin may return custom context for the connection which is passed downstream to each invocation of the intercept plugin. 75 | 76 | For more information on writing and using plugins, please see the [plugin documentation](./doc/PLUGIN.md). 77 | 78 | ## Sources 79 | 80 | ### Kafka 81 | 82 | Currently, Kiwi primarily supports Kafka as a data source, with plans to support additional sources in the future. Kafka sources are backed by a high-performance Rust Kafka client, [rust-rdkafka](https://github.com/fede1024/rust-rdkafka), and support automatic partition discovery. 83 | 84 | Notably, Kiwi does not leverage balanced consumer groups for Kafka sources. Instead, it subscribes to the entire set of partitions for a given topic and invokes the configured intercept plugin, if any, for each event. This has a few implications: 85 | 86 | - Kiwi may not be suitable for very high-throughput Kafka topics, as the freshness of events may be impacted by the combination of the high volume of events across partitions and per-event processing time. There are plans to support a deterministic partitioning plugin for clients in the future to address this limitation for supported use cases. 87 | - Event processing cannot be parallelized across multiple instances of Kiwi. If vertical scaling is not sufficient to handle the combined throughput of all configured sources, Kiwi may not be the best fit. 88 | 89 | ### Counter 90 | 91 | Kiwi also includes a simple counter source for testing and demonstration purposes. The counter source emits a monotonically increasing integer at a configurable interval, and is primarily used to demonstrate the behavior of Kiwi with a simple source. 92 | 93 | ## Protocol 94 | 95 | Details on the Kiwi protocol can be found in the [protocol documentation](./doc/PROTOCOL.md). 96 | 97 | ## Configuration 98 | 99 | Details on configuring Kiwi can be found in the [configuration documentation](./doc/CONFIGURATION.md). 100 | 101 | ## Considerations 102 | 103 | Kiwi is designed as a real-time event notification service, leveraging WebAssembly (WASM) plugins to enrich and control the flow of data. While Kiwi supports certain operations commonly associated with stream processing, such as map and filter, it is not intended to replace full-fledged stream processing frameworks. 104 | 105 | Kiwi excels at handling event-driven communication with efficient backpressure management, making it suitable for real-time messaging and lightweight data transformation tasks. However, users requiring advanced stream processing capabilities—such as complex event processing (CEP), stateful computations, windowing, and aggregation over unbounded datasets—are encouraged to use specialized stream processing systems. 106 | 107 | Kiwi is designed to be a part of a broader architecture where it can work in conjunction with such systems, rather than serve as a standalone solution for high-throughput data processing needs. 108 | -------------------------------------------------------------------------------- /assets/kiwi-bird.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rkrishn7/kiwi/682f1bf2410ccb35143cf41d5f3479c7919fe27a/assets/kiwi-bird.png -------------------------------------------------------------------------------- /doc/CONFIGURATION.md: -------------------------------------------------------------------------------- 1 | # Kiwi Configuration 2 | 3 | Kiwi can be configured using a YAML file. By default, Kiwi looks for configuration in `/etc/kiwi/config/kiwi.yml`. However, you may specify a different configuration file using the `--config` option CLI option. 4 | 5 | The configuration format is described in detail below. Note that values shown are examples, and not defaults. 6 | 7 | ```yaml 8 | # Hooks (plugins) are custom WASM modules that Kiwi loads and executes at various points 9 | # in the event processing pipeline. Hooks are optional and may be used to extend Kiwi's 10 | # functionality. 11 | # 12 | ## Optional 13 | hooks: 14 | # The authenticate hook is executed during the WebSocket handshake. It can be used to authenticate 15 | # clients using custom mechanisms and add context to the connection that will be passed downstream 16 | # to the intercept hook. 17 | # 18 | ## Optional (default: null) 19 | authenticate: 'my-authenticate-hook/target/wasm32-wasip1/debug/authenticate_http.wasm' 20 | 21 | # This plugin executes once Kiwi ingest's the source event, but before it decides whether to 22 | # forward the event to any of the source's subscribers. 23 | # 24 | ## Optional (default: null) 25 | intercept: 'my-intercept-hook/target/wasm32-wasip1/debug/intercept.wasm' 26 | 27 | # Source Configuration 28 | # 29 | # Currently, Kiwi supports two types of sources: Kafka and Counter sources. Each source type 30 | # is denoted by the `type` field and has its own set of required and optional fields. 31 | # 32 | ## Required 33 | sources: 34 | - type: kafka 35 | 36 | # The source ID for this counter source. The source ID is used as a unique identifier, thus must be 37 | # distinct from other source IDs, regardless of type. 38 | # 39 | ## Optional (defaults to `topic`) 40 | id: my-kafka-source 41 | 42 | # The topic name for this Kafka source. The source ID defaults to the topic name. 43 | # 44 | ## Required 45 | topic: 'my-topic' 46 | 47 | - type: counter 48 | 49 | # The source ID for this counter source. The source ID is used as a unique identifier, thus must be 50 | # distinct from other source IDs, regardless of type. 51 | # 52 | ## Required 53 | id: counter1 54 | 55 | # The interval at which the counter source emits events 56 | # 57 | ## Required 58 | interval_ms: 1000 59 | 60 | # Setting `lazy` to true will cause the counter source to start emitting events only upon its first 61 | # subscription. 62 | # 63 | ## Optional (default: false) 64 | lazy: true 65 | 66 | # The maximum value of the counter. Setting this value marks the source as a finite source. Once 67 | # the counter reaches this value, it will stop emitting events and disallow new subscriptions. 68 | # 69 | ## Optional (default: `u64::MAX`) 70 | max: 1000 71 | 72 | # The initial value of the counter 73 | # 74 | ## Required 75 | min: 0 76 | 77 | # Kafka Consumer Configuration 78 | # 79 | ## Required if any Kafka sources are defined. Optional otherwise 80 | kafka: 81 | # The list of bootstrap servers to connect to 82 | # 83 | ## Required 84 | bootstrap_servers: 85 | - 'localhost:9092' 86 | 87 | # Whether to enable partition discovery. If enabled, Kiwi will periodically 88 | # query the Kafka cluster to discover new partitions. 89 | # 90 | ## Optional (default: true) 91 | partition_discovery_enabled: true 92 | 93 | # The interval at which to query the Kafka cluster to discover new partitions 94 | # 95 | ## Optional (default: 300000) 96 | partition_discovery_interval_ms: 300000 97 | 98 | # WebSocket Server Configuration 99 | # 100 | ## Required 101 | server: 102 | # The address to bind the WebSocket server to 103 | # 104 | ## Required 105 | address: '127.0.0.1:8000' 106 | 107 | # Whether to enable the health check endpoint at `/health` 108 | # 109 | ## Optional (default: true) 110 | healthcheck: true 111 | 112 | # TLS Configuration 113 | # 114 | ## Optional 115 | tls: 116 | # The path to the certificate file 117 | # 118 | ## Required 119 | cert: '/path/to/cert.pem' 120 | # The path to the private key file 121 | # 122 | ## Required 123 | key: '/path/to/key.pem' 124 | 125 | 126 | # Subscriber Configuration 127 | # 128 | # Subscribers are clients that connect to Kiwi and receive events from the sources 129 | # they are subscribed to. 130 | # 131 | ## Optional 132 | subscriber: 133 | # For pull-based subscribers, Kiwi is capable of maintaining internal buffers to 134 | # handle backpressure. This setting controls the maximum number of events that may 135 | # be buffered for any given subscription. Once the capacity is reached, buffered events 136 | # will be dropped in a LRU fashion. 137 | # 138 | # If this option is not set, Kiwi will drop events when a pull-based subscriber is 139 | # unable to keep up with its respective source result frequency. 140 | # 141 | ## Optional (default: null) 142 | buffer_capacity: 100 143 | 144 | # The maximum number of events that a pull-based subscriber is permitted to 145 | # lag behind the source before Kiwi begins emitting lag notices. If not set, 146 | # Kiwi will not emit any lag notices. 147 | # 148 | ## Optional (default: null) 149 | lag_notice_threshold: 50 150 | ``` 151 | -------------------------------------------------------------------------------- /doc/PLUGIN.md: -------------------------------------------------------------------------------- 1 | # Kiwi Plugins 2 | 3 | Kiwi allows operators to create WASM plugins that can be loaded at start-up to extend the functionality of the server. 4 | 5 | 6 | ## Creating a Plugin 7 | 8 | A plugin is a WebAssembly module that implements a specific interface. Currently, Kiwi offers a Rust SDK to streamline the process of creating plugins 9 | 10 | First, bootstrap a new plugin project using the `cargo` command-line tool: 11 | 12 | ```sh 13 | cargo new --lib my-kiwi-plugin 14 | ``` 15 | 16 | Next, in the project folder, run the following command to add the `kiwi-sdk` crate: 17 | 18 | ```sh 19 | cargo add kiwi-sdk 20 | ``` 21 | 22 | Before writing any code, ensure that crate type is specified as `cdylib` in the `Cargo.toml` file. This is necessary to compile a dynamic library that can be loaded by Kiwi at runtime. 23 | 24 | ```toml 25 | [lib] 26 | crate-type = ["cdylib"] 27 | ``` 28 | 29 | **Note**: The plugin must be built with the `--target` flag set to `wasm32-wasip1` 30 | 31 | ```sh 32 | cargo build --target wasm32-wasip1 33 | ``` 34 | 35 | Nice! You're ready to start writing your plugin. Take a look in the [examples](../examples) directory for Kiwi plugin samples. 36 | -------------------------------------------------------------------------------- /doc/PROTOCOL.md: -------------------------------------------------------------------------------- 1 | # Kiwi Protocol 2 | 3 | Kiwi uses a simple, text-based protocol to allow clients to interact with the server. Acting as a WebSocket gateway for real-time event sources, Kiwi is designed to be language-agnostic and can be used with any WebSocket client library. The protocol is implemented in JSON and intended to be simple and easy to understand. 4 | 5 | In short, the protocol consists of a set of commands that clients can issue, which in turn trigger responses from the server. Additionally, the server may, at any time, send asynchronous messages to the client. Typically, these messages are events from subscribed sources, but they can also be error messages or other notifications. 6 | 7 | - [Kiwi Protocol](#kiwi-protocol) 8 | - [Subscriptions](#subscriptions) 9 | - [Subscribing to Sources](#subscribing-to-sources) 10 | - [Unsubscribing from Sources](#unsubscribing-from-sources) 11 | - [Requesting Events (Pull-Based Subscriptions)](#requesting-events-pull-based-subscriptions) 12 | - [Subscription Results](#subscription-results) 13 | - [Notices](#notices) 14 | - [Lag Notices](#lag-notices) 15 | - [Subscription Closed Notices](#subscription-closed-notices) 16 | 17 | ## Subscriptions 18 | 19 | ### Subscribing to Sources 20 | 21 | Clients can subscribe to sources by sending a `SUBSCRIBE` command to the server: 22 | 23 | ```json 24 | { 25 | "type": "SUBSCRIBE", 26 | "sourceId": string, 27 | // Optional (defaults to "push") 28 | "mode": "pull" | "push" 29 | } 30 | ``` 31 | 32 | In the payload schema above, `sourceId` is the unique identifier of the defined source in the server's configuration. The `mode` field is optional and defaults to `push`. When set to `pull`, the server will not send events to the client until the client explicitly requests them using the [`REQUEST` command](#requesting-events-pull-based-subscriptions). This mode is useful for clients that want to have control over the rate at which they receive events. 33 | 34 | Upon successful subscription, the server will respond with a `SUBSCRIBE_OK` command response: 35 | 36 | ```json 37 | { 38 | "type": "SUBSCRIBE_OK", 39 | "data": { 40 | "sourceId": string 41 | } 42 | } 43 | ``` 44 | 45 | Here, the `sourceId` field will match the `sourceId` from the original `SUBSCRIBE` command. 46 | 47 | If the subscription fails, the server will respond with a `SUBSCRIBE_ERROR` command: 48 | 49 | ```json 50 | { 51 | "type": "SUBSCRIBE_ERROR", 52 | "data": { 53 | "sourceId": string, 54 | "error": string 55 | } 56 | } 57 | ``` 58 | 59 | The `error` field will contain a human-readable error message explaining why the subscription failed. 60 | 61 | ### Unsubscribing from Sources 62 | 63 | Clients can unsubscribe from subscribed sources by sending an `UNSUBSCRIBE` command to the server: 64 | 65 | ```json 66 | { 67 | "type": "UNSUBSCRIBE", 68 | "sourceId": string 69 | } 70 | ``` 71 | 72 | The `sourceId` specified above must be associated with an active subscription maintained by the server for the client, otherwise the server will respond with an `UNSUBSCRIBE_ERROR` command: 73 | 74 | ```json 75 | { 76 | "type": "UNSUBSCRIBE_ERROR", 77 | "data": { 78 | "sourceId": string, 79 | "error": string 80 | } 81 | } 82 | ``` 83 | 84 | Upon successful unsubscription, the server will respond with an `UNSUBSCRIBE_OK` command: 85 | 86 | ```json 87 | { 88 | "type": "UNSUBSCRIBE_OK", 89 | "data": { 90 | "sourceId": string 91 | } 92 | } 93 | ``` 94 | 95 | ### Requesting Events (Pull-Based Subscriptions) 96 | 97 | Clients can request events from the server for pull-based subscriptions by sending a `REQUEST` command: 98 | 99 | ```json 100 | { 101 | "type": "REQUEST", 102 | "sourceId": string, 103 | "n": number 104 | } 105 | ``` 106 | 107 | The `sourceId` field must match the `sourceId` of an active pull-based subscription. The `n` field specifies the number of events the client is requesting from the server. 108 | 109 | > NOTE: `REQUEST` commands are additive in that they do not replace the previous request. Instead, the server will accumulate the number of requested events across multiple `REQUEST` commands and send events to the client accordingly. 110 | 111 | Upon successful request, the server will respond with a `REQUEST_OK` command: 112 | 113 | ```json 114 | { 115 | "type": "REQUEST_OK", 116 | "data": { 117 | "sourceId": string, 118 | "n": number 119 | } 120 | } 121 | ``` 122 | 123 | If the request fails, the server will respond with a `REQUEST_ERROR` command: 124 | 125 | ```json 126 | { 127 | "type": "REQUEST_ERROR", 128 | "data" { 129 | "sourceId": string, 130 | "error": string 131 | } 132 | } 133 | ``` 134 | 135 | ### Subscription Results 136 | 137 | The server will issue `RESULT` messages to the client when events are available for any subscribed sources. The payload of a `RESULT` message will contain the event data: 138 | 139 | ```json 140 | { 141 | "type": "RESULT", 142 | "data": SourceData 143 | } 144 | ``` 145 | 146 | Where `SourceData` is a source-specific data structure that contains the event payload, source ID, and any other relevant metadata. The type of `SourceData` is represented as the following: 147 | 148 | ```ts 149 | type SourceData = KafkaSourceData | CounterSourceData; 150 | 151 | type KafkaSourceData = { 152 | sourceId: string, 153 | sourceType: "kafka", 154 | // The payload is base64 encoded 155 | payload: string, 156 | partition: number, 157 | offset: number, 158 | key: string, 159 | value: string, 160 | timestamp?: number 161 | }; 162 | 163 | type CounterSourceData = { 164 | sourceId: string, 165 | sourceType: "counter", 166 | count: number 167 | }; 168 | ``` 169 | 170 | ## Notices 171 | 172 | The server may send notices to the client at any time. These notices can be informational, error messages. 173 | 174 | ### Lag Notices 175 | 176 | When a client is subscribed to a Kafka source, the server may send lag notices to the client. Lag most commonly may occur during pull-based subscriptions, where the client is not consuming events as fast as they are being produced and the buffer capacity has been reached. A lag notice looks like this: 177 | 178 | ```json 179 | { 180 | "type": "NOTICE", 181 | "data": { 182 | "type": "LAG", 183 | "sourceId": string, 184 | "count": number, 185 | } 186 | } 187 | ``` 188 | 189 | Here, `count` is the number of events that the client is lagging behind. The `sourceId` field will match the `sourceId` of the source for which the lag notice is being sent. 190 | 191 | ### Subscription Closed Notices 192 | 193 | If a subscription is closed by the server not due to an explicit unsubscription request from the client, the server will send a subscription closed notice to the client: 194 | 195 | ```json 196 | { 197 | "type": "NOTICE", 198 | "data": { 199 | "type": "SUBSCRIPTION_CLOSED", 200 | "sourceId": string, 201 | "message": string 202 | } 203 | } 204 | ``` 205 | 206 | The `sourceId` field will match the `sourceId` of the source for which the subscription was closed. The `message` field will contain a human-readable message explaining why the subscription was closed. 207 | 208 | Subscriptions may close due to various reasons, such as the source ending (for finite sources), the source metadata changing, the source being deleted in the configuration, or some server error. 209 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | zookeeper: 5 | image: confluentinc/cp-zookeeper:latest 6 | hostname: zookeeper 7 | container_name: zookeeper 8 | ports: 9 | - "2181:2181" 10 | environment: 11 | ZOOKEEPER_CLIENT_PORT: 2181 12 | ZOOKEEPER_SERVER_ID: 1 13 | ZOOKEEPER_SERVERS: zookeeper:2888:3888 14 | 15 | kafka: 16 | image: confluentinc/cp-kafka:latest 17 | hostname: kafka 18 | container_name: kafka 19 | restart: always 20 | healthcheck: 21 | test: kafka-topics --list --bootstrap-server localhost:9092 22 | interval: 10s 23 | timeout: 20s 24 | retries: 5 25 | start_period: 10s 26 | ports: 27 | - "9092:9092" 28 | - "29092:29092" 29 | - "9999:9999" 30 | environment: 31 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 32 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT 33 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL 34 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 35 | KAFKA_BROKER_ID: 1 36 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 37 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 38 | KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" 39 | KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer 40 | KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" 41 | depends_on: 42 | - zookeeper 43 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples of running Kiwi 🥝 2 | 3 | This directory contains examples of running Kiwi in various configurations. Each example is self-contained and includes a `README.md` file with instructions on how to run it. 4 | 5 | ### Prerequisites 6 | 7 | - [Rust](https://www.rust-lang.org/tools/install) - The Rust toolchain is required to build WASM hooks. 8 | - [Docker](https://docs.docker.com/get-docker/) - Docker is utilized in each example as a simple way to run Kiwi. 9 | - [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) - The Node.js package manager is required to install [wscat](https://www.npmjs.com/package/wscat), a WebSocket client used to interact with the Kiwi server. 10 | 11 | ## Examples 12 | 13 | - [Intercept (Simple)](./intercept-simple): A simple example that demonstrates how to write WASM hooks using the Kiwi SDK and load them into the Kiwi runtime. 14 | - [Authenticate w/ Outbound HTTP](./authenticate-http): An example that demonstrates how to make outbound HTTP requests in the authenticate hook using the Kiwi SDK. 15 | -------------------------------------------------------------------------------- /examples/authenticate-http/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /examples/authenticate-http/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "anyhow" 7 | version = "1.0.79" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" 10 | 11 | [[package]] 12 | name = "authenticate-http" 13 | version = "0.0.0" 14 | dependencies = [ 15 | "anyhow", 16 | "http", 17 | "kiwi-sdk", 18 | ] 19 | 20 | [[package]] 21 | name = "bitflags" 22 | version = "2.4.2" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" 25 | 26 | [[package]] 27 | name = "bytes" 28 | version = "1.5.0" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" 31 | 32 | [[package]] 33 | name = "equivalent" 34 | version = "1.0.1" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" 37 | 38 | [[package]] 39 | name = "fnv" 40 | version = "1.0.7" 41 | source = "registry+https://github.com/rust-lang/crates.io-index" 42 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 43 | 44 | [[package]] 45 | name = "hashbrown" 46 | version = "0.14.3" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" 49 | 50 | [[package]] 51 | name = "heck" 52 | version = "0.4.1" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" 55 | dependencies = [ 56 | "unicode-segmentation", 57 | ] 58 | 59 | [[package]] 60 | name = "http" 61 | version = "1.0.0" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" 64 | dependencies = [ 65 | "bytes", 66 | "fnv", 67 | "itoa", 68 | ] 69 | 70 | [[package]] 71 | name = "id-arena" 72 | version = "2.2.1" 73 | source = "registry+https://github.com/rust-lang/crates.io-index" 74 | checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" 75 | 76 | [[package]] 77 | name = "indexmap" 78 | version = "2.2.2" 79 | source = "registry+https://github.com/rust-lang/crates.io-index" 80 | checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" 81 | dependencies = [ 82 | "equivalent", 83 | "hashbrown", 84 | "serde", 85 | ] 86 | 87 | [[package]] 88 | name = "itoa" 89 | version = "1.0.10" 90 | source = "registry+https://github.com/rust-lang/crates.io-index" 91 | checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" 92 | 93 | [[package]] 94 | name = "kiwi-macro" 95 | version = "0.1.1" 96 | dependencies = [ 97 | "proc-macro2", 98 | "quote", 99 | "syn 1.0.109", 100 | ] 101 | 102 | [[package]] 103 | name = "kiwi-sdk" 104 | version = "0.1.1" 105 | dependencies = [ 106 | "anyhow", 107 | "http", 108 | "kiwi-macro", 109 | "wit-bindgen", 110 | ] 111 | 112 | [[package]] 113 | name = "leb128" 114 | version = "0.2.5" 115 | source = "registry+https://github.com/rust-lang/crates.io-index" 116 | checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" 117 | 118 | [[package]] 119 | name = "log" 120 | version = "0.4.20" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" 123 | 124 | [[package]] 125 | name = "proc-macro2" 126 | version = "1.0.78" 127 | source = "registry+https://github.com/rust-lang/crates.io-index" 128 | checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" 129 | dependencies = [ 130 | "unicode-ident", 131 | ] 132 | 133 | [[package]] 134 | name = "quote" 135 | version = "1.0.35" 136 | source = "registry+https://github.com/rust-lang/crates.io-index" 137 | checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" 138 | dependencies = [ 139 | "proc-macro2", 140 | ] 141 | 142 | [[package]] 143 | name = "ryu" 144 | version = "1.0.16" 145 | source = "registry+https://github.com/rust-lang/crates.io-index" 146 | checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" 147 | 148 | [[package]] 149 | name = "semver" 150 | version = "1.0.21" 151 | source = "registry+https://github.com/rust-lang/crates.io-index" 152 | checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" 153 | 154 | [[package]] 155 | name = "serde" 156 | version = "1.0.196" 157 | source = "registry+https://github.com/rust-lang/crates.io-index" 158 | checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" 159 | dependencies = [ 160 | "serde_derive", 161 | ] 162 | 163 | [[package]] 164 | name = "serde_derive" 165 | version = "1.0.196" 166 | source = "registry+https://github.com/rust-lang/crates.io-index" 167 | checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" 168 | dependencies = [ 169 | "proc-macro2", 170 | "quote", 171 | "syn 2.0.48", 172 | ] 173 | 174 | [[package]] 175 | name = "serde_json" 176 | version = "1.0.113" 177 | source = "registry+https://github.com/rust-lang/crates.io-index" 178 | checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" 179 | dependencies = [ 180 | "itoa", 181 | "ryu", 182 | "serde", 183 | ] 184 | 185 | [[package]] 186 | name = "smallvec" 187 | version = "1.13.1" 188 | source = "registry+https://github.com/rust-lang/crates.io-index" 189 | checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" 190 | 191 | [[package]] 192 | name = "spdx" 193 | version = "0.10.3" 194 | source = "registry+https://github.com/rust-lang/crates.io-index" 195 | checksum = "62bde1398b09b9f93fc2fc9b9da86e362693e999d3a54a8ac47a99a5a73f638b" 196 | dependencies = [ 197 | "smallvec", 198 | ] 199 | 200 | [[package]] 201 | name = "syn" 202 | version = "1.0.109" 203 | source = "registry+https://github.com/rust-lang/crates.io-index" 204 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 205 | dependencies = [ 206 | "proc-macro2", 207 | "quote", 208 | "unicode-ident", 209 | ] 210 | 211 | [[package]] 212 | name = "syn" 213 | version = "2.0.48" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" 216 | dependencies = [ 217 | "proc-macro2", 218 | "quote", 219 | "unicode-ident", 220 | ] 221 | 222 | [[package]] 223 | name = "unicode-ident" 224 | version = "1.0.12" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 227 | 228 | [[package]] 229 | name = "unicode-segmentation" 230 | version = "1.10.1" 231 | source = "registry+https://github.com/rust-lang/crates.io-index" 232 | checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" 233 | 234 | [[package]] 235 | name = "unicode-xid" 236 | version = "0.2.4" 237 | source = "registry+https://github.com/rust-lang/crates.io-index" 238 | checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" 239 | 240 | [[package]] 241 | name = "wasm-encoder" 242 | version = "0.201.0" 243 | source = "registry+https://github.com/rust-lang/crates.io-index" 244 | checksum = "b9c7d2731df60006819b013f64ccc2019691deccf6e11a1804bc850cd6748f1a" 245 | dependencies = [ 246 | "leb128", 247 | ] 248 | 249 | [[package]] 250 | name = "wasm-metadata" 251 | version = "0.201.0" 252 | source = "registry+https://github.com/rust-lang/crates.io-index" 253 | checksum = "0fd83062c17b9f4985d438603cde0a5e8c5c8198201a6937f778b607924c7da2" 254 | dependencies = [ 255 | "anyhow", 256 | "indexmap", 257 | "serde", 258 | "serde_derive", 259 | "serde_json", 260 | "spdx", 261 | "wasm-encoder", 262 | "wasmparser", 263 | ] 264 | 265 | [[package]] 266 | name = "wasmparser" 267 | version = "0.201.0" 268 | source = "registry+https://github.com/rust-lang/crates.io-index" 269 | checksum = "84e5df6dba6c0d7fafc63a450f1738451ed7a0b52295d83e868218fa286bf708" 270 | dependencies = [ 271 | "bitflags", 272 | "indexmap", 273 | "semver", 274 | ] 275 | 276 | [[package]] 277 | name = "wit-bindgen" 278 | version = "0.21.0" 279 | source = "registry+https://github.com/rust-lang/crates.io-index" 280 | checksum = "dbdedb8dd38c89c2cfa71e7450515f1c17f94cc2853881652d005b10f3f2559c" 281 | dependencies = [ 282 | "bitflags", 283 | "wit-bindgen-rt", 284 | "wit-bindgen-rust-macro", 285 | ] 286 | 287 | [[package]] 288 | name = "wit-bindgen-core" 289 | version = "0.21.0" 290 | source = "registry+https://github.com/rust-lang/crates.io-index" 291 | checksum = "4ebcbf07363368a9e6e8b89c18bff176c4f35ed2dd2f2f5f9c473bb56813369b" 292 | dependencies = [ 293 | "anyhow", 294 | "wit-parser", 295 | ] 296 | 297 | [[package]] 298 | name = "wit-bindgen-rt" 299 | version = "0.21.0" 300 | source = "registry+https://github.com/rust-lang/crates.io-index" 301 | checksum = "026d24a27f6712541fa534f2954bd9e0eb66172f033c2157c0f31d106255c497" 302 | 303 | [[package]] 304 | name = "wit-bindgen-rust" 305 | version = "0.21.0" 306 | source = "registry+https://github.com/rust-lang/crates.io-index" 307 | checksum = "a2a4d36bf13b5ef534599d24dae792b1ae2b40fe1248c2754fd3f7343fb2ca70" 308 | dependencies = [ 309 | "anyhow", 310 | "heck", 311 | "indexmap", 312 | "wasm-metadata", 313 | "wit-bindgen-core", 314 | "wit-component", 315 | ] 316 | 317 | [[package]] 318 | name = "wit-bindgen-rust-macro" 319 | version = "0.21.0" 320 | source = "registry+https://github.com/rust-lang/crates.io-index" 321 | checksum = "bf4bf1b15b5227d1ca9ba7fc6a7850c72f9df0fbac3c46a0855763cc454ff11a" 322 | dependencies = [ 323 | "anyhow", 324 | "proc-macro2", 325 | "quote", 326 | "syn 2.0.48", 327 | "wit-bindgen-core", 328 | "wit-bindgen-rust", 329 | ] 330 | 331 | [[package]] 332 | name = "wit-component" 333 | version = "0.201.0" 334 | source = "registry+https://github.com/rust-lang/crates.io-index" 335 | checksum = "421c0c848a0660a8c22e2fd217929a0191f14476b68962afd2af89fd22e39825" 336 | dependencies = [ 337 | "anyhow", 338 | "bitflags", 339 | "indexmap", 340 | "log", 341 | "serde", 342 | "serde_derive", 343 | "serde_json", 344 | "wasm-encoder", 345 | "wasm-metadata", 346 | "wasmparser", 347 | "wit-parser", 348 | ] 349 | 350 | [[package]] 351 | name = "wit-parser" 352 | version = "0.201.0" 353 | source = "registry+https://github.com/rust-lang/crates.io-index" 354 | checksum = "196d3ecfc4b759a8573bf86a9b3f8996b304b3732e4c7de81655f875f6efdca6" 355 | dependencies = [ 356 | "anyhow", 357 | "id-arena", 358 | "indexmap", 359 | "log", 360 | "semver", 361 | "serde", 362 | "serde_derive", 363 | "serde_json", 364 | "unicode-xid", 365 | "wasmparser", 366 | ] 367 | -------------------------------------------------------------------------------- /examples/authenticate-http/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "authenticate-http" 3 | version = "0.0.0" 4 | publish = false 5 | edition = "2021" 6 | rust-version = "1.80.0" 7 | 8 | [lib] 9 | crate-type = ["cdylib"] 10 | 11 | [dependencies] 12 | anyhow = "1.0.79" 13 | kiwi-sdk = { path = "../../src/kiwi-sdk" } 14 | http = "1.0.0" 15 | 16 | [workspace] 17 | -------------------------------------------------------------------------------- /examples/authenticate-http/README.md: -------------------------------------------------------------------------------- 1 | # Authenticate w/ Outbound HTTP 2 | 3 | This example demonstrates how to make outbound HTTP requests in the authenticate hook using the Kiwi SDK. 4 | 5 | - [Authenticate w/ Outbound HTTP](#authenticate-w-outbound-http) 6 | - [Running the Example](#running-the-example) 7 | - [Building the WASM Hook](#building-the-wasm-hook) 8 | - [Running Kiwi](#running-kiwi) 9 | - [Establishing a Connection](#establishing-a-connection) 10 | - [Recap](#recap) 11 | 12 | ## Running the Example 13 | 14 | > **NOTE**: The commands in this example should be run from this directory (`examples/authenticate-http`). 15 | 16 | ### Building the WASM Hook 17 | 18 | The `wasm32-wasip1` target is required to build the WASM hook. This target is not installed by default, so it must be added using the following command: 19 | 20 | ```sh 21 | rustup target add wasm32-wasip1 22 | ``` 23 | 24 | Once the target is installed, the WASM hook can be built using the following command: 25 | 26 | ```sh 27 | cargo build --target wasm32-wasip1 28 | ``` 29 | 30 | This command will produce the WASM hook at `target/wasm32-wasip1/debug/authenticate_http.wasm`. 31 | 32 | ### Running Kiwi 33 | 34 | Now that the WASM hook is built, it can be run with Kiwi. The following command will run Kiwi with the WASM hook and the provided configuration file: 35 | 36 | ```sh 37 | docker run -p 8000:8000 -v $(pwd)/kiwi.yml:/etc/kiwi/config/kiwi.yml \ 38 | -v $(pwd)/target/wasm32-wasip1/debug/authenticate_http.wasm:/etc/kiwi/hook/authenticate.wasm \ 39 | ghcr.io/rkrishn7/kiwi:main 40 | ``` 41 | 42 | ### Establishing a Connection 43 | 44 | Now we can interact with the Kiwi server at `ws://localhost:8000`. Let's try it out by subscribing to a counter source and emitting some events. First, let's try to connect to the server using `wscat`, without providing an `x-api-key` query parameter: 45 | 46 | ```sh 47 | wscat -c ws://127.0.0.1:8000 48 | ``` 49 | 50 | The connection should be rejected by the server. Now, let's try to connect to the server, making sure to provide an `x-api-key` query parameter: 51 | 52 | ```sh 53 | wscat -c ws://127.0.0.1:8000/some-path?x-api-key=secret 54 | ``` 55 | 56 | Success! The connection should be accepted by the server. Behind the scenes, your WASM hook is making an outbound HTTP request to a mock authentication server to verify the `x-api-key` query parameter. 57 | 58 | ## Recap 59 | 60 | This example demonstrated how to write an authentication hook using the Kiwi SDK and load it into Kiwi for execution. The hook makes an outbound HTTP request to a mock authentication server to verify the `x-api-key` query parameter. 61 | 62 | Real-world use cases for this type of hook include verifying JWT tokens, checking for specific user roles, and more. 63 | -------------------------------------------------------------------------------- /examples/authenticate-http/kiwi.yml: -------------------------------------------------------------------------------- 1 | hooks: 2 | authenticate: '/etc/kiwi/hook/authenticate.wasm' 3 | 4 | sources: 5 | - type: counter 6 | id: counter1 7 | interval_ms: 1000 8 | lazy: true 9 | min: 0 10 | 11 | server: 12 | address: '0.0.0.0:8000' 13 | -------------------------------------------------------------------------------- /examples/authenticate-http/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! An example authenticate hook that parses an API key from the request query string 2 | //! and makes a request to an API to verify the key 3 | 4 | use http::request::Builder; 5 | use kiwi_sdk::hook::authenticate::{authenticate, Outcome}; 6 | use kiwi_sdk::http::{request as http_request, Request}; 7 | 8 | /// You must use the `#[intercept]` macro to define an intercept hook. 9 | #[authenticate] 10 | fn handle(req: Request<()>) -> Outcome { 11 | let query = match req.uri().query() { 12 | Some(query) => query, 13 | // Returning `Outcome::Reject` instructs Kiwi to reject the connection 14 | None => return Outcome::Reject, 15 | }; 16 | 17 | let parts: Vec<&str> = query.split('&').collect(); 18 | 19 | // Parse the query string to find the API key 20 | // If the API key is not found, reject the connection 21 | let key = { 22 | let mut token = None; 23 | for (key, value) in parts.iter().map(|part| { 24 | let mut parts = part.split('='); 25 | (parts.next().unwrap(), parts.next().unwrap()) 26 | }) { 27 | if key == "x-api-key" { 28 | token = Some(value); 29 | break; 30 | } 31 | } 32 | 33 | if let Some(token) = token { 34 | token 35 | } else { 36 | return Outcome::Reject; 37 | } 38 | }; 39 | 40 | let request = Builder::new() 41 | .method("GET") 42 | .uri("https://example.com") 43 | .header("x-api-key", key) 44 | .body(Vec::new()) 45 | .unwrap(); 46 | 47 | // Make a request to the API to verify the API key 48 | match http_request(request) { 49 | Ok(res) => { 50 | if res.status() == 200 { 51 | // Returning `Outcome::Authenticate` instructs Kiwi to allow the connection to be established. 52 | Outcome::Authenticate 53 | } else { 54 | Outcome::Reject 55 | } 56 | } 57 | Err(_) => { 58 | Outcome::Reject 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /examples/intercept-simple/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /examples/intercept-simple/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "anyhow" 7 | version = "1.0.79" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" 10 | 11 | [[package]] 12 | name = "bitflags" 13 | version = "2.4.2" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" 16 | 17 | [[package]] 18 | name = "bytes" 19 | version = "1.5.0" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" 22 | 23 | [[package]] 24 | name = "equivalent" 25 | version = "1.0.1" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" 28 | 29 | [[package]] 30 | name = "fnv" 31 | version = "1.0.7" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 34 | 35 | [[package]] 36 | name = "hashbrown" 37 | version = "0.14.3" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" 40 | 41 | [[package]] 42 | name = "heck" 43 | version = "0.4.1" 44 | source = "registry+https://github.com/rust-lang/crates.io-index" 45 | checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" 46 | dependencies = [ 47 | "unicode-segmentation", 48 | ] 49 | 50 | [[package]] 51 | name = "http" 52 | version = "1.0.0" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" 55 | dependencies = [ 56 | "bytes", 57 | "fnv", 58 | "itoa", 59 | ] 60 | 61 | [[package]] 62 | name = "id-arena" 63 | version = "2.2.1" 64 | source = "registry+https://github.com/rust-lang/crates.io-index" 65 | checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" 66 | 67 | [[package]] 68 | name = "indexmap" 69 | version = "2.2.2" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" 72 | dependencies = [ 73 | "equivalent", 74 | "hashbrown", 75 | "serde", 76 | ] 77 | 78 | [[package]] 79 | name = "intercept" 80 | version = "0.0.0" 81 | dependencies = [ 82 | "kiwi-sdk", 83 | ] 84 | 85 | [[package]] 86 | name = "itoa" 87 | version = "1.0.10" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" 90 | 91 | [[package]] 92 | name = "kiwi-macro" 93 | version = "0.1.1-alpha.7" 94 | dependencies = [ 95 | "proc-macro2", 96 | "quote", 97 | "syn 1.0.109", 98 | ] 99 | 100 | [[package]] 101 | name = "kiwi-sdk" 102 | version = "0.1.1-alpha.7" 103 | dependencies = [ 104 | "anyhow", 105 | "http", 106 | "kiwi-macro", 107 | "wit-bindgen", 108 | ] 109 | 110 | [[package]] 111 | name = "leb128" 112 | version = "0.2.5" 113 | source = "registry+https://github.com/rust-lang/crates.io-index" 114 | checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" 115 | 116 | [[package]] 117 | name = "log" 118 | version = "0.4.20" 119 | source = "registry+https://github.com/rust-lang/crates.io-index" 120 | checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" 121 | 122 | [[package]] 123 | name = "proc-macro2" 124 | version = "1.0.78" 125 | source = "registry+https://github.com/rust-lang/crates.io-index" 126 | checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" 127 | dependencies = [ 128 | "unicode-ident", 129 | ] 130 | 131 | [[package]] 132 | name = "quote" 133 | version = "1.0.35" 134 | source = "registry+https://github.com/rust-lang/crates.io-index" 135 | checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" 136 | dependencies = [ 137 | "proc-macro2", 138 | ] 139 | 140 | [[package]] 141 | name = "ryu" 142 | version = "1.0.16" 143 | source = "registry+https://github.com/rust-lang/crates.io-index" 144 | checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" 145 | 146 | [[package]] 147 | name = "semver" 148 | version = "1.0.21" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" 151 | 152 | [[package]] 153 | name = "serde" 154 | version = "1.0.196" 155 | source = "registry+https://github.com/rust-lang/crates.io-index" 156 | checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" 157 | dependencies = [ 158 | "serde_derive", 159 | ] 160 | 161 | [[package]] 162 | name = "serde_derive" 163 | version = "1.0.196" 164 | source = "registry+https://github.com/rust-lang/crates.io-index" 165 | checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" 166 | dependencies = [ 167 | "proc-macro2", 168 | "quote", 169 | "syn 2.0.48", 170 | ] 171 | 172 | [[package]] 173 | name = "serde_json" 174 | version = "1.0.113" 175 | source = "registry+https://github.com/rust-lang/crates.io-index" 176 | checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" 177 | dependencies = [ 178 | "itoa", 179 | "ryu", 180 | "serde", 181 | ] 182 | 183 | [[package]] 184 | name = "smallvec" 185 | version = "1.13.1" 186 | source = "registry+https://github.com/rust-lang/crates.io-index" 187 | checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" 188 | 189 | [[package]] 190 | name = "spdx" 191 | version = "0.10.3" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "62bde1398b09b9f93fc2fc9b9da86e362693e999d3a54a8ac47a99a5a73f638b" 194 | dependencies = [ 195 | "smallvec", 196 | ] 197 | 198 | [[package]] 199 | name = "syn" 200 | version = "1.0.109" 201 | source = "registry+https://github.com/rust-lang/crates.io-index" 202 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 203 | dependencies = [ 204 | "proc-macro2", 205 | "quote", 206 | "unicode-ident", 207 | ] 208 | 209 | [[package]] 210 | name = "syn" 211 | version = "2.0.48" 212 | source = "registry+https://github.com/rust-lang/crates.io-index" 213 | checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" 214 | dependencies = [ 215 | "proc-macro2", 216 | "quote", 217 | "unicode-ident", 218 | ] 219 | 220 | [[package]] 221 | name = "unicode-ident" 222 | version = "1.0.12" 223 | source = "registry+https://github.com/rust-lang/crates.io-index" 224 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 225 | 226 | [[package]] 227 | name = "unicode-segmentation" 228 | version = "1.10.1" 229 | source = "registry+https://github.com/rust-lang/crates.io-index" 230 | checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" 231 | 232 | [[package]] 233 | name = "unicode-xid" 234 | version = "0.2.4" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" 237 | 238 | [[package]] 239 | name = "wasm-encoder" 240 | version = "0.201.0" 241 | source = "registry+https://github.com/rust-lang/crates.io-index" 242 | checksum = "b9c7d2731df60006819b013f64ccc2019691deccf6e11a1804bc850cd6748f1a" 243 | dependencies = [ 244 | "leb128", 245 | ] 246 | 247 | [[package]] 248 | name = "wasm-metadata" 249 | version = "0.201.0" 250 | source = "registry+https://github.com/rust-lang/crates.io-index" 251 | checksum = "0fd83062c17b9f4985d438603cde0a5e8c5c8198201a6937f778b607924c7da2" 252 | dependencies = [ 253 | "anyhow", 254 | "indexmap", 255 | "serde", 256 | "serde_derive", 257 | "serde_json", 258 | "spdx", 259 | "wasm-encoder", 260 | "wasmparser", 261 | ] 262 | 263 | [[package]] 264 | name = "wasmparser" 265 | version = "0.201.0" 266 | source = "registry+https://github.com/rust-lang/crates.io-index" 267 | checksum = "84e5df6dba6c0d7fafc63a450f1738451ed7a0b52295d83e868218fa286bf708" 268 | dependencies = [ 269 | "bitflags", 270 | "indexmap", 271 | "semver", 272 | ] 273 | 274 | [[package]] 275 | name = "wit-bindgen" 276 | version = "0.21.0" 277 | source = "registry+https://github.com/rust-lang/crates.io-index" 278 | checksum = "dbdedb8dd38c89c2cfa71e7450515f1c17f94cc2853881652d005b10f3f2559c" 279 | dependencies = [ 280 | "bitflags", 281 | "wit-bindgen-rt", 282 | "wit-bindgen-rust-macro", 283 | ] 284 | 285 | [[package]] 286 | name = "wit-bindgen-core" 287 | version = "0.21.0" 288 | source = "registry+https://github.com/rust-lang/crates.io-index" 289 | checksum = "4ebcbf07363368a9e6e8b89c18bff176c4f35ed2dd2f2f5f9c473bb56813369b" 290 | dependencies = [ 291 | "anyhow", 292 | "wit-parser", 293 | ] 294 | 295 | [[package]] 296 | name = "wit-bindgen-rt" 297 | version = "0.21.0" 298 | source = "registry+https://github.com/rust-lang/crates.io-index" 299 | checksum = "026d24a27f6712541fa534f2954bd9e0eb66172f033c2157c0f31d106255c497" 300 | 301 | [[package]] 302 | name = "wit-bindgen-rust" 303 | version = "0.21.0" 304 | source = "registry+https://github.com/rust-lang/crates.io-index" 305 | checksum = "a2a4d36bf13b5ef534599d24dae792b1ae2b40fe1248c2754fd3f7343fb2ca70" 306 | dependencies = [ 307 | "anyhow", 308 | "heck", 309 | "indexmap", 310 | "wasm-metadata", 311 | "wit-bindgen-core", 312 | "wit-component", 313 | ] 314 | 315 | [[package]] 316 | name = "wit-bindgen-rust-macro" 317 | version = "0.21.0" 318 | source = "registry+https://github.com/rust-lang/crates.io-index" 319 | checksum = "bf4bf1b15b5227d1ca9ba7fc6a7850c72f9df0fbac3c46a0855763cc454ff11a" 320 | dependencies = [ 321 | "anyhow", 322 | "proc-macro2", 323 | "quote", 324 | "syn 2.0.48", 325 | "wit-bindgen-core", 326 | "wit-bindgen-rust", 327 | ] 328 | 329 | [[package]] 330 | name = "wit-component" 331 | version = "0.201.0" 332 | source = "registry+https://github.com/rust-lang/crates.io-index" 333 | checksum = "421c0c848a0660a8c22e2fd217929a0191f14476b68962afd2af89fd22e39825" 334 | dependencies = [ 335 | "anyhow", 336 | "bitflags", 337 | "indexmap", 338 | "log", 339 | "serde", 340 | "serde_derive", 341 | "serde_json", 342 | "wasm-encoder", 343 | "wasm-metadata", 344 | "wasmparser", 345 | "wit-parser", 346 | ] 347 | 348 | [[package]] 349 | name = "wit-parser" 350 | version = "0.201.0" 351 | source = "registry+https://github.com/rust-lang/crates.io-index" 352 | checksum = "196d3ecfc4b759a8573bf86a9b3f8996b304b3732e4c7de81655f875f6efdca6" 353 | dependencies = [ 354 | "anyhow", 355 | "id-arena", 356 | "indexmap", 357 | "log", 358 | "semver", 359 | "serde", 360 | "serde_derive", 361 | "serde_json", 362 | "unicode-xid", 363 | "wasmparser", 364 | ] 365 | -------------------------------------------------------------------------------- /examples/intercept-simple/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intercept" 3 | version = "0.0.0" 4 | publish = false 5 | edition = "2021" 6 | rust-version = "1.80.0" 7 | 8 | [lib] 9 | crate-type = ["cdylib"] 10 | 11 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 12 | 13 | [dependencies] 14 | kiwi-sdk = { path = "../../src/kiwi-sdk" } 15 | 16 | [workspace] 17 | -------------------------------------------------------------------------------- /examples/intercept-simple/README.md: -------------------------------------------------------------------------------- 1 | # Intercept (Simple) 2 | 3 | This example highlights how to write a simple WebAssembly (WASM) hook using the Kiwi SDK and load it into the Kiwi runtime. The hook is a simple plugin that intercepts all all events emitted by counter sources and discards them if they are odd. 4 | 5 | - [Intercept (Simple)](#intercept-simple) 6 | - [Running the Example](#running-the-example) 7 | - [Building the WASM Hook](#building-the-wasm-hook) 8 | - [Running Kiwi](#running-kiwi) 9 | - [Interacting with Kiwi](#interacting-with-kiwi) 10 | - [Recap](#recap) 11 | 12 | ## Running the Example 13 | 14 | > **NOTE**: The commands in this example should be run from this directory (`examples/intercept-simple`). 15 | 16 | ### Building the WASM Hook 17 | 18 | The `wasm32-wasip1` target is required to build the WASM hook. This target is not installed by default, so it must be added using the following command: 19 | 20 | ```sh 21 | rustup target add wasm32-wasip1 22 | ``` 23 | 24 | Once the target is installed, the WASM hook can be built using the following command: 25 | 26 | ```sh 27 | cargo build --target wasm32-wasip1 28 | ``` 29 | 30 | This command will produce the WASM hook at `target/wasm32-wasip1/debug/intercept.wasm`. 31 | 32 | ### Running Kiwi 33 | 34 | Now that the WASM hook is built, it can be run with Kiwi. The following command will run Kiwi with the WASM hook and the provided configuration file: 35 | 36 | ```sh 37 | docker run -p 8000:8000 -v $(pwd)/kiwi.yml:/etc/kiwi/config/kiwi.yml \ 38 | -v $(pwd)/target/wasm32-wasip1/debug/intercept.wasm:/etc/kiwi/hook/intercept.wasm \ 39 | ghcr.io/rkrishn7/kiwi:main 40 | ``` 41 | 42 | ### Interacting with Kiwi 43 | 44 | Now we can interact with the Kiwi server at `ws://localhost:8000`. Let's try it out by subscribing to a counter source and emitting some events. First, let's connect to the server using `wscat`: 45 | 46 | ```sh 47 | wscat -c ws://127.0.0.1:8000 48 | ``` 49 | 50 | Now that we're connected, let's subscribe to the counter source. In the `wscat` terminal, send the following message: 51 | 52 | ```json 53 | {"type":"SUBSCRIBE","sourceId":"counter1"} 54 | ``` 55 | 56 | The server should respond with a message indicating that the subscription was successful. After this, you should start receiving events from the counter source. Note that even though the counter source is emitting all the natural numbers, the WASM hook is intercepting and discarding all odd numbers. Thus, you only see the even numbers in the terminal! 57 | 58 | ## Recap 59 | 60 | This example demonstrated how to write a simple WebAssembly (WASM) hook using the Kiwi SDK and load it into Kiwi for execution. 61 | -------------------------------------------------------------------------------- /examples/intercept-simple/kiwi.yml: -------------------------------------------------------------------------------- 1 | hooks: 2 | intercept: '/etc/kiwi/hook/intercept.wasm' 3 | 4 | sources: 5 | - type: counter 6 | id: counter1 7 | interval_ms: 1000 8 | lazy: true 9 | min: 0 10 | 11 | server: 12 | address: '0.0.0.0:8000' 13 | -------------------------------------------------------------------------------- /examples/intercept-simple/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A simple intercept hook that discards odd numbers from all counter sources 2 | 3 | use kiwi_sdk::hook::intercept::{intercept, Action, Context, CounterEventCtx, EventCtx}; 4 | 5 | /// You must use the `#[intercept]` macro to define an intercept hook. 6 | #[intercept] 7 | fn handle(ctx: Context) -> Action { 8 | match ctx.event { 9 | // We only care about counter sources in this example 10 | EventCtx::Counter(CounterEventCtx { 11 | source_id: _, 12 | count, 13 | }) => { 14 | if count % 2 == 0 { 15 | // Returning `Action::Forward` instructs Kiwi to forward the event 16 | // to the associated client. 17 | return Action::Forward; 18 | } else { 19 | // Returning `Action::Discard` instructs Kiwi to discard the event, 20 | // preventing it from reaching the associated client. 21 | return Action::Discard; 22 | } 23 | } 24 | _ => {} 25 | } 26 | 27 | Action::Forward 28 | } 29 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | tab_spaces = 4 3 | -------------------------------------------------------------------------------- /src/kiwi-sdk/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiwi-sdk" 3 | version = "0.1.1" 4 | rust-version = "1.71.1" 5 | edition.workspace = true 6 | authors.workspace = true 7 | license.workspace = true 8 | repository.workspace = true 9 | description = "The Kiwi SDK allows you to write pluggable modules that hook into the Kiwi runtime." 10 | 11 | [dependencies] 12 | anyhow = "1.0.79" 13 | kiwi-macro = { version = "~0.1.1", path = "./macro" } 14 | wit-bindgen = "0.21.0" 15 | http = "1.0.0" 16 | -------------------------------------------------------------------------------- /src/kiwi-sdk/macro/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiwi-macro" 3 | version = "0.1.1" 4 | rust-version = "1.71.1" 5 | edition.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "Macros for the Kiwi SDK" 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | 12 | [lib] 13 | proc-macro = true 14 | 15 | [dependencies] 16 | proc-macro2 = "1" 17 | quote = "1.0" 18 | syn = { version = "1.0", features = ["full"] } 19 | -------------------------------------------------------------------------------- /src/kiwi-sdk/macro/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate exports macros that are are used to generate the necessary code 2 | //! to turn a source file into a `Guest` module, suitable for compilation and 3 | //! execution within Kiwi's embedded WASM component runtime ([wasmtime](https://github.com/bytecodealliance/wasmtime)). 4 | //! 5 | //! ### NOTE 6 | //! This crate is intended for use only via the Kiwi SDK and should not be used directly. 7 | 8 | use proc_macro::TokenStream; 9 | use quote::quote; 10 | 11 | /// Wit sources are packaged as part of the release process, so we can reference them 12 | /// from the crate root. We need to hoist the path here to ensure it references the correct 13 | /// location. 14 | const WIT_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/wit"); 15 | 16 | /// Macro necessary for creating an intercept hook. 17 | #[proc_macro_attribute] 18 | pub fn intercept(_attr: TokenStream, item: TokenStream) -> TokenStream { 19 | let func = syn::parse_macro_input!(item as syn::ItemFn); 20 | let func_name = &func.sig.ident; 21 | 22 | // Note that intercept modules don't link in WASI, so there's no need 23 | // to remap any WASI imports to their counterparts located in the Kiwi SDK. 24 | // In fact, because Kiwi does not link in WASI for intercept modules, attempting 25 | // to use WASI imports should result in an error. 26 | quote!( 27 | #func 28 | mod __kiwi_intercept { 29 | mod bindings { 30 | #![allow(missing_docs)] 31 | ::kiwi_sdk::wit_bindgen::generate!({ 32 | path: #WIT_PATH, 33 | world: "intercept-hook", 34 | runtime_path: "::kiwi_sdk::wit_bindgen::rt", 35 | }); 36 | } 37 | 38 | struct Kiwi; 39 | 40 | impl bindings::Guest for Kiwi { 41 | fn intercept(ctx: self::bindings::kiwi::kiwi::intercept_types::Context) -> self::bindings::kiwi::kiwi::intercept_types::Action { 42 | super::#func_name(ctx.into()).into() 43 | } 44 | } 45 | 46 | impl From for ::kiwi_sdk::hook::intercept::Context { 47 | fn from(value: self::bindings::kiwi::kiwi::intercept_types::Context) -> Self { 48 | Self { 49 | auth: value.auth.map(|raw| { 50 | ::kiwi_sdk::hook::intercept::AuthCtx { 51 | raw, 52 | } 53 | }), 54 | connection: value.connection.into(), 55 | event: value.event.into(), 56 | } 57 | } 58 | } 59 | 60 | impl From for ::kiwi_sdk::hook::intercept::EventCtx { 61 | fn from(value: self::bindings::kiwi::kiwi::intercept_types::EventCtx) -> Self { 62 | match value { 63 | self::bindings::kiwi::kiwi::intercept_types::EventCtx::Kafka(ctx) => Self::Kafka(ctx.into()), 64 | self::bindings::kiwi::kiwi::intercept_types::EventCtx::Counter(ctx) => Self::Counter(ctx.into()), 65 | } 66 | } 67 | } 68 | 69 | impl From for ::kiwi_sdk::hook::intercept::CounterEventCtx { 70 | fn from(value: self::bindings::kiwi::kiwi::intercept_types::CounterEventCtx) -> Self { 71 | Self { 72 | source_id: value.source_id, 73 | count: value.count, 74 | } 75 | } 76 | } 77 | 78 | impl From for ::kiwi_sdk::hook::intercept::KafkaEventCtx { 79 | fn from(value: self::bindings::kiwi::kiwi::intercept_types::KafkaEventCtx) -> Self { 80 | let timestamp: Option = value.timestamp.map(|t| t.try_into().expect("timestamp conversion must not fail")); 81 | let partition: i32 = value.partition.try_into().expect("partition conversion must not fail"); 82 | let offset: i64 = value.offset.try_into().expect("offset conversion must not fail"); 83 | 84 | Self { 85 | payload: value.payload, 86 | topic: value.topic, 87 | timestamp, 88 | partition, 89 | offset, 90 | } 91 | } 92 | } 93 | 94 | impl From for ::kiwi_sdk::hook::intercept::ConnectionCtx { 95 | fn from(value: self::bindings::kiwi::kiwi::intercept_types::ConnectionCtx) -> Self { 96 | match value { 97 | self::bindings::kiwi::kiwi::intercept_types::ConnectionCtx::Websocket(ctx) => Self::WebSocket(ctx.into()), 98 | } 99 | } 100 | } 101 | 102 | impl From for ::kiwi_sdk::hook::intercept::WebSocketConnectionCtx { 103 | fn from(value: self::bindings::kiwi::kiwi::intercept_types::Websocket) -> Self { 104 | Self { 105 | addr: value.addr, 106 | } 107 | } 108 | } 109 | 110 | impl From<::kiwi_sdk::hook::intercept::Action> for self::bindings::kiwi::kiwi::intercept_types::Action { 111 | fn from(value: ::kiwi_sdk::hook::intercept::Action) -> Self { 112 | match value { 113 | ::kiwi_sdk::hook::intercept::Action::Forward => Self::Forward, 114 | ::kiwi_sdk::hook::intercept::Action::Discard => Self::Discard, 115 | ::kiwi_sdk::hook::intercept::Action::Transform(payload) => Self::Transform(payload.into()), 116 | } 117 | } 118 | } 119 | 120 | impl From<::kiwi_sdk::hook::intercept::TransformedPayload> for self::bindings::kiwi::kiwi::intercept_types::TransformedPayload { 121 | fn from(value: ::kiwi_sdk::hook::intercept::TransformedPayload) -> Self { 122 | match value { 123 | ::kiwi_sdk::hook::intercept::TransformedPayload::Kafka(payload) => Self::Kafka(payload), 124 | ::kiwi_sdk::hook::intercept::TransformedPayload::Counter(count) => Self::Counter(count), 125 | } 126 | } 127 | } 128 | 129 | bindings::export!(Kiwi with_types_in bindings); 130 | } 131 | ) 132 | .into() 133 | } 134 | 135 | /// Macro necessary for creating an authenticate hook. 136 | #[proc_macro_attribute] 137 | pub fn authenticate(_attr: TokenStream, item: TokenStream) -> TokenStream { 138 | let func = syn::parse_macro_input!(item as syn::ItemFn); 139 | let func_name = &func.sig.ident; 140 | 141 | // Kiwi does link in WASI for authenticate modules, so we need to remap the 142 | // WASI imports to their counterparts located in the Kiwi SDK. 143 | quote!( 144 | #func 145 | mod __kiwi_authenticate { 146 | mod bindings { 147 | #![allow(missing_docs)] 148 | ::kiwi_sdk::wit_bindgen::generate!({ 149 | path: #WIT_PATH, 150 | world: "authenticate-hook", 151 | runtime_path: "::kiwi_sdk::wit_bindgen::rt", 152 | with: { 153 | "wasi:http/outgoing-handler@0.2.0": ::kiwi_sdk::wit::wasi::http::outgoing_handler, 154 | "wasi:http/types@0.2.0": ::kiwi_sdk::wit::wasi::http::types, 155 | "wasi:clocks/monotonic-clock@0.2.0": ::kiwi_sdk::wit::wasi::clocks::monotonic_clock, 156 | "wasi:io/poll@0.2.0": ::kiwi_sdk::wit::wasi::io::poll, 157 | "wasi:io/streams@0.2.0": ::kiwi_sdk::wit::wasi::io::streams, 158 | "wasi:io/error@0.2.0": ::kiwi_sdk::wit::wasi::io::error, 159 | }, 160 | }); 161 | } 162 | 163 | struct Kiwi; 164 | 165 | impl bindings::Guest for Kiwi { 166 | fn authenticate(incoming: self::bindings::kiwi::kiwi::authenticate_types::HttpRequest) -> self::bindings::kiwi::kiwi::authenticate_types::Outcome { 167 | super::#func_name(incoming.into()).into() 168 | } 169 | } 170 | 171 | impl From<::kiwi_sdk::hook::authenticate::Outcome> for self::bindings::kiwi::kiwi::authenticate_types::Outcome { 172 | fn from(value: ::kiwi_sdk::hook::authenticate::Outcome) -> Self { 173 | match value { 174 | ::kiwi_sdk::hook::authenticate::Outcome::Authenticate => Self::Authenticate, 175 | ::kiwi_sdk::hook::authenticate::Outcome::Reject => Self::Reject, 176 | ::kiwi_sdk::hook::authenticate::Outcome::WithContext(payload) => Self::WithContext(payload), 177 | } 178 | } 179 | } 180 | 181 | impl From for ::kiwi_sdk::http::Request<()> { 182 | fn from(value: self::bindings::kiwi::kiwi::authenticate_types::HttpRequest) -> Self { 183 | let mut uri_builder = ::kiwi_sdk::http::Uri::builder(); 184 | 185 | if let Some(scheme) = value.scheme { 186 | let scheme = match scheme { 187 | ::kiwi_sdk::wit::wasi::http::types::Scheme::Http => ::kiwi_sdk::http::Scheme::HTTP, 188 | ::kiwi_sdk::wit::wasi::http::types::Scheme::Https => ::kiwi_sdk::http::Scheme::HTTPS, 189 | ::kiwi_sdk::wit::wasi::http::types::Scheme::Other(scheme) => scheme.as_str().parse().expect("failed to parse scheme"), 190 | }; 191 | 192 | uri_builder = uri_builder.scheme(scheme); 193 | } 194 | 195 | if let Some(authority) = value.authority { 196 | uri_builder = uri_builder.authority(authority.as_str()); 197 | } 198 | 199 | if let Some(path_with_query) = value.path_with_query { 200 | uri_builder = uri_builder.path_and_query(path_with_query.as_str()); 201 | } 202 | 203 | let uri = uri_builder.build().expect("failed to build uri"); 204 | 205 | let method = match value.method { 206 | ::kiwi_sdk::wit::wasi::http::types::Method::Get => ::kiwi_sdk::http::Method::GET, 207 | ::kiwi_sdk::wit::wasi::http::types::Method::Head => ::kiwi_sdk::http::Method::HEAD, 208 | ::kiwi_sdk::wit::wasi::http::types::Method::Post => ::kiwi_sdk::http::Method::POST, 209 | ::kiwi_sdk::wit::wasi::http::types::Method::Put => ::kiwi_sdk::http::Method::PUT, 210 | ::kiwi_sdk::wit::wasi::http::types::Method::Delete => ::kiwi_sdk::http::Method::DELETE, 211 | ::kiwi_sdk::wit::wasi::http::types::Method::Connect => ::kiwi_sdk::http::Method::CONNECT, 212 | ::kiwi_sdk::wit::wasi::http::types::Method::Options => ::kiwi_sdk::http::Method::OPTIONS, 213 | ::kiwi_sdk::wit::wasi::http::types::Method::Trace => ::kiwi_sdk::http::Method::TRACE, 214 | ::kiwi_sdk::wit::wasi::http::types::Method::Patch => ::kiwi_sdk::http::Method::PATCH, 215 | ::kiwi_sdk::wit::wasi::http::types::Method::Other(_) => panic!("Unknown method"), 216 | }; 217 | 218 | let mut request_builder = ::kiwi_sdk::http::Request::builder() 219 | .method(method) 220 | .uri(uri); 221 | 222 | for (key, value) in value.headers { 223 | request_builder = request_builder.header(key, value); 224 | } 225 | 226 | request_builder.body(()).expect("failed to build request") 227 | } 228 | } 229 | 230 | bindings::export!(Kiwi with_types_in bindings); 231 | } 232 | ) 233 | .into() 234 | } 235 | -------------------------------------------------------------------------------- /src/kiwi-sdk/macro/wit: -------------------------------------------------------------------------------- 1 | ../../wit -------------------------------------------------------------------------------- /src/kiwi-sdk/src/hook/authenticate.rs: -------------------------------------------------------------------------------- 1 | //! Types and macros for building authentication hooks 2 | 3 | pub use kiwi_macro::authenticate; 4 | 5 | /// The outcome of an authentication hook 6 | pub enum Outcome { 7 | /// Authenticate the connection 8 | Authenticate, 9 | /// Reject the connection 10 | Reject, 11 | /// Authenticate the connection and attach custom context for use in the 12 | /// intercept hook 13 | WithContext(Vec), 14 | } 15 | -------------------------------------------------------------------------------- /src/kiwi-sdk/src/hook/intercept.rs: -------------------------------------------------------------------------------- 1 | //! Types and macros for building intercept hooks 2 | 3 | pub use kiwi_macro::intercept; 4 | 5 | #[derive(Debug, Clone)] 6 | /// Represents a transformed source payload 7 | pub enum TransformedPayload { 8 | /// A transformed Kafka payload 9 | Kafka(Option>), 10 | /// A transformed counter payload 11 | Counter(u64), 12 | } 13 | 14 | #[derive(Debug, Clone)] 15 | /// Represents an action to take for a given event 16 | pub enum Action { 17 | /// Forward the event to the client 18 | Forward, 19 | /// Discard the event, preventing it from reaching the client 20 | Discard, 21 | /// Transform the event and send it to the client 22 | Transform(TransformedPayload), 23 | } 24 | 25 | #[derive(Debug, Clone)] 26 | /// An event bundled with additional context 27 | pub struct Context { 28 | /// The authentication context, if any 29 | pub auth: Option, 30 | /// The connection context 31 | pub connection: ConnectionCtx, 32 | /// The event context 33 | pub event: EventCtx, 34 | } 35 | 36 | #[derive(Debug, Clone)] 37 | /// Represents the authentication context as provided by the authentication hook 38 | pub struct AuthCtx { 39 | pub raw: Vec, 40 | } 41 | 42 | #[derive(Debug, Clone)] 43 | /// Represents the connection context 44 | pub enum ConnectionCtx { 45 | /// A WebSocket connection context 46 | WebSocket(WebSocketConnectionCtx), 47 | } 48 | 49 | #[derive(Debug, Clone)] 50 | /// Represents the WebSocket connection context 51 | pub struct WebSocketConnectionCtx { 52 | /// The IP address of the client 53 | pub addr: Option, 54 | } 55 | 56 | #[derive(Debug, Clone)] 57 | /// Represents the event context 58 | pub enum EventCtx { 59 | /// A Kafka event context 60 | Kafka(KafkaEventCtx), 61 | /// A counter event context 62 | Counter(CounterEventCtx), 63 | } 64 | 65 | #[derive(Debug, Clone)] 66 | /// A Kafka event context 67 | pub struct KafkaEventCtx { 68 | /// The payload of the event 69 | pub payload: Option>, 70 | /// The topic to which the event was published 71 | pub topic: String, 72 | /// The timestamp of the event 73 | pub timestamp: Option, 74 | /// The topic partition of the event 75 | pub partition: i32, 76 | /// The offset of the event 77 | pub offset: i64, 78 | } 79 | 80 | #[derive(Debug, Clone)] 81 | /// A counter event context 82 | pub struct CounterEventCtx { 83 | /// The source ID of the counter source 84 | pub source_id: String, 85 | /// The current count of the counter 86 | pub count: u64, 87 | } 88 | -------------------------------------------------------------------------------- /src/kiwi-sdk/src/hook/mod.rs: -------------------------------------------------------------------------------- 1 | //! Types and macros for building Kiwi hooks 2 | 3 | pub mod authenticate; 4 | pub mod intercept; 5 | -------------------------------------------------------------------------------- /src/kiwi-sdk/src/http.rs: -------------------------------------------------------------------------------- 1 | use crate::wit::wasi::http as wasi_http; 2 | 3 | // Re-export some types from the `http` crate for convenience. 4 | pub use http::request::{Builder as RequestBuilder, Request}; 5 | pub use http::response::Response; 6 | pub use http::uri::{Scheme, Uri}; 7 | pub use http::Method; 8 | 9 | impl From<&Method> for wasi_http::types::Method { 10 | fn from(method: &http::Method) -> Self { 11 | if method == http::Method::GET { 12 | wasi_http::types::Method::Get 13 | } else if method == http::Method::HEAD { 14 | wasi_http::types::Method::Head 15 | } else if method == http::Method::POST { 16 | wasi_http::types::Method::Post 17 | } else if method == http::Method::PUT { 18 | wasi_http::types::Method::Put 19 | } else if method == http::Method::DELETE { 20 | wasi_http::types::Method::Delete 21 | } else if method == http::Method::CONNECT { 22 | wasi_http::types::Method::Connect 23 | } else if method == http::Method::OPTIONS { 24 | wasi_http::types::Method::Options 25 | } else if method == http::Method::TRACE { 26 | wasi_http::types::Method::Trace 27 | } else if method == http::Method::PATCH { 28 | wasi_http::types::Method::Patch 29 | } else { 30 | wasi_http::types::Method::Other(method.to_string()) 31 | } 32 | } 33 | } 34 | 35 | // NOTE: This implementation is adapted from https://github.com/bytecodealliance/wasmtime/blob/main/crates/test-programs/src/http.rs 36 | /// Make an outbound HTTP request 37 | pub fn request>(req: Request) -> anyhow::Result>> { 38 | let additional_headers: Vec<(String, Vec)> = req 39 | .headers() 40 | .iter() 41 | .map(|(k, v)| (k.to_string(), v.as_ref().to_owned())) 42 | .collect(); 43 | 44 | let headers = wasi_http::types::Headers::from_list( 45 | &[ 46 | &[( 47 | "User-agent".to_string(), 48 | "WASI-HTTP/0.0.1".to_string().into_bytes(), 49 | )], 50 | &additional_headers[..], 51 | ] 52 | .concat(), 53 | )?; 54 | let scheme = req.uri().scheme().map(|scheme| { 55 | if scheme == &http::uri::Scheme::HTTP { 56 | return wasi_http::types::Scheme::Http; 57 | } 58 | 59 | if scheme == &http::uri::Scheme::HTTPS { 60 | return wasi_http::types::Scheme::Https; 61 | } 62 | 63 | wasi_http::types::Scheme::Other(req.uri().scheme_str().unwrap().to_owned()) 64 | }); 65 | let authority = req.uri().authority().map(|authority| authority.as_str()); 66 | let body = req.body().as_ref(); 67 | let body = if body.is_empty() { None } else { Some(body) }; 68 | let path_with_query = req.uri().path_and_query().map(|x| x.as_str()); 69 | 70 | let request = wasi_http::types::OutgoingRequest::new(headers); 71 | 72 | request 73 | .set_method(&req.method().into()) 74 | .map_err(|()| anyhow::anyhow!("failed to set method"))?; 75 | request 76 | .set_scheme(scheme.as_ref()) 77 | .map_err(|()| anyhow::anyhow!("failed to set scheme"))?; 78 | request 79 | .set_authority(authority) 80 | .map_err(|()| anyhow::anyhow!("failed to set authority"))?; 81 | request 82 | .set_path_with_query(path_with_query) 83 | .map_err(|()| anyhow::anyhow!("failed to set path_with_query"))?; 84 | 85 | let outgoing_body = request 86 | .body() 87 | .map_err(|_| anyhow::anyhow!("outgoing request write failed"))?; 88 | 89 | if let Some(mut buf) = body { 90 | let request_body = outgoing_body 91 | .write() 92 | .map_err(|_| anyhow::anyhow!("outgoing request write failed"))?; 93 | 94 | let pollable = request_body.subscribe(); 95 | while !buf.is_empty() { 96 | pollable.block(); 97 | 98 | let permit = match request_body.check_write() { 99 | Ok(n) => n, 100 | Err(_) => anyhow::bail!("output stream error"), 101 | }; 102 | 103 | let len = buf.len().min(permit as usize); 104 | let (chunk, rest) = buf.split_at(len); 105 | buf = rest; 106 | 107 | if request_body.write(chunk).is_err() { 108 | anyhow::bail!("output stream error"); 109 | } 110 | } 111 | 112 | if request_body.flush().is_err() { 113 | anyhow::bail!("output stream error"); 114 | } 115 | 116 | pollable.block(); 117 | 118 | match request_body.check_write() { 119 | Ok(_) => {} 120 | Err(_) => anyhow::bail!("output stream error"), 121 | }; 122 | } 123 | 124 | let future_response = wasi_http::outgoing_handler::handle(request, None)?; 125 | 126 | wasi_http::types::OutgoingBody::finish(outgoing_body, None)?; 127 | 128 | let incoming_response = match future_response.get() { 129 | Some(result) => result.map_err(|()| anyhow::anyhow!("response already taken"))?, 130 | None => { 131 | let pollable = future_response.subscribe(); 132 | pollable.block(); 133 | future_response 134 | .get() 135 | .expect("incoming response available") 136 | .map_err(|()| anyhow::anyhow!("response already taken"))? 137 | } 138 | }?; 139 | 140 | drop(future_response); 141 | 142 | let status = incoming_response.status(); 143 | 144 | let headers_handle = incoming_response.headers(); 145 | let headers = headers_handle.entries(); 146 | drop(headers_handle); 147 | 148 | let incoming_body = incoming_response 149 | .consume() 150 | .map_err(|()| anyhow::anyhow!("incoming response has no body stream"))?; 151 | 152 | drop(incoming_response); 153 | 154 | let input_stream = incoming_body.stream().unwrap(); 155 | let input_stream_pollable = input_stream.subscribe(); 156 | 157 | let mut body = Vec::new(); 158 | loop { 159 | input_stream_pollable.block(); 160 | 161 | let mut body_chunk = match input_stream.read(1024 * 1024) { 162 | Ok(c) => c, 163 | Err(crate::wit::wasi::io::streams::StreamError::Closed) => break, 164 | Err(e) => Err(anyhow::anyhow!("input_stream read failed: {e:?}"))?, 165 | }; 166 | 167 | if !body_chunk.is_empty() { 168 | body.append(&mut body_chunk); 169 | } 170 | } 171 | 172 | let mut builder = http::response::Builder::new().status(status); 173 | 174 | for (name, value) in headers { 175 | builder = builder.header(name, value); 176 | } 177 | 178 | let response = builder.body(body)?; 179 | 180 | Ok(response) 181 | } 182 | -------------------------------------------------------------------------------- /src/kiwi-sdk/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! # Kiwi SDK 2 | //! This crate provides utilities necessary for writing [Kiwi](https://github.com/rkrishn7/kiwi) plugins. 3 | //! 4 | //! # Examples 5 | //! 6 | //! ## Intercept 7 | //! ```ignore 8 | //! //! A simple intercept hook that discards odd numbers from all counter sources 9 | //! use kiwi_sdk::hook::intercept::{intercept, Action, Context, CounterEventCtx, EventCtx}; 10 | //! 11 | //! /// You must use the `#[intercept]` macro to define an intercept hook. 12 | //! #[intercept] 13 | //! fn handle(ctx: Context) -> Action { 14 | //! match ctx.event { 15 | //! // We only care about counter sources in this example 16 | //! EventCtx::Counter(CounterEventCtx { 17 | //! source_id: _, 18 | //! count, 19 | //! }) => { 20 | //! if count % 2 == 0 { 21 | //! // Returning `Action::Forward` instructs Kiwi to forward the event 22 | //! // to the associated client. 23 | //! return Action::Forward; 24 | //! } else { 25 | //! // Returning `Action::Discard` instructs Kiwi to discard the event, 26 | //! // preventing it from reaching the associated client. 27 | //! return Action::Discard; 28 | //! } 29 | //! } 30 | //! _ => {} 31 | //! } 32 | //! 33 | //! Action::Forward 34 | //! } 35 | //! ``` 36 | //! 37 | //! ## Authenticate 38 | //! ```ignore 39 | //! //! A simple authenticate hook that allows all incoming HTTP requests 40 | //! use kiwi_sdk::hook::authenticate::{authenticate, Outcome}; 41 | //! use kiwi_sdk::http::Request; 42 | //! 43 | //! /// You must use the `#[authenticate]` macro to define an authenticate hook. 44 | //! #[authenticate] 45 | //! fn handle(req: Request<()>) -> Outcome { 46 | //! // Returning `Outcome::Authenticate` instructs Kiwi to allow the connection to be established. 47 | //! Outcome::Authenticate 48 | //! } 49 | //! ``` 50 | 51 | pub mod hook; 52 | 53 | #[doc(hidden)] 54 | pub mod wit { 55 | #![allow(missing_docs)] 56 | #![allow(clippy::missing_safety_doc)] 57 | #![allow(clippy::transmute_int_to_bool)] 58 | 59 | wit_bindgen::generate!({ 60 | path: "./wit", 61 | world: "internal", 62 | }); 63 | } 64 | 65 | /// Re-export for macro use. 66 | #[doc(hidden)] 67 | pub use wit_bindgen; 68 | 69 | pub mod http; 70 | -------------------------------------------------------------------------------- /src/kiwi-sdk/wit: -------------------------------------------------------------------------------- 1 | ../wit -------------------------------------------------------------------------------- /src/kiwi/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiwi" 3 | version = "0.1.1" 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | publish = false 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | async-trait = "0.1.77" 12 | anyhow = "1.0.79" 13 | base64 = "0.21.5" 14 | clap = { version = "4.4.16", features = ["derive", "env"] } 15 | futures = "0.3.29" 16 | jwt = "0.16.0" 17 | maplit = "1.0.2" 18 | nanoid = "0.4.0" 19 | once_cell = "1.19.0" 20 | rdkafka = { version = "0.36.2", features = ["cmake-build", "tracing"] } 21 | serde = "1.0.197" 22 | serde_json = "1.0.114" 23 | serde_yaml = "0.9" 24 | thiserror = "1.0.57" 25 | tokio = { version = "1", features = ["full"] } 26 | tokio-stream = { version = "0.1.14", features = ["sync"] } 27 | tracing = "0.1.40" 28 | tracing-subscriber = "0.3.18" 29 | wasi-preview1-component-adapter-provider = "24.0.0" 30 | wasmtime = { version = "24.0.0", features = ["component-model", "async"] } 31 | wasmtime-wasi = "24.0.0" 32 | wasmtime-wasi-http = "24.0.0" 33 | wat = "1.0.85" 34 | wit-component = "0.20.1" 35 | ringbuf = "0.3.3" 36 | async-stream = "0.3.5" 37 | futures-util = "0.3.30" 38 | http = "1.0.0" 39 | notify = "6.1.1" 40 | arc-swap = "1.7.0" 41 | fastwebsockets = { version = "0.7.0", features = ["upgrade"] } 42 | http-body-util = "0.1.1" 43 | hyper = "1.2.0" 44 | hyper-util = { version = "0.1.3", features = ["server", "http1", "http2"] } 45 | tokio-rustls = "0.26.0" 46 | rustls-pemfile = "2.1.1" 47 | bytes = "1.5.0" 48 | 49 | [dev-dependencies] 50 | tempfile = "3" 51 | nix = { version = "0.28.0", features = ["signal"] } 52 | reqwest = "0.11.26" 53 | hyper = "1.2.0" 54 | hyper-util = "0.1.3" 55 | bytes = "1.5.0" 56 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/authenticate/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod types; 2 | pub mod wasm; 3 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/authenticate/types.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use http::Request as HttpRequest; 3 | 4 | #[derive(Debug, Clone)] 5 | pub enum Outcome { 6 | Authenticate, 7 | Reject, 8 | WithContext(Vec), 9 | } 10 | 11 | #[async_trait] 12 | pub trait Authenticate { 13 | async fn authenticate(&self, request: HttpRequest<()>) -> anyhow::Result; 14 | } 15 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/authenticate/wasm/bindgen.rs: -------------------------------------------------------------------------------- 1 | wasmtime::component::bindgen!({ 2 | world: "authenticate-hook", 3 | path: "../wit", 4 | async: true, 5 | tracing: true, 6 | with: { 7 | "wasi:io/error": wasmtime_wasi::bindings::io::error, 8 | "wasi:io/streams": wasmtime_wasi::bindings::io::streams, 9 | "wasi:io/poll": wasmtime_wasi::bindings::io::poll, 10 | "wasi:clocks/monotonic-clock": wasmtime_wasi::bindings::clocks::monotonic_clock, 11 | "wasi:http/types": wasmtime_wasi_http::bindings::http::types, 12 | "wasi:http/outgoing-handler": wasmtime_wasi_http::bindings::http::outgoing_handler, 13 | } 14 | }); 15 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/authenticate/wasm/bridge.rs: -------------------------------------------------------------------------------- 1 | //! Bridge between WIT types and local plugin types 2 | 3 | use super::bindgen::kiwi::kiwi::authenticate_types::Outcome; 4 | use crate::hook::authenticate::types; 5 | use http::Request as HttpRequest; 6 | 7 | impl From for types::Outcome { 8 | fn from(value: Outcome) -> Self { 9 | match value { 10 | Outcome::Authenticate => Self::Authenticate, 11 | Outcome::Reject => Self::Reject, 12 | Outcome::WithContext(payload) => Self::WithContext(payload), 13 | } 14 | } 15 | } 16 | 17 | impl From> for super::bindgen::kiwi::kiwi::authenticate_types::HttpRequest { 18 | fn from(value: HttpRequest<()>) -> Self { 19 | let (parts, _) = value.into_parts(); 20 | 21 | let scheme = parts.uri.scheme().map(|scheme| { 22 | if scheme == &http::uri::Scheme::HTTP { 23 | return super::bindgen::kiwi::kiwi::authenticate_types::Scheme::Http; 24 | } 25 | 26 | if scheme == &http::uri::Scheme::HTTPS { 27 | return super::bindgen::kiwi::kiwi::authenticate_types::Scheme::Https; 28 | } 29 | 30 | super::bindgen::kiwi::kiwi::authenticate_types::Scheme::Other( 31 | parts.uri.scheme_str().unwrap().to_owned(), 32 | ) 33 | }); 34 | 35 | Self { 36 | method: parts.method.into(), 37 | path_with_query: Some(parts.uri.to_string()), 38 | scheme, 39 | authority: parts.uri.authority().map(|a| a.as_str().into()), 40 | headers: parts 41 | .headers 42 | .iter() 43 | .map(|(k, v)| (k.as_str().into(), v.as_bytes().into())) 44 | .collect(), 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/authenticate/wasm/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod bindgen; 2 | mod bridge; 3 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/intercept/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod types; 2 | pub mod wasm; 3 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/intercept/types.rs: -------------------------------------------------------------------------------- 1 | use std::net::SocketAddr; 2 | 3 | use async_trait::async_trait; 4 | 5 | #[derive(Debug, Clone)] 6 | pub enum TransformedPayload { 7 | Kafka(Option>), 8 | Counter(u64), 9 | } 10 | 11 | #[derive(Debug, Clone)] 12 | pub enum Action { 13 | Forward, 14 | Discard, 15 | Transform(TransformedPayload), 16 | } 17 | 18 | #[derive(Debug, Clone)] 19 | /// Context needed to execute a plugin 20 | pub struct Context { 21 | pub(crate) auth: Option, 22 | pub(crate) connection: ConnectionCtx, 23 | pub(crate) event: EventCtx, 24 | } 25 | 26 | #[derive(Debug, Clone)] 27 | pub struct AuthCtx { 28 | pub(crate) raw: Vec, 29 | } 30 | 31 | impl AuthCtx { 32 | pub fn from_bytes(raw: Vec) -> Self { 33 | Self { raw } 34 | } 35 | } 36 | 37 | #[derive(Debug, Clone)] 38 | pub enum ConnectionCtx { 39 | WebSocket(WebSocketConnectionCtx), 40 | } 41 | 42 | #[derive(Debug, Clone)] 43 | pub struct WebSocketConnectionCtx { 44 | pub(crate) addr: SocketAddr, 45 | } 46 | 47 | #[derive(Debug, Clone)] 48 | pub enum EventCtx { 49 | Kafka(KafkaEventCtx), 50 | Counter(CounterEventCtx), 51 | } 52 | 53 | #[derive(Debug, Clone)] 54 | pub struct KafkaEventCtx { 55 | pub(crate) payload: Option>, 56 | pub(crate) topic: String, 57 | pub(crate) timestamp: Option, 58 | pub(crate) partition: i32, 59 | pub(crate) offset: i64, 60 | } 61 | 62 | #[derive(Debug, Clone)] 63 | pub struct CounterEventCtx { 64 | pub(crate) source_id: String, 65 | pub(crate) count: u64, 66 | } 67 | 68 | #[async_trait] 69 | pub trait Intercept { 70 | async fn intercept(&self, context: &Context) -> anyhow::Result; 71 | } 72 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/intercept/wasm/bindgen.rs: -------------------------------------------------------------------------------- 1 | wasmtime::component::bindgen!({ 2 | world: "intercept-hook", 3 | path: "../wit", 4 | async: true, 5 | tracing: true, 6 | }); 7 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/intercept/wasm/bridge.rs: -------------------------------------------------------------------------------- 1 | //! Bridge between WIT types and local plugin types 2 | use super::bindgen::kiwi::kiwi::intercept_types::*; 3 | use crate::hook::intercept::types; 4 | use crate::util::macros::try_conv_bail; 5 | 6 | impl From for Context { 7 | fn from(value: types::Context) -> Self { 8 | Self { 9 | auth: value.auth.map(|a| a.raw), 10 | connection: value.connection.into(), 11 | event: value.event.into(), 12 | } 13 | } 14 | } 15 | 16 | impl From for EventCtx { 17 | fn from(value: types::EventCtx) -> Self { 18 | match value { 19 | types::EventCtx::Kafka(ctx) => Self::Kafka(ctx.into()), 20 | types::EventCtx::Counter(ctx) => Self::Counter(ctx.into()), 21 | } 22 | } 23 | } 24 | 25 | impl From for CounterEventCtx { 26 | fn from(value: types::CounterEventCtx) -> Self { 27 | Self { 28 | source_id: value.source_id, 29 | count: value.count, 30 | } 31 | } 32 | } 33 | 34 | impl From for KafkaEventCtx { 35 | fn from(value: types::KafkaEventCtx) -> Self { 36 | let timestamp: Option = value 37 | .timestamp 38 | .map(|ts| try_conv_bail!(ts, "timestamp conversion must not fail")); 39 | let partition = try_conv_bail!(value.partition, "partition conversion must not fail"); 40 | let offset = try_conv_bail!(value.offset, "offset conversion must not fail"); 41 | Self { 42 | payload: value.payload, 43 | topic: value.topic.clone(), 44 | // TODO: When Kafka sources include a custom source ID, use it here 45 | source_id: value.topic, 46 | timestamp, 47 | partition, 48 | offset, 49 | } 50 | } 51 | } 52 | 53 | impl From for ConnectionCtx { 54 | fn from(value: types::ConnectionCtx) -> Self { 55 | match value { 56 | types::ConnectionCtx::WebSocket(ctx) => Self::Websocket(ctx.into()), 57 | } 58 | } 59 | } 60 | 61 | impl From for Websocket { 62 | fn from(value: types::WebSocketConnectionCtx) -> Self { 63 | Self { 64 | addr: Some(value.addr.to_string()), 65 | } 66 | } 67 | } 68 | 69 | impl From for types::Action { 70 | fn from(value: Action) -> Self { 71 | match value { 72 | Action::Forward => Self::Forward, 73 | Action::Discard => Self::Discard, 74 | Action::Transform(transformed) => Self::Transform(transformed.into()), 75 | } 76 | } 77 | } 78 | 79 | impl From for types::TransformedPayload { 80 | fn from(value: TransformedPayload) -> Self { 81 | match value { 82 | TransformedPayload::Kafka(payload) => Self::Kafka(payload), 83 | TransformedPayload::Counter(count) => Self::Counter(count), 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/intercept/wasm/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod bindgen; 2 | pub mod bridge; 3 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod authenticate; 2 | pub mod intercept; 3 | pub mod wasm; 4 | -------------------------------------------------------------------------------- /src/kiwi/src/hook/wasm/mod.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use async_trait::async_trait; 4 | use http::Request as HttpRequest; 5 | use once_cell::sync::Lazy; 6 | use wasi_preview1_component_adapter_provider::WASI_SNAPSHOT_PREVIEW1_REACTOR_ADAPTER; 7 | use wasmtime::component::{Component, InstancePre, Linker, ResourceTable}; 8 | use wasmtime::{Config, Engine, Store}; 9 | use wasmtime_wasi::{Stdout, WasiCtx, WasiCtxBuilder, WasiImpl, WasiView}; 10 | use wasmtime_wasi_http::{WasiHttpCtx, WasiHttpView}; 11 | 12 | use anyhow::Context; 13 | use wit_component::ComponentEncoder; 14 | 15 | use super::authenticate; 16 | use super::authenticate::types::{Authenticate, Outcome}; 17 | use super::authenticate::wasm::bindgen::AuthenticateHookPre; 18 | use super::intercept; 19 | use super::intercept::types::Intercept; 20 | use super::intercept::wasm::bindgen::InterceptHookPre; 21 | 22 | static ENGINE: Lazy = Lazy::new(|| { 23 | let mut config = Config::new(); 24 | config.wasm_component_model(true); 25 | config.async_support(true); 26 | Engine::new(&config).expect("failed to instantiate engine") 27 | }); 28 | 29 | /// Encode a WebAssembly module into a component suitable for execution in the 30 | /// Kiwi hook runtime. 31 | pub fn encode_component>(input: P) -> anyhow::Result> { 32 | let parsed = wat::parse_file(input).context("failed to parse wat")?; 33 | 34 | let mut encoder = ComponentEncoder::default().validate(true).module(&parsed)?; 35 | encoder = encoder.adapter( 36 | "wasi_snapshot_preview1", 37 | WASI_SNAPSHOT_PREVIEW1_REACTOR_ADAPTER, 38 | )?; 39 | 40 | encoder.encode().context("failed to encode component") 41 | } 42 | 43 | pub struct Host { 44 | table: ResourceTable, 45 | wasi: WasiCtx, 46 | http: WasiHttpCtx, 47 | } 48 | 49 | impl WasiHttpView for Host { 50 | fn ctx(&mut self) -> &mut WasiHttpCtx { 51 | &mut self.http 52 | } 53 | 54 | fn table(&mut self) -> &mut ResourceTable { 55 | &mut self.table 56 | } 57 | } 58 | 59 | impl WasiView for Host { 60 | fn table(&mut self) -> &mut ResourceTable { 61 | &mut self.table 62 | } 63 | 64 | fn ctx(&mut self) -> &mut WasiCtx { 65 | &mut self.wasi 66 | } 67 | } 68 | 69 | impl authenticate::wasm::bindgen::kiwi::kiwi::authenticate_types::Host for WasiImpl {} 70 | impl intercept::wasm::bindgen::kiwi::kiwi::intercept_types::Host for Host {} 71 | 72 | pub(super) fn get_linker() -> anyhow::Result> { 73 | let mut linker = Linker::new(&ENGINE); 74 | wasmtime_wasi::add_to_linker_async(&mut linker)?; 75 | wasmtime_wasi_http::add_only_http_to_linker_async(&mut linker)?; 76 | 77 | Ok(linker) 78 | } 79 | 80 | pub(super) fn create_instance_pre>(file: P) -> anyhow::Result> { 81 | let linker = get_linker()?; 82 | let bytes = encode_component(file)?; 83 | let component = Component::from_binary(&ENGINE, &bytes)?; 84 | 85 | let instance_pre = linker.instantiate_pre(&component)?; 86 | 87 | Ok(instance_pre) 88 | } 89 | 90 | pub trait WasmHook { 91 | /// Create a new instance of the hook from a file 92 | fn from_file>(file: P) -> anyhow::Result 93 | where 94 | Self: Sized; 95 | /// Path to the WebAssembly module 96 | fn path(&self) -> &std::path::Path; 97 | } 98 | 99 | pub struct WasmAuthenticateHook { 100 | instance_pre: AuthenticateHookPre, 101 | path: std::path::PathBuf, 102 | } 103 | 104 | impl WasmHook for WasmAuthenticateHook { 105 | fn from_file>(file: P) -> anyhow::Result { 106 | let path = file.as_ref().to_path_buf(); 107 | let instance_pre = create_instance_pre(file)?; 108 | let instance_pre = AuthenticateHookPre::new(instance_pre)?; 109 | 110 | Ok(Self { instance_pre, path }) 111 | } 112 | 113 | fn path(&self) -> &std::path::Path { 114 | &self.path 115 | } 116 | } 117 | 118 | pub struct WasmInterceptHook { 119 | instance_pre: InterceptHookPre, 120 | path: std::path::PathBuf, 121 | } 122 | 123 | impl WasmHook for WasmInterceptHook { 124 | fn from_file>(file: P) -> anyhow::Result { 125 | let path = file.as_ref().to_path_buf(); 126 | let instance_pre = create_instance_pre(file)?; 127 | let instance_pre = InterceptHookPre::new(instance_pre)?; 128 | 129 | Ok(Self { instance_pre, path }) 130 | } 131 | 132 | fn path(&self) -> &std::path::Path { 133 | &self.path 134 | } 135 | } 136 | 137 | #[async_trait] 138 | impl Authenticate for WasmAuthenticateHook { 139 | async fn authenticate(&self, request: HttpRequest<()>) -> anyhow::Result { 140 | let mut builder = WasiCtxBuilder::new(); 141 | 142 | builder.stdout(Stdout); 143 | 144 | let state = Host { 145 | table: ResourceTable::new(), 146 | wasi: builder.build(), 147 | http: WasiHttpCtx::new(), 148 | }; 149 | 150 | let mut store = Store::new(&ENGINE, state); 151 | 152 | let bindings = self.instance_pre.instantiate_async(&mut store).await?; 153 | 154 | let res = bindings 155 | .call_authenticate(&mut store, &request.into()) 156 | .await?; 157 | 158 | Ok(res.into()) 159 | } 160 | } 161 | 162 | #[async_trait] 163 | impl Intercept for WasmInterceptHook { 164 | async fn intercept( 165 | &self, 166 | ctx: &super::intercept::types::Context, 167 | ) -> anyhow::Result { 168 | let mut builder = WasiCtxBuilder::new(); 169 | 170 | let mut store = Store::new( 171 | &ENGINE, 172 | Host { 173 | table: ResourceTable::new(), 174 | wasi: builder.build(), 175 | http: WasiHttpCtx::new(), 176 | }, 177 | ); 178 | 179 | let bindings = self.instance_pre.instantiate_async(&mut store).await?; 180 | 181 | let res = bindings 182 | .call_intercept(&mut store, &ctx.clone().into()) 183 | .await?; 184 | 185 | Ok(res.into()) 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/kiwi/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod config; 2 | pub mod connection; 3 | pub mod hook; 4 | pub mod protocol; 5 | pub mod source; 6 | pub mod subscription; 7 | pub mod tls; 8 | pub mod util; 9 | pub mod ws; 10 | -------------------------------------------------------------------------------- /src/kiwi/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::net::SocketAddr; 3 | use std::sync::Arc; 4 | use std::sync::Mutex; 5 | 6 | use arc_swap::ArcSwapOption; 7 | use clap::Parser; 8 | 9 | use kiwi::config::Config; 10 | use kiwi::config::ConfigReconciler; 11 | use kiwi::source::kafka::start_partition_discovery; 12 | use kiwi::source::Source; 13 | use kiwi::source::SourceId; 14 | 15 | /// kiwi is a bridge between your backend services and front-end applications. 16 | /// It seamlessly and efficiently manages the flow of real-time Kafka events 17 | /// through WebSockets, ensuring that your applications stay reactive and up-to-date 18 | /// with the latest data. 19 | #[derive(Parser, Debug)] 20 | #[command(author, version, about, long_about = None)] 21 | struct Args { 22 | /// Path to the configuration file 23 | #[arg(short, long, env, default_value_t = String::from("/etc/kiwi/config/kiwi.yml"))] 24 | pub config: String, 25 | 26 | /// Log level 27 | #[arg(short, long, default_value_t = tracing::Level::INFO, env)] 28 | pub log_level: tracing::Level, 29 | } 30 | 31 | #[tokio::main] 32 | async fn main() -> anyhow::Result<()> { 33 | let args = Args::parse(); 34 | 35 | tracing_subscriber::fmt() 36 | .with_max_level(args.log_level) 37 | .init(); 38 | 39 | let config_path = args.config.clone(); 40 | 41 | let config = Config::parse(&config_path)?; 42 | 43 | let sources: Arc>>> = 44 | Arc::new(Mutex::new(BTreeMap::new())); 45 | 46 | let intercept = Arc::new(ArcSwapOption::new(None)); 47 | let authenticate = Arc::new(ArcSwapOption::new(None)); 48 | 49 | let config_reconciler: ConfigReconciler = ConfigReconciler::new( 50 | Arc::clone(&sources), 51 | Arc::clone(&intercept), 52 | Arc::clone(&authenticate), 53 | ); 54 | 55 | config_reconciler.reconcile_sources(&config)?; 56 | config_reconciler.reconcile_hooks(&config)?; 57 | 58 | if let Some(kafka_config) = config.kafka.as_ref() { 59 | if kafka_config.partition_discovery_enabled { 60 | start_partition_discovery( 61 | &kafka_config.bootstrap_servers, 62 | Arc::clone(&sources), 63 | std::time::Duration::from_millis( 64 | kafka_config.partition_discovery_interval_ms.into(), 65 | ), 66 | )?; 67 | } 68 | } 69 | 70 | tokio::spawn(async move { 71 | if let Err(e) = config_reconciler.watch(config_path.into()).await { 72 | tracing::error!( 73 | "Configuration watcher exited unexpectedly with the error: {}", 74 | e 75 | ); 76 | } 77 | }); 78 | 79 | let listen_addr: SocketAddr = config.server.address.parse()?; 80 | 81 | #[cfg(windows)] 82 | let mut term = tokio::signal::windows::ctrl_close().unwrap(); 83 | #[cfg(unix)] 84 | let mut term = 85 | tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap(); 86 | 87 | tokio::select! { 88 | _ = term.recv() => { 89 | tracing::info!("Received SIGTERM, shutting down"); 90 | Ok(()) 91 | } 92 | _ = tokio::signal::ctrl_c() => { 93 | tracing::info!("Received SIGINT, shutting down"); 94 | Ok(()) 95 | } 96 | res = kiwi::ws::serve( 97 | &listen_addr, 98 | sources, 99 | intercept, 100 | authenticate, 101 | config.subscriber, 102 | config.server.tls, 103 | config.server.healthcheck, 104 | ) => { 105 | res 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/kiwi/src/protocol.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use thiserror::Error; 3 | 4 | use crate::source::{self, SourceId}; 5 | 6 | /// The subscription mode to use for a source subscription 7 | #[derive(Debug, Default, Clone, Deserialize, Serialize, PartialEq, Eq)] 8 | #[serde(rename_all = "camelCase")] 9 | pub enum SubscriptionMode { 10 | /// Pull subscriptions require the client to request events from the source 11 | Pull, 12 | /// Push subscriptions send events to the client as they are produced 13 | #[default] 14 | Push, 15 | } 16 | 17 | /// Commands are issued by kiwi clients to the server 18 | #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] 19 | #[serde(tag = "type")] 20 | #[serde(rename_all = "SCREAMING_SNAKE_CASE")] 21 | pub enum Command { 22 | /// Subscribe to the specified source 23 | #[serde(rename_all = "camelCase")] 24 | Subscribe { 25 | /// The ID for the source to subscribe to 26 | source_id: SourceId, 27 | /// The subscription mode to use 28 | #[serde(default)] 29 | mode: SubscriptionMode, 30 | }, 31 | /// Unsubscribe from the specified source 32 | #[serde(rename_all = "camelCase")] 33 | Unsubscribe { 34 | /// The ID for the source to unsubscribe from. The source must be 35 | /// associated with an active subscription for the request to be valid 36 | source_id: SourceId, 37 | }, 38 | /// Request the next `n` events from the source. This is only valid for 39 | /// pull-based subscriptions 40 | #[serde(rename_all = "camelCase")] 41 | Request { 42 | /// The ID of the source to request data from 43 | source_id: SourceId, 44 | /// The (additive) number of events to request 45 | n: u64, 46 | }, 47 | } 48 | 49 | /// Command responses are issued by the server to clients in response to 50 | /// commands 51 | #[derive(Debug, Clone, Deserialize, Serialize)] 52 | #[serde(tag = "type")] 53 | #[serde(rename_all = "SCREAMING_SNAKE_CASE")] 54 | pub enum CommandResponse { 55 | /// The subscription was successful 56 | #[serde(rename_all = "camelCase")] 57 | SubscribeOk { source_id: SourceId }, 58 | /// The unsubscription was successful 59 | #[serde(rename_all = "camelCase")] 60 | UnsubscribeOk { source_id: SourceId }, 61 | /// An error occurred while attempting to subscribe 62 | #[serde(rename_all = "camelCase")] 63 | SubscribeError { source_id: SourceId, error: String }, 64 | /// An error occurred while attempting to unsubscribe 65 | #[serde(rename_all = "camelCase")] 66 | UnsubscribeError { source_id: SourceId, error: String }, 67 | /// The request operation was successful. The returned value 68 | /// for `requests` is the total number of requests remaining. 69 | /// This may be more than the number specified in the previous 70 | /// request operation as each request operation is additive. 71 | #[serde(rename_all = "camelCase")] 72 | RequestOk { source_id: SourceId, requests: u64 }, 73 | /// An error occurred while attempting to request events 74 | #[serde(rename_all = "camelCase")] 75 | RequestError { source_id: SourceId, error: String }, 76 | } 77 | 78 | #[derive(Debug, Clone, Deserialize, Serialize)] 79 | #[serde(tag = "type")] 80 | #[serde(rename_all = "SCREAMING_SNAKE_CASE")] 81 | /// An info or error message that may be pushed to a client. A notice, in many 82 | /// cases is not issued as a direct result of a command 83 | pub enum Notice { 84 | /// Indicates that the source has lagged behind by `count` events 85 | #[serde(rename_all = "camelCase")] 86 | Lag { source_id: SourceId, count: u64 }, 87 | /// Indicates that the subscription to the source has been closed. 88 | /// This may be due to a few reasons, such as the source being removed, 89 | /// the source closing, source metadata changing, or an error occurring. 90 | #[serde(rename_all = "camelCase")] 91 | SubscriptionClosed { 92 | source_id: SourceId, 93 | message: Option, 94 | }, 95 | } 96 | 97 | #[derive(Debug, Clone, Deserialize, Serialize)] 98 | #[serde(tag = "type", content = "data")] 99 | #[serde(rename_all = "SCREAMING_SNAKE_CASE")] 100 | /// An outbound message that is sent from the server to a client 101 | pub enum Message { 102 | CommandResponse(CommandResponse), 103 | Notice(Notice), 104 | Result(SourceResult), 105 | } 106 | 107 | impl From for Message { 108 | fn from(value: source::SourceResult) -> Self { 109 | Message::Result(value.into()) 110 | } 111 | } 112 | 113 | #[derive(Debug, Clone, Deserialize, Serialize)] 114 | #[serde(tag = "sourceType", rename_all = "camelCase")] 115 | pub enum SourceResult { 116 | #[serde(rename_all = "camelCase")] 117 | Kafka { 118 | #[serde(with = "crate::util::serde::base64")] 119 | /// Event key 120 | key: Option>, 121 | #[serde(with = "crate::util::serde::base64")] 122 | /// base64 encoded event payload 123 | payload: Option>, 124 | /// Source ID this event was produced from 125 | source_id: SourceId, 126 | /// Timestamp at which the message was produced 127 | timestamp: Option, 128 | /// Partition ID this event was produced from 129 | partition: i32, 130 | /// Offset at which the message was produced 131 | offset: i64, 132 | }, 133 | #[serde(rename_all = "camelCase")] 134 | Counter { 135 | /// Source ID this counter event was produced from 136 | source_id: SourceId, 137 | /// Event count 138 | count: u64, 139 | }, 140 | } 141 | 142 | impl From for SourceResult { 143 | fn from(value: source::SourceResult) -> Self { 144 | match value { 145 | source::SourceResult::Kafka(kafka) => Self::Kafka { 146 | key: kafka.key, 147 | payload: kafka.payload, 148 | source_id: kafka.id, 149 | partition: kafka.partition, 150 | offset: kafka.offset, 151 | timestamp: kafka.timestamp, 152 | }, 153 | source::SourceResult::Counter(counter) => Self::Counter { 154 | source_id: counter.source_id, 155 | count: counter.count, 156 | }, 157 | } 158 | } 159 | } 160 | 161 | #[derive(Debug, Error)] 162 | pub enum ProtocolError { 163 | #[error("Unsupported command form. Only UTF-8 encoded text is supported")] 164 | UnsupportedCommandForm, 165 | #[error("Encountered an error while deserializing the command payload {0}")] 166 | CommandDeserialization(String), 167 | } 168 | 169 | #[cfg(test)] 170 | mod tests { 171 | use super::*; 172 | use base64::Engine; 173 | 174 | #[test] 175 | fn test_command_de() { 176 | let command = r#"{"type":"SUBSCRIBE","sourceId":"test"}"#; 177 | let deserialized: Command = serde_json::from_str(command).unwrap(); 178 | assert_eq!( 179 | deserialized, 180 | Command::Subscribe { 181 | source_id: "test".into(), 182 | mode: SubscriptionMode::Push 183 | } 184 | ); 185 | 186 | let command = r#"{"type":"UNSUBSCRIBE","sourceId":"test"}"#; 187 | let deserialized: Command = serde_json::from_str(command).unwrap(); 188 | assert_eq!( 189 | deserialized, 190 | Command::Unsubscribe { 191 | source_id: "test".into() 192 | } 193 | ); 194 | } 195 | 196 | #[test] 197 | fn test_message_ser() { 198 | let message: Message = Message::CommandResponse(CommandResponse::SubscribeOk { 199 | source_id: "test".into(), 200 | }); 201 | 202 | let serialized = serde_json::to_string(&message).unwrap(); 203 | assert_eq!( 204 | serialized, 205 | r#"{"type":"COMMAND_RESPONSE","data":{"type":"SUBSCRIBE_OK","sourceId":"test"}}"# 206 | ); 207 | 208 | let message: Message = Message::CommandResponse(CommandResponse::UnsubscribeOk { 209 | source_id: "test".into(), 210 | }); 211 | 212 | let serialized = serde_json::to_string(&message).unwrap(); 213 | assert_eq!( 214 | serialized, 215 | r#"{"type":"COMMAND_RESPONSE","data":{"type":"UNSUBSCRIBE_OK","sourceId":"test"}}"# 216 | ); 217 | 218 | let message: Message = Message::CommandResponse(CommandResponse::SubscribeError { 219 | source_id: "test".into(), 220 | error: "test".into(), 221 | }); 222 | 223 | let serialized = serde_json::to_string(&message).unwrap(); 224 | assert_eq!( 225 | serialized, 226 | r#"{"type":"COMMAND_RESPONSE","data":{"type":"SUBSCRIBE_ERROR","sourceId":"test","error":"test"}}"# 227 | ); 228 | 229 | let message: Message = Message::CommandResponse(CommandResponse::UnsubscribeError { 230 | source_id: "test".into(), 231 | error: "test".into(), 232 | }); 233 | 234 | let serialized = serde_json::to_string(&message).unwrap(); 235 | assert_eq!( 236 | serialized, 237 | r#"{"type":"COMMAND_RESPONSE","data":{"type":"UNSUBSCRIBE_ERROR","sourceId":"test","error":"test"}}"# 238 | ); 239 | 240 | let message: Message = Message::Notice(Notice::Lag { 241 | source_id: "test".into(), 242 | count: 1, 243 | }); 244 | 245 | let serialized = serde_json::to_string(&message).unwrap(); 246 | assert_eq!( 247 | serialized, 248 | r#"{"type":"NOTICE","data":{"type":"LAG","sourceId":"test","count":1}}"# 249 | ); 250 | 251 | let message: Message = Message::Notice(Notice::SubscriptionClosed { 252 | source_id: "test".into(), 253 | message: Some("New partition added".to_string()), 254 | }); 255 | 256 | let serialized = serde_json::to_string(&message).unwrap(); 257 | assert_eq!( 258 | serialized, 259 | r#"{"type":"NOTICE","data":{"type":"SUBSCRIPTION_CLOSED","sourceId":"test","message":"New partition added"}}"# 260 | ); 261 | 262 | let message = Message::Result(SourceResult::Kafka { 263 | payload: Some("test".into()), 264 | source_id: "test".into(), 265 | key: None, 266 | timestamp: None, 267 | partition: 0, 268 | offset: 1, 269 | }); 270 | 271 | let serialized = serde_json::to_string(&message).unwrap(); 272 | let encoded = base64::engine::general_purpose::STANDARD.encode("test".as_bytes()); 273 | assert_eq!( 274 | serialized, 275 | r#"{"type":"RESULT","data":{"sourceType":"kafka","key":null,"payload":"$encoded","sourceId":"test","timestamp":null,"partition":0,"offset":1}}"#.replace("$encoded", encoded.as_str()) 276 | ); 277 | 278 | let message = Message::Result(SourceResult::Counter { 279 | source_id: "test".into(), 280 | count: 1, 281 | }); 282 | 283 | let serialized = serde_json::to_string(&message).unwrap(); 284 | assert_eq!( 285 | serialized, 286 | r#"{"type":"RESULT","data":{"sourceType":"counter","sourceId":"test","count":1}}"# 287 | ); 288 | } 289 | } 290 | -------------------------------------------------------------------------------- /src/kiwi/src/source/counter.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Weak}; 2 | 3 | use futures_util::{future::Fuse, FutureExt}; 4 | use tokio::sync::broadcast::{Receiver, Sender}; 5 | 6 | use crate::hook; 7 | 8 | use super::{Source, SourceId, SourceMessage, SourceMetadata, SourceResult, SubscribeError}; 9 | 10 | type ShutdownTrigger = tokio::sync::oneshot::Sender<()>; 11 | type ShutdownReceiver = tokio::sync::oneshot::Receiver<()>; 12 | 13 | #[derive(Debug, Clone, PartialEq, Eq)] 14 | pub struct CounterSourceResult { 15 | pub source_id: String, 16 | pub count: u64, 17 | } 18 | 19 | impl From for hook::intercept::types::CounterEventCtx { 20 | fn from(value: CounterSourceResult) -> Self { 21 | Self { 22 | count: value.count, 23 | source_id: value.source_id, 24 | } 25 | } 26 | } 27 | 28 | pub struct CounterSource { 29 | id: String, 30 | tx: Weak>, 31 | initial_subscription_tx: Option>, 32 | _shutdown_trigger: ShutdownTrigger, 33 | } 34 | 35 | impl CounterSource { 36 | pub fn new( 37 | id: String, 38 | min: u64, 39 | max: Option, 40 | interval: std::time::Duration, 41 | lazy: bool, 42 | ) -> Self { 43 | let (tx, _) = tokio::sync::broadcast::channel(1_000); 44 | let (shutdown_trigger, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); 45 | let (initial_subscription_tx, initial_subscription_rx) = 46 | tokio::sync::oneshot::channel::<()>(); 47 | 48 | let tx = Arc::new(tx); 49 | 50 | // The counter task should be the only thing holding a strong reference 51 | // to the sender. Handing the source a weak reference to the sender 52 | // allows downstream subscriptions to properly detect when the source 53 | // has ended. This is important for finite sources. 54 | let weak_tx = Arc::downgrade(&tx); 55 | 56 | let task = CounterTask { 57 | source_id: id.clone(), 58 | min, 59 | max, 60 | interval, 61 | tx, 62 | lazy, 63 | initial_subscription_rx, 64 | shutdown_rx: shutdown_rx.fuse(), 65 | }; 66 | 67 | tokio::spawn(task.run()); 68 | 69 | Self { 70 | id, 71 | _shutdown_trigger: shutdown_trigger, 72 | tx: weak_tx, 73 | initial_subscription_tx: Some(initial_subscription_tx), 74 | } 75 | } 76 | } 77 | 78 | impl Source for CounterSource { 79 | fn subscribe(&mut self) -> Result, SubscribeError> { 80 | if let Some(tx) = self.initial_subscription_tx.take() { 81 | let _ = tx.send(()); 82 | } 83 | 84 | // If the counter task (our sole sender) has ended, it indicates that 85 | // the source has ended 86 | if let Some(tx) = self.tx.upgrade() { 87 | Ok(tx.subscribe()) 88 | } else { 89 | Err(SubscribeError::FiniteSourceEnded) 90 | } 91 | } 92 | 93 | fn source_id(&self) -> &SourceId { 94 | &self.id 95 | } 96 | 97 | fn metadata_tx(&self) -> &Option> { 98 | &None 99 | } 100 | 101 | fn as_any(&self) -> &dyn std::any::Any { 102 | self 103 | } 104 | } 105 | 106 | pub struct CounterTask { 107 | source_id: String, 108 | min: u64, 109 | max: Option, 110 | interval: std::time::Duration, 111 | tx: Arc>, 112 | lazy: bool, 113 | initial_subscription_rx: tokio::sync::oneshot::Receiver<()>, 114 | shutdown_rx: Fuse, 115 | } 116 | 117 | impl CounterTask { 118 | pub async fn run(mut self) { 119 | let mut current = self.min; 120 | 121 | if self.lazy { 122 | let _ = self.initial_subscription_rx.await; 123 | } 124 | 125 | let mut interval = tokio::time::interval(self.interval); 126 | 127 | loop { 128 | tokio::select! { 129 | _ = &mut self.shutdown_rx => break, 130 | _ = interval.tick() => { 131 | if let Some(max) = self.max { 132 | if current > max { 133 | break; 134 | } 135 | } 136 | 137 | let _ = self.tx.send(SourceMessage::Result(SourceResult::Counter(CounterSourceResult { 138 | source_id: self.source_id.clone(), 139 | count: current, 140 | }))); 141 | 142 | current += 1; 143 | } 144 | } 145 | } 146 | 147 | tracing::debug!("Counter task for source {} shutting down", self.source_id); 148 | } 149 | } 150 | 151 | pub trait CounterSourceBuilder { 152 | fn build_source( 153 | id: String, 154 | min: u64, 155 | max: Option, 156 | interval: std::time::Duration, 157 | lazy: bool, 158 | ) -> Box { 159 | Box::new(CounterSource::new(id, min, max, interval, lazy)) 160 | } 161 | } 162 | 163 | #[cfg(test)] 164 | mod tests { 165 | use super::*; 166 | 167 | #[tokio::test] 168 | async fn test_subscribing_fails_after_source_ended() { 169 | let mut source = CounterSource::new( 170 | "test".into(), 171 | 0, 172 | Some(3), 173 | std::time::Duration::from_millis(5), 174 | false, 175 | ); 176 | 177 | tokio::time::sleep(std::time::Duration::from_millis(25)).await; 178 | 179 | assert!(matches!( 180 | source.subscribe(), 181 | Err(SubscribeError::FiniteSourceEnded) 182 | )); 183 | } 184 | 185 | #[tokio::test] 186 | async fn test_lazy_starts_after_first_subscription() { 187 | let mut source = CounterSource::new( 188 | "test".into(), 189 | 0, 190 | Some(3), 191 | std::time::Duration::from_millis(1), 192 | true, 193 | ); 194 | 195 | let mut rx = source.subscribe().unwrap(); 196 | 197 | let msg = rx.recv().await.unwrap(); 198 | 199 | assert!(matches!( 200 | msg, 201 | SourceMessage::Result(SourceResult::Counter(CounterSourceResult { 202 | source_id, 203 | count 204 | })) if source_id == "test" && count == 0, 205 | )); 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /src/kiwi/src/source/mod.rs: -------------------------------------------------------------------------------- 1 | use tokio::sync::broadcast::Receiver; 2 | 3 | use crate::hook; 4 | 5 | use self::{counter::CounterSourceBuilder, kafka::KafkaSourceBuilder}; 6 | 7 | pub mod counter; 8 | pub mod kafka; 9 | 10 | #[derive(Clone, Debug, PartialEq, Eq)] 11 | pub enum SourceMessage { 12 | /// A source-specific event 13 | Result(SourceResult), 14 | /// Source metadata has changed 15 | MetadataChanged(String), 16 | } 17 | 18 | #[derive(Debug, Clone, PartialEq, Eq)] 19 | pub enum SourceResult { 20 | Kafka(kafka::KafkaSourceResult), 21 | Counter(counter::CounterSourceResult), 22 | } 23 | 24 | pub enum SourceMetadata { 25 | Kafka(kafka::KafkaSourceMetadata), 26 | } 27 | 28 | #[derive(Debug, thiserror::Error)] 29 | pub enum SubscribeError { 30 | #[error("Finite source has ended")] 31 | FiniteSourceEnded, 32 | } 33 | 34 | pub trait Source { 35 | fn subscribe(&mut self) -> Result, SubscribeError>; 36 | 37 | fn source_id(&self) -> &SourceId; 38 | 39 | fn metadata_tx(&self) -> &Option>; 40 | 41 | fn as_any(&self) -> &dyn std::any::Any; 42 | } 43 | 44 | pub type SourceId = String; 45 | 46 | impl From for hook::intercept::types::EventCtx { 47 | fn from(value: SourceResult) -> Self { 48 | match value { 49 | SourceResult::Kafka(kafka_result) => Self::Kafka(kafka_result.into()), 50 | SourceResult::Counter(counter_result) => Self::Counter(counter_result.into()), 51 | } 52 | } 53 | } 54 | 55 | pub struct SourceBuilder; 56 | 57 | impl KafkaSourceBuilder for SourceBuilder {} 58 | impl CounterSourceBuilder for SourceBuilder {} 59 | -------------------------------------------------------------------------------- /src/kiwi/src/tls.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::BufReader; 3 | use std::path::Path; 4 | use std::sync::Arc; 5 | use std::{ 6 | pin::Pin, 7 | task::{Context, Poll}, 8 | }; 9 | use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; 10 | use tokio_rustls::rustls::pki_types::{CertificateDer, PrivateKeyDer}; 11 | use tokio_rustls::server::TlsStream; 12 | use tokio_rustls::{rustls, TlsAcceptor}; 13 | 14 | fn load_certs(path: impl AsRef) -> std::io::Result>> { 15 | rustls_pemfile::certs(&mut BufReader::new(File::open(path)?)).collect() 16 | } 17 | 18 | fn load_key(path: impl AsRef) -> anyhow::Result>> { 19 | Ok(rustls_pemfile::private_key(&mut BufReader::new( 20 | File::open(path)?, 21 | ))?) 22 | } 23 | 24 | pub fn tls_acceptor(cert: impl AsRef, key: impl AsRef) -> anyhow::Result { 25 | let key = load_key(key)?.expect("no key found"); 26 | let certs = load_certs(cert)?; 27 | 28 | let config = rustls::ServerConfig::builder() 29 | .with_no_client_auth() 30 | .with_single_cert(certs, key)?; 31 | 32 | Ok(TlsAcceptor::from(Arc::new(config))) 33 | } 34 | 35 | pub enum MaybeTlsStream { 36 | Plain(S), 37 | Tls(Box>), 38 | } 39 | 40 | impl AsyncRead for MaybeTlsStream { 41 | fn poll_read( 42 | self: Pin<&mut Self>, 43 | cx: &mut Context<'_>, 44 | buf: &mut ReadBuf<'_>, 45 | ) -> Poll> { 46 | match self.get_mut() { 47 | MaybeTlsStream::Plain(ref mut s) => Pin::new(s).poll_read(cx, buf), 48 | MaybeTlsStream::Tls(s) => Pin::new(s).poll_read(cx, buf), 49 | } 50 | } 51 | } 52 | 53 | impl AsyncWrite for MaybeTlsStream { 54 | fn poll_write( 55 | self: Pin<&mut Self>, 56 | cx: &mut Context<'_>, 57 | buf: &[u8], 58 | ) -> Poll> { 59 | match self.get_mut() { 60 | MaybeTlsStream::Plain(ref mut s) => Pin::new(s).poll_write(cx, buf), 61 | MaybeTlsStream::Tls(s) => Pin::new(s).poll_write(cx, buf), 62 | } 63 | } 64 | 65 | fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 66 | match self.get_mut() { 67 | MaybeTlsStream::Plain(ref mut s) => Pin::new(s).poll_flush(cx), 68 | MaybeTlsStream::Tls(s) => Pin::new(s).poll_flush(cx), 69 | } 70 | } 71 | 72 | fn poll_shutdown( 73 | self: Pin<&mut Self>, 74 | cx: &mut Context<'_>, 75 | ) -> Poll> { 76 | match self.get_mut() { 77 | MaybeTlsStream::Plain(ref mut s) => Pin::new(s).poll_shutdown(cx), 78 | MaybeTlsStream::Tls(s) => Pin::new(s).poll_shutdown(cx), 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/kiwi/src/util/macros.rs: -------------------------------------------------------------------------------- 1 | macro_rules! try_conv_bail { 2 | ($name:expr, $s:expr) => { 3 | $name.try_into().expect($s) 4 | }; 5 | } 6 | 7 | pub(crate) use try_conv_bail; 8 | -------------------------------------------------------------------------------- /src/kiwi/src/util/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod macros; 2 | pub mod serde; 3 | pub mod stream; 4 | -------------------------------------------------------------------------------- /src/kiwi/src/util/serde.rs: -------------------------------------------------------------------------------- 1 | pub mod base64 { 2 | use base64::Engine; 3 | use serde::{Deserialize, Serialize}; 4 | use serde::{Deserializer, Serializer}; 5 | 6 | pub fn serialize(v: &Option>, s: S) -> Result { 7 | let base64 = v 8 | .as_ref() 9 | .map(|v| base64::engine::general_purpose::STANDARD.encode(v)); 10 | 11 | >::serialize(&base64, s) 12 | } 13 | 14 | pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result>, D::Error> { 15 | let base64 = >::deserialize(d)?; 16 | match base64 { 17 | Some(v) => base64::engine::general_purpose::STANDARD 18 | .decode(v.as_bytes()) 19 | .map(Some) 20 | .map_err(serde::de::Error::custom), 21 | None => Ok(None), 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/kiwi/src/util/stream.rs: -------------------------------------------------------------------------------- 1 | use futures::stream::{Stream, StreamExt}; 2 | 3 | /// Returns a transformed stream that yields items along with a provided stream identifier 4 | pub fn with_id(id: T, stream: S) -> impl Stream { 5 | stream.map(move |item| (id.clone(), item)) 6 | } 7 | -------------------------------------------------------------------------------- /src/kiwi/src/ws.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::sync::Mutex; 3 | use std::{net::SocketAddr, sync::Arc}; 4 | 5 | use anyhow::Context; 6 | use arc_swap::ArcSwapOption; 7 | use bytes::Bytes; 8 | use fastwebsockets::{upgrade, CloseCode, FragmentCollector, Frame, Payload, WebSocketError}; 9 | use http::{Request, Response, StatusCode}; 10 | use http_body_util::Empty; 11 | use hyper::service::service_fn; 12 | use tokio::io::{AsyncReadExt, AsyncWriteExt}; 13 | 14 | use crate::connection::ConnectionManager; 15 | use crate::hook::authenticate::types::Authenticate; 16 | use crate::hook::authenticate::types::Outcome; 17 | use crate::hook::intercept::types::{AuthCtx, ConnectionCtx, WebSocketConnectionCtx}; 18 | 19 | use crate::hook::intercept::types::Intercept; 20 | use crate::protocol::{Command, Message, ProtocolError}; 21 | use crate::source::{Source, SourceId}; 22 | use crate::tls::{tls_acceptor, MaybeTlsStream}; 23 | 24 | type Sources = Arc>>>; 25 | 26 | /// Starts a WebSocket server with the specified configuration 27 | pub async fn serve( 28 | listen_addr: &SocketAddr, 29 | sources: Sources, 30 | intercept: Arc>, 31 | authenticate: Arc>, 32 | subscriber_config: crate::config::Subscriber, 33 | tls_config: Option, 34 | healthcheck: bool, 35 | ) -> anyhow::Result<()> 36 | where 37 | I: Intercept + Send + Sync + 'static, 38 | A: Authenticate + Send + Sync + Unpin + 'static, 39 | { 40 | let acceptor = if let Some(tls) = tls_config { 41 | Some(tls_acceptor(&tls.cert, &tls.key).context("Failed to build TLS acceptor")?) 42 | } else { 43 | None 44 | }; 45 | let listener = tokio::net::TcpListener::bind(listen_addr).await?; 46 | tracing::info!("Server listening on: {listen_addr}"); 47 | 48 | loop { 49 | let (stream, addr) = listener.accept().await?; 50 | tracing::debug!(addr = ?addr, "Accepted connection"); 51 | let acceptor = acceptor.clone(); 52 | let authenticate = Arc::clone(&authenticate); 53 | let intercept = Arc::clone(&intercept); 54 | let sources = Arc::clone(&sources); 55 | let subscriber_config = subscriber_config.clone(); 56 | 57 | tokio::spawn(async move { 58 | let io = if let Some(acceptor) = acceptor { 59 | match acceptor.accept(stream).await { 60 | Ok(stream) => { 61 | hyper_util::rt::TokioIo::new(MaybeTlsStream::Tls(Box::new(stream))) 62 | } 63 | Err(e) => { 64 | tracing::error!(addr = ?addr, "Failed to accept TLS connection: {}", e); 65 | return; 66 | } 67 | } 68 | } else { 69 | hyper_util::rt::TokioIo::new(MaybeTlsStream::Plain(stream)) 70 | }; 71 | 72 | let builder = 73 | hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()); 74 | let conn_fut = builder.serve_connection_with_upgrades( 75 | io, 76 | service_fn(move |req: Request| { 77 | let authenticate = Arc::clone(&authenticate); 78 | let sources = Arc::clone(&sources); 79 | let intercept = Arc::clone(&intercept); 80 | let subscriber_config = subscriber_config.clone(); 81 | 82 | async move { 83 | if healthcheck && req.uri().path() == "/health" { 84 | return Response::builder() 85 | .status(StatusCode::OK) 86 | .body(Empty::new()); 87 | } 88 | 89 | let response = handle_ws( 90 | sources, 91 | intercept, 92 | authenticate, 93 | subscriber_config, 94 | addr, 95 | req, 96 | ) 97 | .await; 98 | 99 | Ok(response) 100 | } 101 | }), 102 | ); 103 | 104 | if let Err(e) = conn_fut.await { 105 | tracing::error!(addr = ?addr, "Error occurred while serving connection: {}", e); 106 | } 107 | }); 108 | } 109 | } 110 | 111 | #[tracing::instrument(skip_all)] 112 | async fn load_auth_ctx( 113 | authenticate: Arc>, 114 | request: Request, 115 | ) -> Result, ()> 116 | where 117 | A: Authenticate + Send + Sync + Unpin + 'static, 118 | { 119 | if let Some(hook) = authenticate.load().as_ref() { 120 | let outcome = hook.authenticate(request.map(|_| ())).await; 121 | 122 | match outcome { 123 | Ok(Outcome::Authenticate) => Ok(None), 124 | Ok(Outcome::WithContext(ctx)) => Ok(Some(AuthCtx::from_bytes(ctx))), 125 | outcome => { 126 | if outcome.is_err() { 127 | tracing::error!( 128 | "Failure occurred while running authentication hook: {:?}", 129 | outcome.unwrap_err() 130 | ); 131 | } 132 | 133 | return Err(()); 134 | } 135 | } 136 | } else { 137 | Ok(None) 138 | } 139 | } 140 | 141 | async fn handle_ws( 142 | sources: Sources, 143 | intercept: Arc>, 144 | authenticate: Arc>, 145 | subscriber_config: crate::config::Subscriber, 146 | addr: SocketAddr, 147 | mut request: Request, 148 | ) -> Response> 149 | where 150 | I: Intercept + Send + Sync + 'static, 151 | A: Authenticate + Send + Sync + Unpin + 'static, 152 | { 153 | let (response, fut) = upgrade::upgrade(&mut request).expect("Failed to upgrade connection"); 154 | 155 | let authenticate = Arc::clone(&authenticate); 156 | 157 | let auth_ctx = if let Ok(auth_ctx) = load_auth_ctx(authenticate, request).await { 158 | auth_ctx 159 | } else { 160 | return Response::builder() 161 | .status(StatusCode::UNAUTHORIZED) 162 | .body(Empty::new()) 163 | .unwrap(); 164 | }; 165 | 166 | let connection_ctx = ConnectionCtx::WebSocket(WebSocketConnectionCtx { addr }); 167 | 168 | tokio::spawn(async move { 169 | if let Err(e) = handle_client( 170 | fut, 171 | sources, 172 | intercept, 173 | subscriber_config, 174 | connection_ctx.clone(), 175 | auth_ctx, 176 | ) 177 | .await 178 | { 179 | tracing::error!( 180 | addr = ?addr, 181 | "Error occurred while serving WebSocket client: {}", 182 | e 183 | ); 184 | } 185 | 186 | tracing::debug!(connection = ?connection_ctx, "WebSocket connection terminated normally"); 187 | }); 188 | 189 | response 190 | } 191 | 192 | async fn handle_client( 193 | fut: upgrade::UpgradeFut, 194 | sources: Sources, 195 | intercept: Arc>, 196 | subscriber_config: crate::config::Subscriber, 197 | connection_ctx: ConnectionCtx, 198 | auth_ctx: Option, 199 | ) -> anyhow::Result<()> 200 | where 201 | I: Intercept + Send + Sync + 'static, 202 | { 203 | let ws = fut.await?; 204 | let mut ws = fastwebsockets::FragmentCollector::new(ws); 205 | 206 | tracing::debug!(connection = ?connection_ctx, "WebSocket connection established"); 207 | 208 | let (msg_tx, mut msg_rx) = tokio::sync::mpsc::unbounded_channel::(); 209 | let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::(); 210 | 211 | let actor = ConnectionManager::new( 212 | sources, 213 | cmd_rx, 214 | msg_tx, 215 | connection_ctx.clone(), 216 | auth_ctx, 217 | intercept, 218 | subscriber_config, 219 | ); 220 | 221 | // Spawn the ingest actor. If it terminates, the connection should be closed 222 | tokio::spawn(async move { 223 | if let Err(err) = actor.run().await { 224 | tracing::error!(connection = ?connection_ctx, "Connection manager terminated with error: {:?}", err); 225 | } 226 | }); 227 | 228 | loop { 229 | tokio::select! { 230 | biased; 231 | 232 | maybe_cmd = recv_cmd(&mut ws) => { 233 | match maybe_cmd { 234 | Some(Ok(cmd)) => { 235 | if cmd_tx.send(cmd).is_err() { 236 | // If the send failed, the channel is closed thus we should 237 | // terminate the connection 238 | break; 239 | } 240 | } 241 | Some(Err(e)) => { 242 | let (close_code, reason) = match e { 243 | RecvError::WebSocket(e) => { 244 | match e { 245 | WebSocketError::ConnectionClosed => break, 246 | e => return Err(e.into()), 247 | } 248 | } 249 | RecvError::Protocol(e) => { 250 | match e { 251 | ProtocolError::CommandDeserialization(_) => { 252 | (CloseCode::Policy, e.to_string()) 253 | } 254 | ProtocolError::UnsupportedCommandForm => { 255 | (CloseCode::Unsupported, e.to_string()) 256 | } 257 | } 258 | }, 259 | }; 260 | 261 | let frame = Frame::close(close_code.into(), reason.as_bytes()); 262 | ws.write_frame(frame).await?; 263 | break; 264 | } 265 | None => { 266 | // The connection has been closed 267 | break; 268 | } 269 | } 270 | }, 271 | msg = msg_rx.recv() => { 272 | match msg { 273 | Some(msg) => { 274 | let txt = serde_json::to_string(&msg).expect("failed to serialize message"); 275 | 276 | let frame = Frame::text(Payload::from(txt.as_bytes())); 277 | 278 | ws.write_frame(frame).await?; 279 | 280 | } 281 | None => { 282 | // The sole sender (our ingest actor) has hung up for some reason so we want to 283 | // terminate the connection 284 | break; 285 | }, 286 | } 287 | } 288 | } 289 | } 290 | 291 | Ok(()) 292 | } 293 | 294 | enum RecvError { 295 | WebSocket(WebSocketError), 296 | Protocol(ProtocolError), 297 | } 298 | 299 | async fn recv_cmd(ws: &mut FragmentCollector) -> Option> 300 | where 301 | S: AsyncReadExt + AsyncWriteExt + Unpin, 302 | { 303 | let frame = match ws.read_frame().await { 304 | Ok(frame) => frame, 305 | Err(e) => { 306 | return Some(Err(RecvError::WebSocket(e))); 307 | } 308 | }; 309 | 310 | match frame.opcode { 311 | fastwebsockets::OpCode::Text => { 312 | Some( 313 | serde_json::from_slice::(&frame.payload).map_err(|_| { 314 | RecvError::Protocol(ProtocolError::CommandDeserialization( 315 | // SAFETY: We know the payload is valid UTF-8 because `read_frame` 316 | // guarantees that text frames payloads are valid UTF-8 317 | unsafe { std::str::from_utf8_unchecked(&frame.payload) }.to_string(), 318 | )) 319 | }), 320 | ) 321 | } 322 | fastwebsockets::OpCode::Binary => Some(Err(RecvError::Protocol( 323 | ProtocolError::UnsupportedCommandForm, 324 | ))), 325 | fastwebsockets::OpCode::Close => None, 326 | _ => panic!("Received unexpected opcode"), 327 | } 328 | } 329 | -------------------------------------------------------------------------------- /src/kiwi/tests/common/healthcheck.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use anyhow::anyhow; 4 | 5 | pub struct Healthcheck<'a> { 6 | pub interval: Duration, 7 | pub attempts: u32, 8 | pub url: &'a str, 9 | } 10 | 11 | impl<'a> Healthcheck<'a> { 12 | pub async fn run(&self) -> anyhow::Result<()> { 13 | for _ in 0..self.attempts { 14 | if let Ok(response) = reqwest::get(self.url).await { 15 | if response.status().is_success() { 16 | return Ok(()); 17 | } 18 | } 19 | 20 | tokio::time::sleep(self.interval).await; 21 | } 22 | 23 | Err(anyhow!( 24 | "Healthcheck failed after {} attempts", 25 | self.attempts 26 | )) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/kiwi/tests/common/kafka.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{BTreeMap, BTreeSet}; 2 | 3 | use maplit::btreemap; 4 | use rdkafka::admin::{AdminClient as RdKafkaAdminClient, AdminOptions, NewPartitions}; 5 | use rdkafka::client::DefaultClientContext; 6 | use rdkafka::config::ClientConfig; 7 | use rdkafka::producer::{FutureProducer, FutureRecord}; 8 | 9 | type KafkaAdminClient = RdKafkaAdminClient; 10 | 11 | pub struct AdminClient { 12 | inner: KafkaAdminClient, 13 | options: AdminOptions, 14 | topics: BTreeSet, 15 | } 16 | 17 | impl AdminClient { 18 | pub fn new(bootstrap_servers: &str) -> anyhow::Result { 19 | Self::new_with_config(btreemap! { 20 | "bootstrap.servers" => bootstrap_servers, 21 | }) 22 | } 23 | 24 | pub fn new_with_config(config: BTreeMap) -> anyhow::Result 25 | where 26 | K: Into, 27 | V: Into, 28 | { 29 | let mut client_config = ClientConfig::new(); 30 | 31 | client_config.extend(config.into_iter().map(|(k, v)| (k.into(), v.into()))); 32 | 33 | let admin_client = client_config.create::()?; 34 | 35 | Ok(Self { 36 | inner: admin_client, 37 | options: Default::default(), 38 | topics: Default::default(), 39 | }) 40 | } 41 | 42 | pub async fn create_random_topic(&mut self, num_partitions: i32) -> anyhow::Result { 43 | let topic = format!("test-{}", nanoid::nanoid!()); 44 | 45 | self.create_topic(&topic, num_partitions, 1).await?; 46 | 47 | Ok(topic) 48 | } 49 | 50 | pub async fn create_named_topic( 51 | &mut self, 52 | topic: &str, 53 | num_partitions: i32, 54 | ) -> anyhow::Result<()> { 55 | self.create_topic(topic, num_partitions, 1).await 56 | } 57 | 58 | pub async fn update_partitions( 59 | &mut self, 60 | topic_name: &str, 61 | new_partition_count: usize, 62 | ) -> anyhow::Result<()> { 63 | let result = self 64 | .inner 65 | .create_partitions( 66 | &[NewPartitions { 67 | topic_name, 68 | new_partition_count, 69 | assignment: None, 70 | }], 71 | &self.options, 72 | ) 73 | .await?; 74 | 75 | result[0].as_ref().map_err(|(topic, error)| { 76 | anyhow::anyhow!("Failed to add partitions to topic {}: {}", topic, error) 77 | })?; 78 | 79 | Ok(()) 80 | } 81 | 82 | async fn create_topic( 83 | &mut self, 84 | topic: &str, 85 | num_partitions: i32, 86 | replication_factor: i32, 87 | ) -> anyhow::Result<()> { 88 | let new_topic = rdkafka::admin::NewTopic::new( 89 | topic, 90 | num_partitions, 91 | rdkafka::admin::TopicReplication::Fixed(replication_factor), 92 | ); 93 | 94 | let result = self 95 | .inner 96 | .create_topics(&[new_topic], &self.options) 97 | .await?; 98 | 99 | result[0].as_ref().map_err(|(topic, error)| { 100 | anyhow::anyhow!("Failed to create topic {}: {}", topic, error) 101 | })?; 102 | 103 | assert!(self.topics.insert(topic.to_string())); 104 | 105 | Ok(()) 106 | } 107 | 108 | async fn delete_topics(&mut self, topics: &[&str]) -> anyhow::Result<()> { 109 | let result = self.inner.delete_topics(topics, &self.options).await?; 110 | 111 | for result in result { 112 | match result { 113 | Ok(topic) => { 114 | self.topics.remove(&topic); 115 | } 116 | Err((topic, error)) => { 117 | return Err(anyhow::anyhow!( 118 | "Failed to delete topic {}: {}", 119 | topic, 120 | error 121 | )); 122 | } 123 | } 124 | } 125 | 126 | Ok(()) 127 | } 128 | } 129 | 130 | impl Drop for AdminClient { 131 | fn drop(&mut self) { 132 | let topics = self.topics.clone(); 133 | let topics: Vec<&str> = topics.iter().map(|s| s.as_ref()).collect(); 134 | 135 | if !topics.is_empty() { 136 | futures::executor::block_on(async { self.delete_topics(topics.as_slice()).await }) 137 | .expect("Failed to delete topics"); 138 | } 139 | } 140 | } 141 | 142 | pub struct Producer { 143 | inner: FutureProducer, 144 | } 145 | 146 | impl Producer { 147 | pub fn new(bootstrap_servers: &str) -> anyhow::Result { 148 | let producer: FutureProducer = rdkafka::config::ClientConfig::new() 149 | .set("bootstrap.servers", bootstrap_servers) 150 | .set("message.timeout.ms", "5000") 151 | .create()?; 152 | 153 | Ok(Self { inner: producer }) 154 | } 155 | 156 | pub async fn send(&self, topic: &str, key: &str, payload: &str) -> anyhow::Result<()> { 157 | let record = FutureRecord::to(topic).payload(payload).key(key); 158 | 159 | self.inner 160 | .send(record, std::time::Duration::from_secs(0)) 161 | .await 162 | .map_err(|(e, _)| anyhow::anyhow!("Failed to send message: {}", e))?; 163 | 164 | Ok(()) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/kiwi/tests/common/kiwi.rs: -------------------------------------------------------------------------------- 1 | use std::io::Write; 2 | 3 | use anyhow::Context; 4 | use nix::sys::signal::{self, Signal}; 5 | use nix::unistd::Pid; 6 | use tempfile::{NamedTempFile, TempPath}; 7 | 8 | pub struct Process { 9 | proc: std::process::Child, 10 | } 11 | 12 | impl Process { 13 | pub fn new_with_args(args: &[&str]) -> anyhow::Result { 14 | let proc = std::process::Command::new("kiwi") 15 | .args(args) 16 | .spawn() 17 | .context("failed to spawn kiwi process")?; 18 | 19 | Ok(Self { proc }) 20 | } 21 | 22 | pub fn kill(&mut self) { 23 | self.proc.kill().expect("failed to kill kiwi process"); 24 | } 25 | 26 | pub fn signal(&self, signal: Signal) -> anyhow::Result<()> { 27 | signal::kill(Pid::from_raw(self.proc.id().try_into().unwrap()), signal)?; 28 | 29 | Ok(()) 30 | } 31 | 32 | pub fn proc(&self) -> &std::process::Child { 33 | &self.proc 34 | } 35 | 36 | pub fn proc_mut(&mut self) -> &mut std::process::Child { 37 | &mut self.proc 38 | } 39 | } 40 | 41 | impl Drop for Process { 42 | fn drop(&mut self) { 43 | self.kill(); 44 | } 45 | } 46 | 47 | pub struct ConfigFile { 48 | inner: NamedTempFile, 49 | } 50 | 51 | impl ConfigFile { 52 | pub fn from_str(s: &str) -> anyhow::Result { 53 | let mut file = NamedTempFile::new().context("failed to create temporary file")?; 54 | file.as_file_mut() 55 | .write_all(s.as_bytes()) 56 | .context("failed to write config to temporary file")?; 57 | 58 | Ok(Self { inner: file }) 59 | } 60 | 61 | pub fn as_file_mut(&mut self) -> &std::fs::File { 62 | self.inner.as_file_mut() 63 | } 64 | 65 | pub fn path(&self) -> &std::path::Path { 66 | self.inner.path() 67 | } 68 | 69 | pub fn path_str(&self) -> &str { 70 | self.path().to_str().expect("path is not valid utf-8") 71 | } 72 | 73 | pub fn into_temp_path(self) -> TempPath { 74 | self.inner.into_temp_path() 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/kiwi/tests/common/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod healthcheck; 2 | pub mod kafka; 3 | pub mod kiwi; 4 | pub mod ws; 5 | -------------------------------------------------------------------------------- /src/kiwi/tests/common/ws.rs: -------------------------------------------------------------------------------- 1 | use fastwebsockets::FragmentCollector; 2 | use fastwebsockets::Frame; 3 | use fastwebsockets::OpCode; 4 | use fastwebsockets::Payload; 5 | use futures::Future; 6 | use http_body_util::Empty; 7 | use tokio::net::TcpStream; 8 | 9 | use bytes::Bytes; 10 | use hyper::body::Incoming; 11 | use hyper::header::CONNECTION; 12 | use hyper::header::UPGRADE; 13 | use hyper::upgrade::Upgraded; 14 | use hyper::{Request, Response, Uri}; 15 | use hyper_util::rt::TokioIo; 16 | 17 | pub struct Client { 18 | ws: FragmentCollector>, 19 | } 20 | 21 | struct SpawnExecutor; 22 | 23 | impl hyper::rt::Executor for SpawnExecutor 24 | where 25 | Fut: Future + Send + 'static, 26 | Fut::Output: Send + 'static, 27 | { 28 | fn execute(&self, fut: Fut) { 29 | tokio::task::spawn(fut); 30 | } 31 | } 32 | 33 | impl Client { 34 | pub async fn connect(uri: &str) -> anyhow::Result<(Self, Response)> { 35 | let uri: Uri = uri.try_into()?; 36 | let stream = TcpStream::connect( 37 | format!("{}:{}", uri.host().unwrap(), uri.port_u16().unwrap()).as_str(), 38 | ) 39 | .await?; 40 | 41 | let req = Request::builder() 42 | .method("GET") 43 | .uri(&uri) 44 | .header("Host", uri.host().unwrap()) 45 | .header(UPGRADE, "websocket") 46 | .header(CONNECTION, "upgrade") 47 | .header( 48 | "Sec-WebSocket-Key", 49 | fastwebsockets::handshake::generate_key(), 50 | ) 51 | .header("Sec-WebSocket-Version", "13") 52 | .body(Empty::::new())?; 53 | 54 | let (ws, res) = fastwebsockets::handshake::client(&SpawnExecutor, req, stream).await?; 55 | 56 | Ok(( 57 | Self { 58 | ws: FragmentCollector::new(ws), 59 | }, 60 | res, 61 | )) 62 | } 63 | 64 | pub async fn send_text(&mut self, text: &str) -> anyhow::Result<()> { 65 | self.ws 66 | .write_frame(Frame::text(Payload::Borrowed(text.as_bytes()))) 67 | .await?; 68 | 69 | Ok(()) 70 | } 71 | 72 | pub async fn send_json(&mut self, value: &T) -> anyhow::Result<()> { 73 | let text = serde_json::to_string(value)?; 74 | self.send_text(&text).await?; 75 | 76 | Ok(()) 77 | } 78 | 79 | pub async fn recv_text_frame(&mut self) -> anyhow::Result> { 80 | let frame = self.ws.read_frame().await?; 81 | 82 | match frame.opcode { 83 | OpCode::Text => Ok(frame), 84 | _ => Err(anyhow::anyhow!("Expected text frame")), 85 | } 86 | } 87 | 88 | pub async fn recv_json(&mut self) -> anyhow::Result { 89 | let text_frame = self.recv_text_frame().await?; 90 | let text = std::str::from_utf8(text_frame.payload.as_ref())?; 91 | let value = serde_json::from_str(&text)?; 92 | 93 | Ok(value) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/kiwi/tests/hook.rs: -------------------------------------------------------------------------------- 1 | pub mod common; 2 | 3 | use std::time::Duration; 4 | 5 | use common::kiwi::{ConfigFile, Process}; 6 | 7 | use crate::common::healthcheck::Healthcheck; 8 | use crate::common::ws::Client as WsClient; 9 | 10 | /// Test that the `authenticate` hook works as expected. 11 | #[tokio::test] 12 | async fn test_authenticate() -> anyhow::Result<()> { 13 | const AUTHENTICATE_PATH: &str = concat!( 14 | env!("CARGO_MANIFEST_DIR"), 15 | "/tests/wasm/authenticate-api-key.wasm" 16 | ); 17 | 18 | let config = ConfigFile::from_str( 19 | format!( 20 | r#" 21 | hooks: 22 | authenticate: {AUTHENTICATE_PATH} 23 | sources: [] 24 | server: 25 | address: '127.0.0.1:8000' 26 | "# 27 | ) 28 | .as_str(), 29 | )?; 30 | 31 | let _kiwi = Process::new_with_args(&["--config", config.path_str()])?; 32 | 33 | Healthcheck { 34 | interval: Duration::from_millis(200), 35 | attempts: 10, 36 | url: "http://127.0.0.1:8000/health", 37 | } 38 | .run() 39 | .await?; 40 | 41 | assert!(WsClient::connect("ws://127.0.0.1:8000").await.is_err()); 42 | assert!(WsClient::connect("ws://127.0.0.1:8000/?x-api-key=wrong") 43 | .await 44 | .is_err()); 45 | assert!(WsClient::connect("ws://127.0.0.1:8000/?x-api-key=12345") 46 | .await 47 | .is_ok()); 48 | 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /src/kiwi/tests/lifecycle.rs: -------------------------------------------------------------------------------- 1 | pub mod common; 2 | 3 | use std::time::Duration; 4 | 5 | use common::kiwi::{ConfigFile, Process}; 6 | use nix::sys::signal; 7 | 8 | static GRACE_PERIOD_MS: u64 = 500; 9 | 10 | #[tokio::test] 11 | async fn test_shuts_down_sigint() -> anyhow::Result<()> { 12 | let config = ConfigFile::from_str( 13 | format!( 14 | r#" 15 | sources: [] 16 | server: 17 | address: '127.0.0.1:8000' 18 | "# 19 | ) 20 | .as_str(), 21 | )?; 22 | 23 | let mut kiwi = Process::new_with_args(&["--config", config.path_str()])?; 24 | 25 | kiwi.signal(signal::SIGINT)?; 26 | 27 | tokio::time::sleep(Duration::from_millis(GRACE_PERIOD_MS)).await; 28 | 29 | let status = kiwi.proc_mut().try_wait()?; 30 | 31 | assert!(status.is_some()); 32 | 33 | Ok(()) 34 | } 35 | 36 | #[tokio::test] 37 | async fn test_shuts_down_sigterm() -> anyhow::Result<()> { 38 | let config = ConfigFile::from_str( 39 | format!( 40 | r#" 41 | sources: [] 42 | server: 43 | address: '127.0.0.1:8000' 44 | "# 45 | ) 46 | .as_str(), 47 | )?; 48 | 49 | let mut kiwi = Process::new_with_args(&["--config", config.path_str()])?; 50 | 51 | kiwi.signal(signal::SIGTERM)?; 52 | 53 | tokio::time::sleep(Duration::from_millis(GRACE_PERIOD_MS)).await; 54 | 55 | let status = kiwi.proc_mut().try_wait()?; 56 | 57 | assert!(status.is_some()); 58 | 59 | Ok(()) 60 | } 61 | -------------------------------------------------------------------------------- /src/kiwi/tests/wasm/authenticate-api-key.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rkrishn7/kiwi/682f1bf2410ccb35143cf41d5f3479c7919fe27a/src/kiwi/tests/wasm/authenticate-api-key.wasm -------------------------------------------------------------------------------- /src/kiwi/tests/wasm/kafka-even-numbers-intercept.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rkrishn7/kiwi/682f1bf2410ccb35143cf41d5f3479c7919fe27a/src/kiwi/tests/wasm/kafka-even-numbers-intercept.wasm -------------------------------------------------------------------------------- /src/wit/authenticate-types.wit: -------------------------------------------------------------------------------- 1 | interface authenticate-types { 2 | use wasi:http/types@0.2.0.{method, scheme, field-key, field-value}; 3 | 4 | variant outcome { 5 | authenticate, 6 | reject, 7 | with-context(list), 8 | } 9 | 10 | // This is the object form of the resource `wasi:http/types@0.2.0.{incoming-request}`. 11 | // Curently, it's duplicated due to the lack of support for including re-mapped types 12 | // in exported functions arguments, via `--with`. The issue is tracked 13 | // [here](https://github.com/bytecodealliance/wit-bindgen/issues/832). 14 | // 15 | // Until the issue is resolved, we have to duplicate the type here, and massage it to be 16 | // compatible with the `Request` type in the kiwi-sdk. 17 | // 18 | // NOTE: Resolving this workaround will likely require a new major version of the SDK to be 19 | // released, as it will be a breaking change unless we can guarantee API compatibility between 20 | // the two types. 21 | record http-request { 22 | /// Returns the method of the incoming request. 23 | method: method, 24 | 25 | /// Returns the path with query parameters from the request, as a string. 26 | path-with-query: option, 27 | 28 | /// Returns the protocol scheme from the request. 29 | scheme: option, 30 | 31 | /// Returns the authority from the request, if it was present. 32 | authority: option, 33 | 34 | /// Get the `headers` associated with the request. 35 | headers: list>, 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/wit/deps/cli/command.wit: -------------------------------------------------------------------------------- 1 | package wasi:cli@0.2.0; 2 | 3 | world command { 4 | include imports; 5 | 6 | export run; 7 | } 8 | -------------------------------------------------------------------------------- /src/wit/deps/cli/environment.wit: -------------------------------------------------------------------------------- 1 | interface environment { 2 | /// Get the POSIX-style environment variables. 3 | /// 4 | /// Each environment variable is provided as a pair of string variable names 5 | /// and string value. 6 | /// 7 | /// Morally, these are a value import, but until value imports are available 8 | /// in the component model, this import function should return the same 9 | /// values each time it is called. 10 | get-environment: func() -> list>; 11 | 12 | /// Get the POSIX-style arguments to the program. 13 | get-arguments: func() -> list; 14 | 15 | /// Return a path that programs should use as their initial current working 16 | /// directory, interpreting `.` as shorthand for this. 17 | initial-cwd: func() -> option; 18 | } 19 | -------------------------------------------------------------------------------- /src/wit/deps/cli/exit.wit: -------------------------------------------------------------------------------- 1 | interface exit { 2 | /// Exit the current instance and any linked instances. 3 | exit: func(status: result); 4 | } 5 | -------------------------------------------------------------------------------- /src/wit/deps/cli/imports.wit: -------------------------------------------------------------------------------- 1 | package wasi:cli@0.2.0; 2 | 3 | world imports { 4 | include wasi:clocks/imports@0.2.0; 5 | include wasi:filesystem/imports@0.2.0; 6 | include wasi:sockets/imports@0.2.0; 7 | include wasi:random/imports@0.2.0; 8 | include wasi:io/imports@0.2.0; 9 | 10 | import environment; 11 | import exit; 12 | import stdin; 13 | import stdout; 14 | import stderr; 15 | import terminal-input; 16 | import terminal-output; 17 | import terminal-stdin; 18 | import terminal-stdout; 19 | import terminal-stderr; 20 | } 21 | -------------------------------------------------------------------------------- /src/wit/deps/cli/run.wit: -------------------------------------------------------------------------------- 1 | interface run { 2 | /// Run the program. 3 | run: func() -> result; 4 | } 5 | -------------------------------------------------------------------------------- /src/wit/deps/cli/stdio.wit: -------------------------------------------------------------------------------- 1 | interface stdin { 2 | use wasi:io/streams@0.2.0.{input-stream}; 3 | 4 | get-stdin: func() -> input-stream; 5 | } 6 | 7 | interface stdout { 8 | use wasi:io/streams@0.2.0.{output-stream}; 9 | 10 | get-stdout: func() -> output-stream; 11 | } 12 | 13 | interface stderr { 14 | use wasi:io/streams@0.2.0.{output-stream}; 15 | 16 | get-stderr: func() -> output-stream; 17 | } 18 | -------------------------------------------------------------------------------- /src/wit/deps/cli/terminal.wit: -------------------------------------------------------------------------------- 1 | /// Terminal input. 2 | /// 3 | /// In the future, this may include functions for disabling echoing, 4 | /// disabling input buffering so that keyboard events are sent through 5 | /// immediately, querying supported features, and so on. 6 | interface terminal-input { 7 | /// The input side of a terminal. 8 | resource terminal-input; 9 | } 10 | 11 | /// Terminal output. 12 | /// 13 | /// In the future, this may include functions for querying the terminal 14 | /// size, being notified of terminal size changes, querying supported 15 | /// features, and so on. 16 | interface terminal-output { 17 | /// The output side of a terminal. 18 | resource terminal-output; 19 | } 20 | 21 | /// An interface providing an optional `terminal-input` for stdin as a 22 | /// link-time authority. 23 | interface terminal-stdin { 24 | use terminal-input.{terminal-input}; 25 | 26 | /// If stdin is connected to a terminal, return a `terminal-input` handle 27 | /// allowing further interaction with it. 28 | get-terminal-stdin: func() -> option; 29 | } 30 | 31 | /// An interface providing an optional `terminal-output` for stdout as a 32 | /// link-time authority. 33 | interface terminal-stdout { 34 | use terminal-output.{terminal-output}; 35 | 36 | /// If stdout is connected to a terminal, return a `terminal-output` handle 37 | /// allowing further interaction with it. 38 | get-terminal-stdout: func() -> option; 39 | } 40 | 41 | /// An interface providing an optional `terminal-output` for stderr as a 42 | /// link-time authority. 43 | interface terminal-stderr { 44 | use terminal-output.{terminal-output}; 45 | 46 | /// If stderr is connected to a terminal, return a `terminal-output` handle 47 | /// allowing further interaction with it. 48 | get-terminal-stderr: func() -> option; 49 | } 50 | -------------------------------------------------------------------------------- /src/wit/deps/clocks/monotonic-clock.wit: -------------------------------------------------------------------------------- 1 | package wasi:clocks@0.2.0; 2 | /// WASI Monotonic Clock is a clock API intended to let users measure elapsed 3 | /// time. 4 | /// 5 | /// It is intended to be portable at least between Unix-family platforms and 6 | /// Windows. 7 | /// 8 | /// A monotonic clock is a clock which has an unspecified initial value, and 9 | /// successive reads of the clock will produce non-decreasing values. 10 | /// 11 | /// It is intended for measuring elapsed time. 12 | interface monotonic-clock { 13 | use wasi:io/poll@0.2.0.{pollable}; 14 | 15 | /// An instant in time, in nanoseconds. An instant is relative to an 16 | /// unspecified initial value, and can only be compared to instances from 17 | /// the same monotonic-clock. 18 | type instant = u64; 19 | 20 | /// A duration of time, in nanoseconds. 21 | type duration = u64; 22 | 23 | /// Read the current value of the clock. 24 | /// 25 | /// The clock is monotonic, therefore calling this function repeatedly will 26 | /// produce a sequence of non-decreasing values. 27 | now: func() -> instant; 28 | 29 | /// Query the resolution of the clock. Returns the duration of time 30 | /// corresponding to a clock tick. 31 | resolution: func() -> duration; 32 | 33 | /// Create a `pollable` which will resolve once the specified instant 34 | /// occured. 35 | subscribe-instant: func( 36 | when: instant, 37 | ) -> pollable; 38 | 39 | /// Create a `pollable` which will resolve once the given duration has 40 | /// elapsed, starting at the time at which this function was called. 41 | /// occured. 42 | subscribe-duration: func( 43 | when: duration, 44 | ) -> pollable; 45 | } 46 | -------------------------------------------------------------------------------- /src/wit/deps/clocks/wall-clock.wit: -------------------------------------------------------------------------------- 1 | package wasi:clocks@0.2.0; 2 | /// WASI Wall Clock is a clock API intended to let users query the current 3 | /// time. The name "wall" makes an analogy to a "clock on the wall", which 4 | /// is not necessarily monotonic as it may be reset. 5 | /// 6 | /// It is intended to be portable at least between Unix-family platforms and 7 | /// Windows. 8 | /// 9 | /// A wall clock is a clock which measures the date and time according to 10 | /// some external reference. 11 | /// 12 | /// External references may be reset, so this clock is not necessarily 13 | /// monotonic, making it unsuitable for measuring elapsed time. 14 | /// 15 | /// It is intended for reporting the current date and time for humans. 16 | interface wall-clock { 17 | /// A time and date in seconds plus nanoseconds. 18 | record datetime { 19 | seconds: u64, 20 | nanoseconds: u32, 21 | } 22 | 23 | /// Read the current value of the clock. 24 | /// 25 | /// This clock is not monotonic, therefore calling this function repeatedly 26 | /// will not necessarily produce a sequence of non-decreasing values. 27 | /// 28 | /// The returned timestamps represent the number of seconds since 29 | /// 1970-01-01T00:00:00Z, also known as [POSIX's Seconds Since the Epoch], 30 | /// also known as [Unix Time]. 31 | /// 32 | /// The nanoseconds field of the output is always less than 1000000000. 33 | /// 34 | /// [POSIX's Seconds Since the Epoch]: https://pubs.opengroup.org/onlinepubs/9699919799/xrat/V4_xbd_chap04.html#tag_21_04_16 35 | /// [Unix Time]: https://en.wikipedia.org/wiki/Unix_time 36 | now: func() -> datetime; 37 | 38 | /// Query the resolution of the clock. 39 | /// 40 | /// The nanoseconds field of the output is always less than 1000000000. 41 | resolution: func() -> datetime; 42 | } 43 | -------------------------------------------------------------------------------- /src/wit/deps/clocks/world.wit: -------------------------------------------------------------------------------- 1 | package wasi:clocks@0.2.0; 2 | 3 | world imports { 4 | import monotonic-clock; 5 | import wall-clock; 6 | } 7 | -------------------------------------------------------------------------------- /src/wit/deps/filesystem/preopens.wit: -------------------------------------------------------------------------------- 1 | package wasi:filesystem@0.2.0; 2 | 3 | interface preopens { 4 | use types.{descriptor}; 5 | 6 | /// Return the set of preopened directories, and their path. 7 | get-directories: func() -> list>; 8 | } 9 | -------------------------------------------------------------------------------- /src/wit/deps/filesystem/world.wit: -------------------------------------------------------------------------------- 1 | package wasi:filesystem@0.2.0; 2 | 3 | world imports { 4 | import types; 5 | import preopens; 6 | } 7 | -------------------------------------------------------------------------------- /src/wit/deps/http/handler.wit: -------------------------------------------------------------------------------- 1 | /// This interface defines a handler of incoming HTTP Requests. It should 2 | /// be exported by components which can respond to HTTP Requests. 3 | interface incoming-handler { 4 | use types.{incoming-request, response-outparam}; 5 | 6 | /// This function is invoked with an incoming HTTP Request, and a resource 7 | /// `response-outparam` which provides the capability to reply with an HTTP 8 | /// Response. The response is sent by calling the `response-outparam.set` 9 | /// method, which allows execution to continue after the response has been 10 | /// sent. This enables both streaming to the response body, and performing other 11 | /// work. 12 | /// 13 | /// The implementor of this function must write a response to the 14 | /// `response-outparam` before returning, or else the caller will respond 15 | /// with an error on its behalf. 16 | handle: func( 17 | request: incoming-request, 18 | response-out: response-outparam 19 | ); 20 | } 21 | 22 | /// This interface defines a handler of outgoing HTTP Requests. It should be 23 | /// imported by components which wish to make HTTP Requests. 24 | interface outgoing-handler { 25 | use types.{ 26 | outgoing-request, request-options, future-incoming-response, error-code 27 | }; 28 | 29 | /// This function is invoked with an outgoing HTTP Request, and it returns 30 | /// a resource `future-incoming-response` which represents an HTTP Response 31 | /// which may arrive in the future. 32 | /// 33 | /// The `options` argument accepts optional parameters for the HTTP 34 | /// protocol's transport layer. 35 | /// 36 | /// This function may return an error if the `outgoing-request` is invalid 37 | /// or not allowed to be made. Otherwise, protocol errors are reported 38 | /// through the `future-incoming-response`. 39 | handle: func( 40 | request: outgoing-request, 41 | options: option 42 | ) -> result; 43 | } 44 | -------------------------------------------------------------------------------- /src/wit/deps/http/proxy.wit: -------------------------------------------------------------------------------- 1 | package wasi:http@0.2.0; 2 | 3 | /// The `wasi:http/proxy` world captures a widely-implementable intersection of 4 | /// hosts that includes HTTP forward and reverse proxies. Components targeting 5 | /// this world may concurrently stream in and out any number of incoming and 6 | /// outgoing HTTP requests. 7 | world proxy { 8 | /// HTTP proxies have access to time and randomness. 9 | include wasi:clocks/imports@0.2.0; 10 | import wasi:random/random@0.2.0; 11 | 12 | /// Proxies have standard output and error streams which are expected to 13 | /// terminate in a developer-facing console provided by the host. 14 | import wasi:cli/stdout@0.2.0; 15 | import wasi:cli/stderr@0.2.0; 16 | 17 | /// TODO: this is a temporary workaround until component tooling is able to 18 | /// gracefully handle the absence of stdin. Hosts must return an eof stream 19 | /// for this import, which is what wasi-libc + tooling will do automatically 20 | /// when this import is properly removed. 21 | import wasi:cli/stdin@0.2.0; 22 | 23 | /// This is the default handler to use when user code simply wants to make an 24 | /// HTTP request (e.g., via `fetch()`). 25 | import outgoing-handler; 26 | 27 | /// The host delivers incoming HTTP requests to a component by calling the 28 | /// `handle` function of this exported interface. A host may arbitrarily reuse 29 | /// or not reuse component instance when delivering incoming HTTP requests and 30 | /// thus a component must be able to handle 0..N calls to `handle`. 31 | export incoming-handler; 32 | } 33 | -------------------------------------------------------------------------------- /src/wit/deps/io/error.wit: -------------------------------------------------------------------------------- 1 | package wasi:io@0.2.0; 2 | 3 | 4 | interface error { 5 | /// A resource which represents some error information. 6 | /// 7 | /// The only method provided by this resource is `to-debug-string`, 8 | /// which provides some human-readable information about the error. 9 | /// 10 | /// In the `wasi:io` package, this resource is returned through the 11 | /// `wasi:io/streams/stream-error` type. 12 | /// 13 | /// To provide more specific error information, other interfaces may 14 | /// provide functions to further "downcast" this error into more specific 15 | /// error information. For example, `error`s returned in streams derived 16 | /// from filesystem types to be described using the filesystem's own 17 | /// error-code type, using the function 18 | /// `wasi:filesystem/types/filesystem-error-code`, which takes a parameter 19 | /// `borrow` and returns 20 | /// `option`. 21 | /// 22 | /// The set of functions which can "downcast" an `error` into a more 23 | /// concrete type is open. 24 | resource error { 25 | /// Returns a string that is suitable to assist humans in debugging 26 | /// this error. 27 | /// 28 | /// WARNING: The returned string should not be consumed mechanically! 29 | /// It may change across platforms, hosts, or other implementation 30 | /// details. Parsing this string is a major platform-compatibility 31 | /// hazard. 32 | to-debug-string: func() -> string; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/wit/deps/io/poll.wit: -------------------------------------------------------------------------------- 1 | package wasi:io@0.2.0; 2 | 3 | /// A poll API intended to let users wait for I/O events on multiple handles 4 | /// at once. 5 | interface poll { 6 | /// `pollable` epresents a single I/O event which may be ready, or not. 7 | resource pollable { 8 | 9 | /// Return the readiness of a pollable. This function never blocks. 10 | /// 11 | /// Returns `true` when the pollable is ready, and `false` otherwise. 12 | ready: func() -> bool; 13 | 14 | /// `block` returns immediately if the pollable is ready, and otherwise 15 | /// blocks until ready. 16 | /// 17 | /// This function is equivalent to calling `poll.poll` on a list 18 | /// containing only this pollable. 19 | block: func(); 20 | } 21 | 22 | /// Poll for completion on a set of pollables. 23 | /// 24 | /// This function takes a list of pollables, which identify I/O sources of 25 | /// interest, and waits until one or more of the events is ready for I/O. 26 | /// 27 | /// The result `list` contains one or more indices of handles in the 28 | /// argument list that is ready for I/O. 29 | /// 30 | /// If the list contains more elements than can be indexed with a `u32` 31 | /// value, this function traps. 32 | /// 33 | /// A timeout can be implemented by adding a pollable from the 34 | /// wasi-clocks API to the list. 35 | /// 36 | /// This function does not return a `result`; polling in itself does not 37 | /// do any I/O so it doesn't fail. If any of the I/O sources identified by 38 | /// the pollables has an error, it is indicated by marking the source as 39 | /// being reaedy for I/O. 40 | poll: func(in: list>) -> list; 41 | } 42 | -------------------------------------------------------------------------------- /src/wit/deps/io/streams.wit: -------------------------------------------------------------------------------- 1 | package wasi:io@0.2.0; 2 | 3 | /// WASI I/O is an I/O abstraction API which is currently focused on providing 4 | /// stream types. 5 | /// 6 | /// In the future, the component model is expected to add built-in stream types; 7 | /// when it does, they are expected to subsume this API. 8 | interface streams { 9 | use error.{error}; 10 | use poll.{pollable}; 11 | 12 | /// An error for input-stream and output-stream operations. 13 | variant stream-error { 14 | /// The last operation (a write or flush) failed before completion. 15 | /// 16 | /// More information is available in the `error` payload. 17 | last-operation-failed(error), 18 | /// The stream is closed: no more input will be accepted by the 19 | /// stream. A closed output-stream will return this error on all 20 | /// future operations. 21 | closed 22 | } 23 | 24 | /// An input bytestream. 25 | /// 26 | /// `input-stream`s are *non-blocking* to the extent practical on underlying 27 | /// platforms. I/O operations always return promptly; if fewer bytes are 28 | /// promptly available than requested, they return the number of bytes promptly 29 | /// available, which could even be zero. To wait for data to be available, 30 | /// use the `subscribe` function to obtain a `pollable` which can be polled 31 | /// for using `wasi:io/poll`. 32 | resource input-stream { 33 | /// Perform a non-blocking read from the stream. 34 | /// 35 | /// This function returns a list of bytes containing the read data, 36 | /// when successful. The returned list will contain up to `len` bytes; 37 | /// it may return fewer than requested, but not more. The list is 38 | /// empty when no bytes are available for reading at this time. The 39 | /// pollable given by `subscribe` will be ready when more bytes are 40 | /// available. 41 | /// 42 | /// This function fails with a `stream-error` when the operation 43 | /// encounters an error, giving `last-operation-failed`, or when the 44 | /// stream is closed, giving `closed`. 45 | /// 46 | /// When the caller gives a `len` of 0, it represents a request to 47 | /// read 0 bytes. If the stream is still open, this call should 48 | /// succeed and return an empty list, or otherwise fail with `closed`. 49 | /// 50 | /// The `len` parameter is a `u64`, which could represent a list of u8 which 51 | /// is not possible to allocate in wasm32, or not desirable to allocate as 52 | /// as a return value by the callee. The callee may return a list of bytes 53 | /// less than `len` in size while more bytes are available for reading. 54 | read: func( 55 | /// The maximum number of bytes to read 56 | len: u64 57 | ) -> result, stream-error>; 58 | 59 | /// Read bytes from a stream, after blocking until at least one byte can 60 | /// be read. Except for blocking, behavior is identical to `read`. 61 | blocking-read: func( 62 | /// The maximum number of bytes to read 63 | len: u64 64 | ) -> result, stream-error>; 65 | 66 | /// Skip bytes from a stream. Returns number of bytes skipped. 67 | /// 68 | /// Behaves identical to `read`, except instead of returning a list 69 | /// of bytes, returns the number of bytes consumed from the stream. 70 | skip: func( 71 | /// The maximum number of bytes to skip. 72 | len: u64, 73 | ) -> result; 74 | 75 | /// Skip bytes from a stream, after blocking until at least one byte 76 | /// can be skipped. Except for blocking behavior, identical to `skip`. 77 | blocking-skip: func( 78 | /// The maximum number of bytes to skip. 79 | len: u64, 80 | ) -> result; 81 | 82 | /// Create a `pollable` which will resolve once either the specified stream 83 | /// has bytes available to read or the other end of the stream has been 84 | /// closed. 85 | /// The created `pollable` is a child resource of the `input-stream`. 86 | /// Implementations may trap if the `input-stream` is dropped before 87 | /// all derived `pollable`s created with this function are dropped. 88 | subscribe: func() -> pollable; 89 | } 90 | 91 | 92 | /// An output bytestream. 93 | /// 94 | /// `output-stream`s are *non-blocking* to the extent practical on 95 | /// underlying platforms. Except where specified otherwise, I/O operations also 96 | /// always return promptly, after the number of bytes that can be written 97 | /// promptly, which could even be zero. To wait for the stream to be ready to 98 | /// accept data, the `subscribe` function to obtain a `pollable` which can be 99 | /// polled for using `wasi:io/poll`. 100 | resource output-stream { 101 | /// Check readiness for writing. This function never blocks. 102 | /// 103 | /// Returns the number of bytes permitted for the next call to `write`, 104 | /// or an error. Calling `write` with more bytes than this function has 105 | /// permitted will trap. 106 | /// 107 | /// When this function returns 0 bytes, the `subscribe` pollable will 108 | /// become ready when this function will report at least 1 byte, or an 109 | /// error. 110 | check-write: func() -> result; 111 | 112 | /// Perform a write. This function never blocks. 113 | /// 114 | /// Precondition: check-write gave permit of Ok(n) and contents has a 115 | /// length of less than or equal to n. Otherwise, this function will trap. 116 | /// 117 | /// returns Err(closed) without writing if the stream has closed since 118 | /// the last call to check-write provided a permit. 119 | write: func( 120 | contents: list 121 | ) -> result<_, stream-error>; 122 | 123 | /// Perform a write of up to 4096 bytes, and then flush the stream. Block 124 | /// until all of these operations are complete, or an error occurs. 125 | /// 126 | /// This is a convenience wrapper around the use of `check-write`, 127 | /// `subscribe`, `write`, and `flush`, and is implemented with the 128 | /// following pseudo-code: 129 | /// 130 | /// ```text 131 | /// let pollable = this.subscribe(); 132 | /// while !contents.is_empty() { 133 | /// // Wait for the stream to become writable 134 | /// poll-one(pollable); 135 | /// let Ok(n) = this.check-write(); // eliding error handling 136 | /// let len = min(n, contents.len()); 137 | /// let (chunk, rest) = contents.split_at(len); 138 | /// this.write(chunk ); // eliding error handling 139 | /// contents = rest; 140 | /// } 141 | /// this.flush(); 142 | /// // Wait for completion of `flush` 143 | /// poll-one(pollable); 144 | /// // Check for any errors that arose during `flush` 145 | /// let _ = this.check-write(); // eliding error handling 146 | /// ``` 147 | blocking-write-and-flush: func( 148 | contents: list 149 | ) -> result<_, stream-error>; 150 | 151 | /// Request to flush buffered output. This function never blocks. 152 | /// 153 | /// This tells the output-stream that the caller intends any buffered 154 | /// output to be flushed. the output which is expected to be flushed 155 | /// is all that has been passed to `write` prior to this call. 156 | /// 157 | /// Upon calling this function, the `output-stream` will not accept any 158 | /// writes (`check-write` will return `ok(0)`) until the flush has 159 | /// completed. The `subscribe` pollable will become ready when the 160 | /// flush has completed and the stream can accept more writes. 161 | flush: func() -> result<_, stream-error>; 162 | 163 | /// Request to flush buffered output, and block until flush completes 164 | /// and stream is ready for writing again. 165 | blocking-flush: func() -> result<_, stream-error>; 166 | 167 | /// Create a `pollable` which will resolve once the output-stream 168 | /// is ready for more writing, or an error has occured. When this 169 | /// pollable is ready, `check-write` will return `ok(n)` with n>0, or an 170 | /// error. 171 | /// 172 | /// If the stream is closed, this pollable is always ready immediately. 173 | /// 174 | /// The created `pollable` is a child resource of the `output-stream`. 175 | /// Implementations may trap if the `output-stream` is dropped before 176 | /// all derived `pollable`s created with this function are dropped. 177 | subscribe: func() -> pollable; 178 | 179 | /// Write zeroes to a stream. 180 | /// 181 | /// this should be used precisely like `write` with the exact same 182 | /// preconditions (must use check-write first), but instead of 183 | /// passing a list of bytes, you simply pass the number of zero-bytes 184 | /// that should be written. 185 | write-zeroes: func( 186 | /// The number of zero-bytes to write 187 | len: u64 188 | ) -> result<_, stream-error>; 189 | 190 | /// Perform a write of up to 4096 zeroes, and then flush the stream. 191 | /// Block until all of these operations are complete, or an error 192 | /// occurs. 193 | /// 194 | /// This is a convenience wrapper around the use of `check-write`, 195 | /// `subscribe`, `write-zeroes`, and `flush`, and is implemented with 196 | /// the following pseudo-code: 197 | /// 198 | /// ```text 199 | /// let pollable = this.subscribe(); 200 | /// while num_zeroes != 0 { 201 | /// // Wait for the stream to become writable 202 | /// poll-one(pollable); 203 | /// let Ok(n) = this.check-write(); // eliding error handling 204 | /// let len = min(n, num_zeroes); 205 | /// this.write-zeroes(len); // eliding error handling 206 | /// num_zeroes -= len; 207 | /// } 208 | /// this.flush(); 209 | /// // Wait for completion of `flush` 210 | /// poll-one(pollable); 211 | /// // Check for any errors that arose during `flush` 212 | /// let _ = this.check-write(); // eliding error handling 213 | /// ``` 214 | blocking-write-zeroes-and-flush: func( 215 | /// The number of zero-bytes to write 216 | len: u64 217 | ) -> result<_, stream-error>; 218 | 219 | /// Read from one stream and write to another. 220 | /// 221 | /// The behavior of splice is equivelant to: 222 | /// 1. calling `check-write` on the `output-stream` 223 | /// 2. calling `read` on the `input-stream` with the smaller of the 224 | /// `check-write` permitted length and the `len` provided to `splice` 225 | /// 3. calling `write` on the `output-stream` with that read data. 226 | /// 227 | /// Any error reported by the call to `check-write`, `read`, or 228 | /// `write` ends the splice and reports that error. 229 | /// 230 | /// This function returns the number of bytes transferred; it may be less 231 | /// than `len`. 232 | splice: func( 233 | /// The stream to read from 234 | src: borrow, 235 | /// The number of bytes to splice 236 | len: u64, 237 | ) -> result; 238 | 239 | /// Read from one stream and write to another, with blocking. 240 | /// 241 | /// This is similar to `splice`, except that it blocks until the 242 | /// `output-stream` is ready for writing, and the `input-stream` 243 | /// is ready for reading, before performing the `splice`. 244 | blocking-splice: func( 245 | /// The stream to read from 246 | src: borrow, 247 | /// The number of bytes to splice 248 | len: u64, 249 | ) -> result; 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /src/wit/deps/io/world.wit: -------------------------------------------------------------------------------- 1 | package wasi:io@0.2.0; 2 | 3 | world imports { 4 | import streams; 5 | import poll; 6 | } 7 | -------------------------------------------------------------------------------- /src/wit/deps/random/insecure-seed.wit: -------------------------------------------------------------------------------- 1 | package wasi:random@0.2.0; 2 | /// The insecure-seed interface for seeding hash-map DoS resistance. 3 | /// 4 | /// It is intended to be portable at least between Unix-family platforms and 5 | /// Windows. 6 | interface insecure-seed { 7 | /// Return a 128-bit value that may contain a pseudo-random value. 8 | /// 9 | /// The returned value is not required to be computed from a CSPRNG, and may 10 | /// even be entirely deterministic. Host implementations are encouraged to 11 | /// provide pseudo-random values to any program exposed to 12 | /// attacker-controlled content, to enable DoS protection built into many 13 | /// languages' hash-map implementations. 14 | /// 15 | /// This function is intended to only be called once, by a source language 16 | /// to initialize Denial Of Service (DoS) protection in its hash-map 17 | /// implementation. 18 | /// 19 | /// # Expected future evolution 20 | /// 21 | /// This will likely be changed to a value import, to prevent it from being 22 | /// called multiple times and potentially used for purposes other than DoS 23 | /// protection. 24 | insecure-seed: func() -> tuple; 25 | } 26 | -------------------------------------------------------------------------------- /src/wit/deps/random/insecure.wit: -------------------------------------------------------------------------------- 1 | package wasi:random@0.2.0; 2 | /// The insecure interface for insecure pseudo-random numbers. 3 | /// 4 | /// It is intended to be portable at least between Unix-family platforms and 5 | /// Windows. 6 | interface insecure { 7 | /// Return `len` insecure pseudo-random bytes. 8 | /// 9 | /// This function is not cryptographically secure. Do not use it for 10 | /// anything related to security. 11 | /// 12 | /// There are no requirements on the values of the returned bytes, however 13 | /// implementations are encouraged to return evenly distributed values with 14 | /// a long period. 15 | get-insecure-random-bytes: func(len: u64) -> list; 16 | 17 | /// Return an insecure pseudo-random `u64` value. 18 | /// 19 | /// This function returns the same type of pseudo-random data as 20 | /// `get-insecure-random-bytes`, represented as a `u64`. 21 | get-insecure-random-u64: func() -> u64; 22 | } 23 | -------------------------------------------------------------------------------- /src/wit/deps/random/random.wit: -------------------------------------------------------------------------------- 1 | package wasi:random@0.2.0; 2 | /// WASI Random is a random data API. 3 | /// 4 | /// It is intended to be portable at least between Unix-family platforms and 5 | /// Windows. 6 | interface random { 7 | /// Return `len` cryptographically-secure random or pseudo-random bytes. 8 | /// 9 | /// This function must produce data at least as cryptographically secure and 10 | /// fast as an adequately seeded cryptographically-secure pseudo-random 11 | /// number generator (CSPRNG). It must not block, from the perspective of 12 | /// the calling program, under any circumstances, including on the first 13 | /// request and on requests for numbers of bytes. The returned data must 14 | /// always be unpredictable. 15 | /// 16 | /// This function must always return fresh data. Deterministic environments 17 | /// must omit this function, rather than implementing it with deterministic 18 | /// data. 19 | get-random-bytes: func(len: u64) -> list; 20 | 21 | /// Return a cryptographically-secure random or pseudo-random `u64` value. 22 | /// 23 | /// This function returns the same type of data as `get-random-bytes`, 24 | /// represented as a `u64`. 25 | get-random-u64: func() -> u64; 26 | } 27 | -------------------------------------------------------------------------------- /src/wit/deps/random/world.wit: -------------------------------------------------------------------------------- 1 | package wasi:random@0.2.0; 2 | 3 | world imports { 4 | import random; 5 | import insecure; 6 | import insecure-seed; 7 | } 8 | -------------------------------------------------------------------------------- /src/wit/deps/sockets/instance-network.wit: -------------------------------------------------------------------------------- 1 | 2 | /// This interface provides a value-export of the default network handle.. 3 | interface instance-network { 4 | use network.{network}; 5 | 6 | /// Get a handle to the default network. 7 | instance-network: func() -> network; 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/wit/deps/sockets/ip-name-lookup.wit: -------------------------------------------------------------------------------- 1 | 2 | interface ip-name-lookup { 3 | use wasi:io/poll@0.2.0.{pollable}; 4 | use network.{network, error-code, ip-address}; 5 | 6 | 7 | /// Resolve an internet host name to a list of IP addresses. 8 | /// 9 | /// Unicode domain names are automatically converted to ASCII using IDNA encoding. 10 | /// If the input is an IP address string, the address is parsed and returned 11 | /// as-is without making any external requests. 12 | /// 13 | /// See the wasi-socket proposal README.md for a comparison with getaddrinfo. 14 | /// 15 | /// This function never blocks. It either immediately fails or immediately 16 | /// returns successfully with a `resolve-address-stream` that can be used 17 | /// to (asynchronously) fetch the results. 18 | /// 19 | /// # Typical errors 20 | /// - `invalid-argument`: `name` is a syntactically invalid domain name or IP address. 21 | /// 22 | /// # References: 23 | /// - 24 | /// - 25 | /// - 26 | /// - 27 | resolve-addresses: func(network: borrow, name: string) -> result; 28 | 29 | resource resolve-address-stream { 30 | /// Returns the next address from the resolver. 31 | /// 32 | /// This function should be called multiple times. On each call, it will 33 | /// return the next address in connection order preference. If all 34 | /// addresses have been exhausted, this function returns `none`. 35 | /// 36 | /// This function never returns IPv4-mapped IPv6 addresses. 37 | /// 38 | /// # Typical errors 39 | /// - `name-unresolvable`: Name does not exist or has no suitable associated IP addresses. (EAI_NONAME, EAI_NODATA, EAI_ADDRFAMILY) 40 | /// - `temporary-resolver-failure`: A temporary failure in name resolution occurred. (EAI_AGAIN) 41 | /// - `permanent-resolver-failure`: A permanent failure in name resolution occurred. (EAI_FAIL) 42 | /// - `would-block`: A result is not available yet. (EWOULDBLOCK, EAGAIN) 43 | resolve-next-address: func() -> result, error-code>; 44 | 45 | /// Create a `pollable` which will resolve once the stream is ready for I/O. 46 | /// 47 | /// Note: this function is here for WASI Preview2 only. 48 | /// It's planned to be removed when `future` is natively supported in Preview3. 49 | subscribe: func() -> pollable; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/wit/deps/sockets/network.wit: -------------------------------------------------------------------------------- 1 | 2 | interface network { 3 | /// An opaque resource that represents access to (a subset of) the network. 4 | /// This enables context-based security for networking. 5 | /// There is no need for this to map 1:1 to a physical network interface. 6 | resource network; 7 | 8 | /// Error codes. 9 | /// 10 | /// In theory, every API can return any error code. 11 | /// In practice, API's typically only return the errors documented per API 12 | /// combined with a couple of errors that are always possible: 13 | /// - `unknown` 14 | /// - `access-denied` 15 | /// - `not-supported` 16 | /// - `out-of-memory` 17 | /// - `concurrency-conflict` 18 | /// 19 | /// See each individual API for what the POSIX equivalents are. They sometimes differ per API. 20 | enum error-code { 21 | /// Unknown error 22 | unknown, 23 | 24 | /// Access denied. 25 | /// 26 | /// POSIX equivalent: EACCES, EPERM 27 | access-denied, 28 | 29 | /// The operation is not supported. 30 | /// 31 | /// POSIX equivalent: EOPNOTSUPP 32 | not-supported, 33 | 34 | /// One of the arguments is invalid. 35 | /// 36 | /// POSIX equivalent: EINVAL 37 | invalid-argument, 38 | 39 | /// Not enough memory to complete the operation. 40 | /// 41 | /// POSIX equivalent: ENOMEM, ENOBUFS, EAI_MEMORY 42 | out-of-memory, 43 | 44 | /// The operation timed out before it could finish completely. 45 | timeout, 46 | 47 | /// This operation is incompatible with another asynchronous operation that is already in progress. 48 | /// 49 | /// POSIX equivalent: EALREADY 50 | concurrency-conflict, 51 | 52 | /// Trying to finish an asynchronous operation that: 53 | /// - has not been started yet, or: 54 | /// - was already finished by a previous `finish-*` call. 55 | /// 56 | /// Note: this is scheduled to be removed when `future`s are natively supported. 57 | not-in-progress, 58 | 59 | /// The operation has been aborted because it could not be completed immediately. 60 | /// 61 | /// Note: this is scheduled to be removed when `future`s are natively supported. 62 | would-block, 63 | 64 | 65 | /// The operation is not valid in the socket's current state. 66 | invalid-state, 67 | 68 | /// A new socket resource could not be created because of a system limit. 69 | new-socket-limit, 70 | 71 | /// A bind operation failed because the provided address is not an address that the `network` can bind to. 72 | address-not-bindable, 73 | 74 | /// A bind operation failed because the provided address is already in use or because there are no ephemeral ports available. 75 | address-in-use, 76 | 77 | /// The remote address is not reachable 78 | remote-unreachable, 79 | 80 | 81 | /// The TCP connection was forcefully rejected 82 | connection-refused, 83 | 84 | /// The TCP connection was reset. 85 | connection-reset, 86 | 87 | /// A TCP connection was aborted. 88 | connection-aborted, 89 | 90 | 91 | /// The size of a datagram sent to a UDP socket exceeded the maximum 92 | /// supported size. 93 | datagram-too-large, 94 | 95 | 96 | /// Name does not exist or has no suitable associated IP addresses. 97 | name-unresolvable, 98 | 99 | /// A temporary failure in name resolution occurred. 100 | temporary-resolver-failure, 101 | 102 | /// A permanent failure in name resolution occurred. 103 | permanent-resolver-failure, 104 | } 105 | 106 | enum ip-address-family { 107 | /// Similar to `AF_INET` in POSIX. 108 | ipv4, 109 | 110 | /// Similar to `AF_INET6` in POSIX. 111 | ipv6, 112 | } 113 | 114 | type ipv4-address = tuple; 115 | type ipv6-address = tuple; 116 | 117 | variant ip-address { 118 | ipv4(ipv4-address), 119 | ipv6(ipv6-address), 120 | } 121 | 122 | record ipv4-socket-address { 123 | /// sin_port 124 | port: u16, 125 | /// sin_addr 126 | address: ipv4-address, 127 | } 128 | 129 | record ipv6-socket-address { 130 | /// sin6_port 131 | port: u16, 132 | /// sin6_flowinfo 133 | flow-info: u32, 134 | /// sin6_addr 135 | address: ipv6-address, 136 | /// sin6_scope_id 137 | scope-id: u32, 138 | } 139 | 140 | variant ip-socket-address { 141 | ipv4(ipv4-socket-address), 142 | ipv6(ipv6-socket-address), 143 | } 144 | 145 | } 146 | -------------------------------------------------------------------------------- /src/wit/deps/sockets/tcp-create-socket.wit: -------------------------------------------------------------------------------- 1 | 2 | interface tcp-create-socket { 3 | use network.{network, error-code, ip-address-family}; 4 | use tcp.{tcp-socket}; 5 | 6 | /// Create a new TCP socket. 7 | /// 8 | /// Similar to `socket(AF_INET or AF_INET6, SOCK_STREAM, IPPROTO_TCP)` in POSIX. 9 | /// On IPv6 sockets, IPV6_V6ONLY is enabled by default and can't be configured otherwise. 10 | /// 11 | /// This function does not require a network capability handle. This is considered to be safe because 12 | /// at time of creation, the socket is not bound to any `network` yet. Up to the moment `bind`/`connect` 13 | /// is called, the socket is effectively an in-memory configuration object, unable to communicate with the outside world. 14 | /// 15 | /// All sockets are non-blocking. Use the wasi-poll interface to block on asynchronous operations. 16 | /// 17 | /// # Typical errors 18 | /// - `not-supported`: The specified `address-family` is not supported. (EAFNOSUPPORT) 19 | /// - `new-socket-limit`: The new socket resource could not be created because of a system limit. (EMFILE, ENFILE) 20 | /// 21 | /// # References 22 | /// - 23 | /// - 24 | /// - 25 | /// - 26 | create-tcp-socket: func(address-family: ip-address-family) -> result; 27 | } 28 | -------------------------------------------------------------------------------- /src/wit/deps/sockets/udp-create-socket.wit: -------------------------------------------------------------------------------- 1 | 2 | interface udp-create-socket { 3 | use network.{network, error-code, ip-address-family}; 4 | use udp.{udp-socket}; 5 | 6 | /// Create a new UDP socket. 7 | /// 8 | /// Similar to `socket(AF_INET or AF_INET6, SOCK_DGRAM, IPPROTO_UDP)` in POSIX. 9 | /// On IPv6 sockets, IPV6_V6ONLY is enabled by default and can't be configured otherwise. 10 | /// 11 | /// This function does not require a network capability handle. This is considered to be safe because 12 | /// at time of creation, the socket is not bound to any `network` yet. Up to the moment `bind` is called, 13 | /// the socket is effectively an in-memory configuration object, unable to communicate with the outside world. 14 | /// 15 | /// All sockets are non-blocking. Use the wasi-poll interface to block on asynchronous operations. 16 | /// 17 | /// # Typical errors 18 | /// - `not-supported`: The specified `address-family` is not supported. (EAFNOSUPPORT) 19 | /// - `new-socket-limit`: The new socket resource could not be created because of a system limit. (EMFILE, ENFILE) 20 | /// 21 | /// # References: 22 | /// - 23 | /// - 24 | /// - 25 | /// - 26 | create-udp-socket: func(address-family: ip-address-family) -> result; 27 | } 28 | -------------------------------------------------------------------------------- /src/wit/deps/sockets/world.wit: -------------------------------------------------------------------------------- 1 | package wasi:sockets@0.2.0; 2 | 3 | world imports { 4 | import instance-network; 5 | import network; 6 | import udp; 7 | import udp-create-socket; 8 | import tcp; 9 | import tcp-create-socket; 10 | import ip-name-lookup; 11 | } 12 | -------------------------------------------------------------------------------- /src/wit/intercept-types.wit: -------------------------------------------------------------------------------- 1 | interface intercept-types { 2 | // Plugin context 3 | record context { 4 | auth: option>, 5 | connection: connection-ctx, 6 | event: event-ctx, 7 | } 8 | 9 | variant connection-ctx { 10 | websocket(websocket), 11 | } 12 | 13 | variant event-ctx { 14 | kafka(kafka-event-ctx), 15 | counter(counter-event-ctx), 16 | } 17 | 18 | record counter-event-ctx { 19 | source-id: string, 20 | count: u64, 21 | } 22 | 23 | record kafka-event-ctx { 24 | payload: option>, 25 | source-id: string, 26 | topic: string, 27 | timestamp: option, 28 | partition: u32, 29 | offset: u64, 30 | } 31 | 32 | record websocket { 33 | addr: option, 34 | } 35 | 36 | variant transformed-payload { 37 | kafka(option>), 38 | counter(u64), 39 | } 40 | 41 | variant action { 42 | forward, 43 | discard, 44 | transform(transformed-payload), 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/wit/world.wit: -------------------------------------------------------------------------------- 1 | package kiwi:kiwi@0.1.0; 2 | 3 | world intercept-hook { 4 | use intercept-types.{context, action}; 5 | 6 | export intercept: func(ctx: context) -> action; 7 | } 8 | 9 | world authenticate-hook { 10 | use authenticate-types.{outcome, http-request}; 11 | 12 | import wasi:http/outgoing-handler@0.2.0; 13 | 14 | export authenticate: func(incoming: http-request) -> outcome; 15 | } 16 | 17 | world internal { 18 | use wasi:http/types@0.2.0.{method, scheme, field-key, field-value}; 19 | 20 | import wasi:http/outgoing-handler@0.2.0; 21 | } 22 | --------------------------------------------------------------------------------