├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ ├── build.yml │ ├── jekyll-gh-pages.yml │ ├── release.yml │ └── update-homebrew.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── CHANGELOG.md ├── Dockerfile ├── Dockerfile.goreleaser ├── Makefile ├── README.md ├── cmd └── server │ └── main.go ├── config ├── config.go └── config_test.go ├── go.mod ├── go.sum ├── kafka ├── client.go ├── client_test.go ├── interface.go └── resource_types.go ├── mcp ├── prompts.go ├── resources.go ├── response_helpers.go ├── server.go └── tools.go ├── plan.md ├── prompts.md ├── release.config.js ├── renovate.json ├── resources.md ├── roots.md └── tools.md /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: ["tuannvm"] 4 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | # Assign pull requests to specific users 9 | assignees: 10 | - "tuannvm" 11 | # Labels to apply to pull requests 12 | labels: 13 | - "ci/cd" 14 | - "dependencies" 15 | # Create a group of updates instead of individual PRs 16 | groups: 17 | github-actions: 18 | patterns: 19 | - "*" 20 | 21 | # Maintain dependencies for Go 22 | - package-ecosystem: "gomod" 23 | directory: "/" 24 | schedule: 25 | interval: "weekly" 26 | assignees: 27 | - "tuannvm" 28 | labels: 29 | - "dependencies" 30 | - "go" 31 | # Limit PRs for gomod to security updates and major version bumps 32 | open-pull-requests-limit: 10 33 | 34 | # Maintain dependencies for Docker 35 | - package-ecosystem: "docker" 36 | directory: "/" 37 | schedule: 38 | interval: "monthly" 39 | assignees: 40 | - "tuannvm" 41 | labels: 42 | - "dependencies" 43 | - "docker" 44 | # Block PRs for major version bumps 45 | ignore: 46 | - dependency-name: "*" 47 | update-types: ["version-update:semver-major"] -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build & Verify Pipeline 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | paths-ignore: 7 | - "**.md" 8 | - ".github/ISSUE_TEMPLATE/**" 9 | - ".gitignore" 10 | pull_request: 11 | paths-ignore: 12 | - "**.md" 13 | - ".github/ISSUE_TEMPLATE/**" 14 | - ".gitignore" 15 | 16 | permissions: 17 | contents: read 18 | packages: write 19 | id-token: write # Required for SLSA provenance 20 | security-events: write # Required for uploading security results 21 | pull-requests: read 22 | 23 | env: 24 | GO_VERSION: "1.24" 25 | REGISTRY: ghcr.io 26 | 27 | jobs: 28 | # Static analysis and code quality check 29 | verify: 30 | name: Code Quality 31 | runs-on: ubuntu-latest 32 | steps: 33 | - name: Checkout code 34 | uses: actions/checkout@v4 35 | with: 36 | fetch-depth: 0 37 | persist-credentials: false 38 | 39 | - name: Set up Go 40 | uses: actions/setup-go@v5 41 | with: 42 | go-version: ${{ env.GO_VERSION }} 43 | cache: true 44 | check-latest: true 45 | 46 | - name: Install dependencies 47 | run: | 48 | go mod download 49 | go mod verify 50 | 51 | - name: Check Go mod tidy 52 | run: | 53 | go mod tidy 54 | if ! git diff --quiet go.mod go.sum; then 55 | echo "go.mod or go.sum is not tidy, run 'go mod tidy'" 56 | git diff go.mod go.sum 57 | exit 1 58 | fi 59 | 60 | - name: Install golangci-lint 61 | uses: golangci/golangci-lint-action@v7 62 | with: 63 | version: latest 64 | args: --timeout=5m 65 | install-mode: binary 66 | skip-pkg-cache: true 67 | skip-build-cache: true 68 | 69 | - name: Run linters 70 | run: golangci-lint run 71 | 72 | # Security vulnerability scanning and SBOM generation 73 | security: 74 | name: Security Scan 75 | runs-on: ubuntu-latest 76 | needs: verify 77 | steps: 78 | - name: Checkout code 79 | uses: actions/checkout@v4 80 | with: 81 | persist-credentials: false 82 | 83 | - name: Set up Go 84 | uses: actions/setup-go@v5 85 | with: 86 | go-version: ${{ env.GO_VERSION }} 87 | cache: true 88 | 89 | - name: Run Go Vulnerability Check 90 | run: | 91 | go install golang.org/x/vuln/cmd/govulncheck@latest 92 | govulncheck ./... 93 | 94 | - name: Run dependency scan 95 | uses: aquasecurity/trivy-action@0.30.0 96 | with: 97 | scan-type: "fs" 98 | scan-ref: "." 99 | format: "sarif" 100 | output: "trivy-results.sarif" 101 | severity: "CRITICAL,HIGH,MEDIUM" 102 | timeout: "10m" 103 | 104 | - name: Upload security scan results 105 | uses: github/codeql-action/upload-sarif@v3 106 | if: always() 107 | with: 108 | sarif_file: "trivy-results.sarif" 109 | 110 | - name: Generate SBOM 111 | uses: anchore/sbom-action@v0.18.0 112 | with: 113 | format: spdx-json 114 | output-file: sbom.spdx.json 115 | 116 | - name: Upload SBOM 117 | uses: actions/upload-artifact@v4 118 | with: 119 | name: sbom 120 | path: sbom.spdx.json 121 | retention-days: 30 122 | 123 | # Run unit and integration tests with code coverage 124 | test: 125 | name: Run Tests 126 | runs-on: ubuntu-latest 127 | needs: verify 128 | steps: 129 | - name: Checkout code 130 | uses: actions/checkout@v4 131 | with: 132 | persist-credentials: false 133 | 134 | - name: Set up Go 135 | uses: actions/setup-go@v5 136 | with: 137 | go-version: ${{ env.GO_VERSION }} 138 | cache: true 139 | 140 | - name: Run tests 141 | run: go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... 142 | 143 | - name: Upload coverage 144 | uses: codecov/codecov-action@v5 145 | with: 146 | file: ./coverage.txt 147 | flags: unittests 148 | fail_ci_if_error: false 149 | 150 | # Simple build verification (for PRs and non-main branches) 151 | build: 152 | name: Build Verification 153 | runs-on: ubuntu-latest 154 | needs: [verify, security] 155 | # Only run for PRs or pushes to non-main branches 156 | if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref != 'refs/heads/main') 157 | steps: 158 | - name: Checkout code 159 | uses: actions/checkout@v4 160 | with: 161 | persist-credentials: false 162 | 163 | - name: Set up Go 164 | uses: actions/setup-go@v5 165 | with: 166 | go-version: ${{ env.GO_VERSION }} 167 | cache: true 168 | 169 | - name: Build 170 | run: go build -v ./... 171 | -------------------------------------------------------------------------------- /.github/workflows/jekyll-gh-pages.yml: -------------------------------------------------------------------------------- 1 | # Sample workflow for building and deploying a Jekyll site to GitHub Pages 2 | name: Deploy Jekyll with GitHub Pages dependencies preinstalled 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | push: 7 | branches: ["main"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: "pages" 22 | cancel-in-progress: false 23 | 24 | jobs: 25 | # Build job 26 | build: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v4 31 | - name: Setup Pages 32 | uses: actions/configure-pages@v5 33 | - name: Build with Jekyll 34 | uses: actions/jekyll-build-pages@v1 35 | with: 36 | source: ./ 37 | destination: ./_site 38 | - name: Upload artifact 39 | uses: actions/upload-pages-artifact@v3 40 | 41 | # Deployment job 42 | deploy: 43 | environment: 44 | name: github-pages 45 | url: ${{ steps.deployment.outputs.page_url }} 46 | runs-on: ubuntu-latest 47 | needs: build 48 | steps: 49 | - name: Deploy to GitHub Pages 50 | id: deployment 51 | uses: actions/deploy-pages@v4 52 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release Pipeline 2 | 3 | on: 4 | workflow_run: 5 | workflows: ["Build & Verify Pipeline"] 6 | branches: [main] 7 | types: 8 | - completed 9 | 10 | permissions: 11 | contents: write 12 | packages: write 13 | issues: write 14 | pull-requests: write 15 | 16 | jobs: 17 | semantic-release: 18 | name: Semantic Release 19 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 20 | runs-on: ubuntu-latest 21 | outputs: 22 | version: ${{ steps.extract_version.outputs.version }} 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v4 26 | with: 27 | fetch-depth: 0 28 | token: ${{ secrets.GITHUB_TOKEN }} 29 | 30 | - name: Setup Node.js 31 | uses: actions/setup-node@v4 32 | with: 33 | node-version: "lts/*" 34 | 35 | - name: Install semantic-release 36 | run: | 37 | npm install -g semantic-release @semantic-release/changelog @semantic-release/git @semantic-release/github 38 | 39 | - name: Set up Go 40 | uses: actions/setup-go@v5 41 | with: 42 | go-version: ${{ env.GO_VERSION }} 43 | cache: true 44 | 45 | - name: Login to GitHub Container Registry 46 | uses: docker/login-action@v3 47 | with: 48 | registry: ${{ env.REGISTRY }} 49 | username: ${{ github.actor }} 50 | password: ${{ secrets.GITHUB_TOKEN }} 51 | 52 | - name: Run semantic-release 53 | id: semantic 54 | env: 55 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 56 | run: | 57 | semantic-release 58 | 59 | - name: Extract version from git tag 60 | id: extract_version 61 | if: steps.semantic.outcome == 'success' 62 | run: | 63 | VERSION=$(git describe --tags --abbrev=0 --always) 64 | VERSION=${VERSION#v} 65 | echo "version=$VERSION" >> $GITHUB_OUTPUT 66 | echo "Released version: $VERSION" 67 | # Dynamically set the tag name for use in later steps 68 | TAG_NAME="v$VERSION" 69 | echo "tag_name=$TAG_NAME" >> $GITHUB_ENV 70 | 71 | - name: Validate Git Tag 72 | run: | 73 | TAG_NAME=${{ env.TAG_NAME }} 74 | EXISTING_TAG=$(git tag --points-at ${GITHUB_SHA}) 75 | if [ "$EXISTING_TAG" != "$TAG_NAME" ]; then 76 | git tag -f $TAG_NAME ${GITHUB_SHA} 77 | git push origin $TAG_NAME --force 78 | fi 79 | 80 | - name: Run GoReleaser 81 | uses: goreleaser/goreleaser-action@v6 82 | if: steps.extract_version.outcome == 'success' 83 | with: 84 | version: v1.18.2 85 | distribution: goreleaser 86 | args: release --clean 87 | env: 88 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 89 | GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} 90 | GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} 91 | GORELEASER_CURRENT_TAG: v${{ steps.extract_version.outputs.version }} 92 | 93 | 94 | # Build and publish Docker image with semantic version tags 95 | publish-docker-image: 96 | name: Publish Docker Image 97 | needs: semantic-release 98 | if: needs.semantic-release.outputs.version != '' 99 | runs-on: ubuntu-latest 100 | steps: 101 | - name: Checkout code 102 | uses: actions/checkout@v4 103 | with: 104 | fetch-depth: 0 105 | 106 | - name: Set up Go 107 | uses: actions/setup-go@v5 108 | with: 109 | go-version: ${{ env.GO_VERSION }} 110 | cache: true 111 | 112 | - name: Docker meta 113 | id: meta 114 | uses: docker/metadata-action@v5 115 | with: 116 | images: ${{ env.REGISTRY }}/${{ github.repository }} 117 | tags: | 118 | type=semver,pattern={{version}},value=v${{ needs.semantic-release.outputs.version }} 119 | type=semver,pattern={{major}}.{{minor}},value=v${{ needs.semantic-release.outputs.version }} 120 | type=semver,pattern={{major}},value=v${{ needs.semantic-release.outputs.version }} 121 | latest 122 | 123 | - name: Set up Docker Buildx 124 | uses: docker/setup-buildx-action@v3 125 | 126 | - name: Login to GitHub Container Registry 127 | uses: docker/login-action@v3 128 | with: 129 | registry: ${{ env.REGISTRY }} 130 | username: ${{ github.actor }} 131 | password: ${{ secrets.GITHUB_TOKEN }} 132 | 133 | - name: Set up QEMU 134 | uses: docker/setup-qemu-action@v3 135 | with: 136 | platforms: 'arm64,amd64,arm' 137 | 138 | - name: Build and push Docker image 139 | uses: docker/build-push-action@v6 140 | with: 141 | context: . 142 | push: true 143 | platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 144 | tags: ${{ steps.meta.outputs.tags }} 145 | labels: ${{ steps.meta.outputs.labels }} 146 | cache-from: type=gha 147 | cache-to: type=gha,mode=max 148 | provenance: true 149 | sbom: true 150 | 151 | env: 152 | GO_VERSION: "1.24" 153 | REGISTRY: ghcr.io 154 | -------------------------------------------------------------------------------- /.github/workflows/update-homebrew.yml: -------------------------------------------------------------------------------- 1 | name: Update Homebrew Formula on Release 2 | 3 | on: 4 | workflow_run: 5 | workflows: ["Release Pipeline"] 6 | types: [completed] 7 | branches: [main] 8 | 9 | jobs: 10 | trigger-homebrew-update: 11 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Set repo name 15 | id: repo-name 16 | run: echo "REPO_NAME=$(echo ${{ github.repository }} | cut -d '/' -f 2)" >> $GITHUB_OUTPUT 17 | 18 | - name: Trigger homebrew formula update 19 | uses: peter-evans/repository-dispatch@v3 20 | with: 21 | token: ${{ secrets.HOMEBREW_TAP_TOKEN }} 22 | repository: tuannvm/homebrew-mcp 23 | event-type: update-formula 24 | client-payload: '{"repository": "${{ github.repository }}", "formula": "${{ steps.repo-name.outputs.REPO_NAME }}.rb"}' 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries 2 | kafka-mcp-server 3 | 4 | # Go dependencies 5 | vendor/ 6 | 7 | # Environment variables file 8 | .env 9 | 10 | # OS generated files 11 | .DS_Store 12 | *.pem 13 | *.crt 14 | *.key 15 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # Define the configuration version 2 | version: "2" 3 | 4 | run: 5 | timeout: 5m 6 | modules-download-mode: readonly 7 | 8 | linters: 9 | default: standard -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # GoReleaser configuration 2 | before: 3 | hooks: 4 | - go mod tidy 5 | 6 | builds: 7 | - main: ./cmd/server/main.go 8 | binary: "{{ .ProjectName }}" 9 | env: 10 | - CGO_ENABLED=0 11 | goos: 12 | - linux 13 | - darwin 14 | - windows 15 | goarch: 16 | - amd64 17 | - arm64 18 | - arm 19 | ldflags: 20 | - -s -w -X main.Version={{.Version}} 21 | 22 | archives: 23 | - format_overrides: 24 | - goos: windows 25 | format: zip 26 | name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" 27 | 28 | checksum: 29 | name_template: "checksums.txt" 30 | 31 | # Use simpler snapshot naming to ensure compatibility 32 | snapshot: 33 | name_template: "next" 34 | 35 | changelog: 36 | sort: asc 37 | filters: 38 | exclude: 39 | - "^docs:" 40 | - "^test:" 41 | - "^chore:" 42 | - "^ci:" 43 | - Merge pull request 44 | - Merge branch 45 | 46 | # Explicitly configure GitHub Releases 47 | release: 48 | github: 49 | owner: "{{.Env.GITHUB_REPOSITORY_OWNER}}" 50 | name: "{{.Env.GITHUB_REPOSITORY_NAME}}" 51 | draft: false 52 | prerelease: auto 53 | name_template: "{{.ProjectName}} v{{.Version}}" 54 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [1.0.3](https://github.com/tuannvm/kafka-mcp-server/compare/v1.0.2...v1.0.3) (2025-05-03) 2 | 3 | ## [1.0.2](https://github.com/tuannvm/kafka-mcp-server/compare/v1.0.1...v1.0.2) (2025-04-25) 4 | 5 | # 1.0.0 (2025-04-20) 6 | 7 | 8 | ### Features 9 | 10 | * enhance CI workflow and add Docker support ([5dee284](https://github.com/tuannvm/kafka-mcp-server/commit/5dee284d3f9f7de450d8413d86bd6cb690b06127)) 11 | * **init:** setup initial project with Go server and Kafka integration ([75a6dfc](https://github.com/tuannvm/kafka-mcp-server/commit/75a6dfc06a4bb04b17549fb4f735a33d5753cfcc)) 12 | * **kafka:** add KafkaClient interface and utilities for MCP server ([d1c5e5b](https://github.com/tuannvm/kafka-mcp-server/commit/d1c5e5b9ff580acab0b1565eabf6692e8946c455)) 13 | * **kafka:** add ListBrokers method for broker retrieval ([55c5c0c](https://github.com/tuannvm/kafka-mcp-server/commit/55c5c0c49d47275313fdb406ab3cbea4b753f531)) 14 | * **makefile:** add test-no-kafka target ([8efd98c](https://github.com/tuannvm/kafka-mcp-server/commit/8efd98ce745e130e2a7bdf0c858f574fc3dbc4d8)) 15 | * **server:** add RegisterPrompts to server initialization ([ab8a9c8](https://github.com/tuannvm/kafka-mcp-server/commit/ab8a9c8b905be8c21e649d2221ff9b61dfe55df7)) 16 | * **server:** implement Kafka MCP server with CLI and tools integration ([6bc3d34](https://github.com/tuannvm/kafka-mcp-server/commit/6bc3d34bc384d7f422fd4235e45db072bab8877d)) 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the official Golang image to create a build artifact. 2 | # This is the builder stage. 3 | FROM golang:1.24-alpine AS builder 4 | 5 | # Set the Current Working Directory inside the container 6 | WORKDIR /app 7 | 8 | # Copy go mod and sum files 9 | COPY go.mod go.sum ./ 10 | 11 | # Download all dependencies. Dependencies will be cached if the go.mod and go.sum files are not changed 12 | RUN go mod download 13 | 14 | # Copy the source code into the container 15 | COPY . . 16 | 17 | # Build the Go app 18 | # -ldflags="-w -s" reduces the size of the binary by removing debug information. 19 | # CGO_ENABLED=0 disables CGO for static linking, useful for alpine base images. 20 | RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o /kafka-mcp-server ./cmd/server 21 | 22 | # --- Start final stage --- # 23 | 24 | # Use a minimal base image like Alpine Linux 25 | FROM alpine:latest 26 | 27 | # Add ca-certificates in case TLS connections need system CAs 28 | RUN apk --no-cache add ca-certificates 29 | 30 | # Set the Current Working Directory inside the container 31 | WORKDIR /app 32 | 33 | # Copy the built binary from the builder stage 34 | COPY --from=builder /kafka-mcp-server . 35 | 36 | # Expose ports if using HTTP transport in the future (optional for stdio) 37 | # EXPOSE 8080 38 | 39 | # Command to run the executable 40 | # The server reads configuration from environment variables. 41 | ENTRYPOINT ["/app/kafka-mcp-server"] 42 | -------------------------------------------------------------------------------- /Dockerfile.goreleaser: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | RUN apk --no-cache add ca-certificates 4 | 5 | WORKDIR /root/ 6 | 7 | # Copy the pre-built binary file from the builder stage 8 | COPY kafka-mcp-server . 9 | 10 | # Default environment variables 11 | ENV KAFKA_BROKERS="localhost:9092" 12 | ENV KAFKA_CLIENT_ID="kafka-mcp-server" 13 | ENV KAFKA_SASL_MECHANISM="" 14 | ENV KAFKA_SASL_USER="" 15 | ENV KAFKA_SASL_PASSWORD="" 16 | ENV TLS_ENABLE="false" 17 | ENV TLS_INSECURE_SKIP_VERIFY="true" 18 | ENV MCP_TRANSPORT="stdio" 19 | ENV MCP_PORT="9097" 20 | 21 | # Expose the port 22 | EXPOSE ${MCP_PORT} 23 | 24 | # Run the application 25 | ENTRYPOINT ["./kafka-mcp-server"] 26 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build test clean run-dev release-snapshot run-docker run docker-compose-up docker-compose-down lint 2 | 3 | # Variables 4 | BINARY_NAME=kafka-mcp-server 5 | VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") 6 | BUILD_DIR=bin 7 | 8 | # Build the application 9 | build: 10 | mkdir -p $(BUILD_DIR) 11 | go build -ldflags "-X main.Version=$(VERSION)" -o $(BUILD_DIR)/$(BINARY_NAME) ./cmd/server 12 | 13 | # Run tests 14 | test: 15 | go test ./... 16 | 17 | # Run tests without Kafka container (skips integration tests) 18 | test-no-kafka: 19 | SKIP_KAFKA_TESTS=true go test ./... 20 | 21 | # Clean build artifacts 22 | clean: 23 | rm -rf $(BUILD_DIR) 24 | 25 | # Run the application in development mode 26 | run-dev: 27 | go run cmd/server/main.go 28 | 29 | # Create a release snapshot using GoReleaser 30 | release-snapshot: 31 | goreleaser release --snapshot --clean 32 | 33 | # Run the application using the built binary 34 | run: 35 | ./$(BUILD_DIR)/$(BINARY_NAME) 36 | 37 | # Build and run Docker image 38 | run-docker: build 39 | docker build -t $(BINARY_NAME):$(VERSION) . 40 | docker run -p 9097:9097 $(BINARY_NAME):$(VERSION) 41 | 42 | # Start the application with Docker Compose 43 | docker-compose-up: 44 | docker-compose up -d 45 | 46 | # Stop Docker Compose services 47 | docker-compose-down: 48 | docker-compose down 49 | 50 | # Run linting checks (same as CI) 51 | lint: 52 | @echo "Running linters..." 53 | @go mod tidy 54 | @if ! git diff --quiet go.mod go.sum; then echo "go.mod or go.sum is not tidy, run 'go mod tidy'"; git diff go.mod go.sum; exit 1; fi 55 | @if ! command -v golangci-lint &> /dev/null; then echo "Installing golangci-lint..." && go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; fi 56 | @golangci-lint run --timeout=5m 57 | 58 | # Default target 59 | all: clean build 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka MCP Server 2 | 3 | A Model Context Protocol (MCP) server for Apache Kafka implemented in Go, leveraging [franz-go](https://github.com/twmb/franz-go) and [mcp-go](https://github.com/mark3labs/mcp-go). 4 | 5 | This server provides an implementation for interacting with Kafka via the MCP protocol, enabling LLM models to perform common Kafka operations through a standardized interface. 6 | 7 | [![Go Report Card](https://goreportcard.com/badge/github.com/tuannvm/kafka-mcp-server)](https://goreportcard.com/report/github.com/tuannvm/kafka-mcp-server) 8 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/tuannvm/kafka-mcp-server/build.yml?branch=main&label=CI%2FCD&logo=github)](https://github.com/tuannvm/kafka-mcp-server/actions/workflows/build.yml) 9 | [![Go Version](https://img.shields.io/github/go-mod/go-version/tuannvm/kafka-mcp-server?logo=go)](https://github.com/tuannvm/kafka-mcp-server/blob/main/go.mod) 10 | [![Trivy Scan](https://img.shields.io/github/actions/workflow/status/tuannvm/kafka-mcp-server/build.yml?branch=main&label=Trivy%20Security%20Scan&logo=aquasec)](https://github.com/tuannvm/kafka-mcp-server/actions/workflows/build.yml) 11 | [![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev) 12 | [![Go Reference](https://pkg.go.dev/badge/github.com/tuannvm/kafka-mcp-server.svg)](https://pkg.go.dev/github.com/tuannvm/kafka-mcp-server) 13 | [![Docker Image](https://img.shields.io/github/v/release/tuannvm/kafka-mcp-server?sort=semver&label=GHCR&logo=docker)](https://github.com/tuannvm/kafka-mcp-server/pkgs/container/kafka-mcp-server) 14 | [![GitHub Release](https://img.shields.io/github/v/release/tuannvm/kafka-mcp-server?sort=semver)](https://github.com/tuannvm/kafka-mcp-server/releases/latest) 15 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 16 | 17 | ## Overview 18 | 19 | The Kafka MCP Server bridges the gap between LLM models and Apache Kafka, allowing them to: 20 | 21 | - Produce and consume messages from topics 22 | - List, describe, and manage topics 23 | - Monitor and manage consumer groups 24 | - Assess cluster health and configuration 25 | - Execute standard Kafka operations 26 | 27 | All through the standardized Model Context Protocol (MCP). 28 | 29 | ![Tools](https://github.com/user-attachments/assets/c70e6ac6-0657-4c7e-814e-ecb18ab8c6ec) 30 | 31 | ![Prompts & Resources](https://github.com/user-attachments/assets/dd5f3165-200f-41ca-bd0a-4b02063a9c57) 32 | 33 | ## Key Features 34 | 35 | - **Kafka Integration**: Implementation of common Kafka operations via MCP 36 | - **Security**: Support for SASL (PLAIN, SCRAM-SHA-256, SCRAM-SHA-512) and TLS authentication 37 | - **Error Handling**: Error handling with meaningful feedback 38 | - **Configuration Options**: Customizable for different environments 39 | - **Pre-Configured Prompts**: Set of prompts for common Kafka operations 40 | - **Compatibility**: Works with MCP-compatible LLM models 41 | 42 | ## Getting Started 43 | 44 | ### Prerequisites 45 | 46 | - Go 1.21 or later 47 | - Docker (for running integration tests) 48 | - Access to a Kafka cluster 49 | 50 | ### Installation 51 | 52 | #### Homebrew (macOS and Linux) 53 | 54 | The easiest way to install kafka-mcp-server is using Homebrew: 55 | 56 | ```bash 57 | # Add the tap repository 58 | brew tap tuannvm/mcp 59 | 60 | # Install kafka-mcp-server 61 | brew install kafka-mcp-server 62 | ``` 63 | 64 | To update to the latest version: 65 | 66 | ```bash 67 | brew update && brew upgrade kafka-mcp-server 68 | ``` 69 | 70 | #### From Source 71 | 72 | ```bash 73 | # Clone the repository 74 | git clone https://github.com/tuannvm/kafka-mcp-server.git 75 | cd kafka-mcp-server 76 | 77 | # Build the server 78 | go build -o kafka-mcp-server ./cmd/server 79 | ``` 80 | 81 | ### MCP Client Integration 82 | 83 | This MCP server can be integrated with several AI applications: 84 | 85 | #### Basic Configuration 86 | 87 | To integrate with MCP-compatible clients, add this configuration to your client's settings: 88 | 89 | ```json 90 | { 91 | "mcpServers": { 92 | "kafka": { 93 | "command": "kafka-mcp-server", 94 | "env": { 95 | "KAFKA_BROKERS": "localhost:9092", 96 | "KAFKA_CLIENT_ID": "kafka-mcp-server", 97 | "MCP_TRANSPORT": "stdio", 98 | "KAFKA_SASL_MECHANISM": "", 99 | "KAFKA_SASL_USER": "", 100 | "KAFKA_SASL_PASSWORD": "", 101 | "KAFKA_TLS_ENABLE": "false", 102 | "KAFKA_TLS_INSECURE_SKIP_VERIFY": "false" 103 | } 104 | } 105 | } 106 | } 107 | ``` 108 | 109 | For secured environments: 110 | 111 | ```json 112 | { 113 | "mcpServers": { 114 | "kafka": { 115 | "command": "kafka-mcp-server", 116 | "env": { 117 | "KAFKA_BROKERS": "kafka-broker-1:9092,kafka-broker-2:9092", 118 | "KAFKA_CLIENT_ID": "kafka-mcp-server", 119 | "MCP_TRANSPORT": "stdio", 120 | "KAFKA_SASL_MECHANISM": "scram-sha-512", 121 | "KAFKA_SASL_USER": "kafka-user", 122 | "KAFKA_SASL_PASSWORD": "kafka-password", 123 | "KAFKA_TLS_ENABLE": "true", 124 | "KAFKA_TLS_INSECURE_SKIP_VERIFY": "false" 125 | } 126 | } 127 | } 128 | } 129 | ``` 130 | 131 | #### Using Docker Image 132 | 133 | To use the Docker image instead of a local binary: 134 | 135 | ```json 136 | { 137 | "mcpServers": { 138 | "kafka": { 139 | "command": "docker", 140 | "args": [ 141 | "run", 142 | "--rm", 143 | "-i", 144 | "-e", "KAFKA_BROKERS=kafka-broker:9092", 145 | "-e", "KAFKA_CLIENT_ID=kafka-mcp-server", 146 | "-e", "MCP_TRANSPORT=stdio", 147 | "-e", "KAFKA_SASL_MECHANISM=", 148 | "-e", "KAFKA_SASL_USER=", 149 | "-e", "KAFKA_SASL_PASSWORD=", 150 | "-e", "KAFKA_TLS_ENABLE=false", 151 | "ghcr.io/tuannvm/kafka-mcp-server:latest" 152 | ], 153 | "env": {} 154 | } 155 | } 156 | } 157 | ``` 158 | 159 | > **Note**: If connecting to Kafka running on your host machine from Docker, use `host.docker.internal` as the broker address on macOS and Windows. For Linux, use `--network=host` in your Docker run command or the host's actual IP address. 160 | 161 | #### Cursor 162 | 163 | To use with [Cursor](https://cursor.sh/), create or edit `~/.cursor/mcp.json`: 164 | 165 | ```json 166 | { 167 | "mcpServers": { 168 | "kafka": { 169 | "command": "kafka-mcp-server", 170 | "args": [], 171 | "env": { 172 | "KAFKA_BROKERS": "localhost:9092", 173 | "KAFKA_CLIENT_ID": "kafka-mcp-server", 174 | "MCP_TRANSPORT": "stdio", 175 | "KAFKA_SASL_MECHANISM": "", 176 | "KAFKA_SASL_USER": "", 177 | "KAFKA_SASL_PASSWORD": "", 178 | "KAFKA_TLS_ENABLE": "false" 179 | } 180 | } 181 | } 182 | } 183 | ``` 184 | 185 | For HTTP+SSE transport mode (if supported): 186 | 187 | ```json 188 | { 189 | "mcpServers": { 190 | "kafka-http": { 191 | "url": "http://localhost:9097/sse" 192 | } 193 | } 194 | } 195 | ``` 196 | 197 | Then start the server in a separate terminal with: 198 | 199 | ```bash 200 | MCP_TRANSPORT=http KAFKA_BROKERS=localhost:9092 kafka-mcp-server 201 | ``` 202 | 203 | #### Claude Desktop 204 | 205 | To use with [Claude Desktop](https://claude.ai/desktop), edit your Claude configuration file: 206 | - macOS: `~/Library/Application Support/Claude/claude_desktop_config.json` 207 | - Windows: `%APPDATA%\Claude\claude_desktop_config.json` 208 | 209 | ```json 210 | { 211 | "mcpServers": { 212 | "kafka": { 213 | "command": "kafka-mcp-server", 214 | "args": [], 215 | "env": { 216 | "KAFKA_BROKERS": "localhost:9092", 217 | "KAFKA_CLIENT_ID": "kafka-mcp-server", 218 | "MCP_TRANSPORT": "stdio", 219 | "KAFKA_SASL_MECHANISM": "", 220 | "KAFKA_SASL_USER": "", 221 | "KAFKA_SASL_PASSWORD": "", 222 | "KAFKA_TLS_ENABLE": "false" 223 | } 224 | } 225 | } 226 | } 227 | ``` 228 | 229 | After updating the configuration, restart Claude Desktop. You should see the Kafka MCP tools available in the tools menu. 230 | 231 | #### Windsurf 232 | 233 | To use with [Windsurf](https://windsurf.com/refer?referral_code=sjqdvqozgx2wyi7r), create or edit your `mcp_config.json`: 234 | 235 | ```json 236 | { 237 | "mcpServers": { 238 | "kafka": { 239 | "command": "kafka-mcp-server", 240 | "args": [], 241 | "env": { 242 | "KAFKA_BROKERS": "localhost:9092", 243 | "KAFKA_CLIENT_ID": "kafka-mcp-server", 244 | "MCP_TRANSPORT": "stdio", 245 | "KAFKA_SASL_MECHANISM": "", 246 | "KAFKA_SASL_USER": "", 247 | "KAFKA_SASL_PASSWORD": "", 248 | "KAFKA_TLS_ENABLE": "false" 249 | } 250 | } 251 | } 252 | } 253 | ``` 254 | 255 | Restart Windsurf to apply the changes. The Kafka MCP tools will be available to the AI assistant. 256 | 257 | #### ChatWise 258 | 259 | To use with [ChatWise](https://chatwise.app?atp=uo1wzc), follow these steps: 260 | 261 | 1. Open ChatWise and go to Settings 262 | 2. Navigate to the Tools section 263 | 3. Click the "+" icon to add a new tool 264 | 4. Select "Command Line MCP" 265 | 5. Configure with the following details: 266 | - ID: `kafka` (or any name you prefer) 267 | - Command: `kafka-mcp-server` 268 | - Args: (leave empty) 269 | - Env: Add the following environment variables: 270 | ``` 271 | KAFKA_BROKERS=localhost:9092 272 | KAFKA_CLIENT_ID=kafka-mcp-server 273 | MCP_TRANSPORT=stdio 274 | KAFKA_SASL_MECHANISM= 275 | KAFKA_SASL_USER= 276 | KAFKA_SASL_PASSWORD= 277 | KAFKA_TLS_ENABLE=false 278 | ``` 279 | 280 | Alternatively, you can import the configuration using the import option. 281 | 282 | ## Simplify Configuration with mcpenetes 283 | 284 | Managing MCP server configurations across multiple clients can become challenging. [mcpenetes](https://github.com/tuannvm/mcpenetes/) is a dedicated tool that makes this process significantly easier: 285 | 286 | ```bash 287 | # Install mcpenetes 288 | go install github.com/tuannvm/mcpenetes@latest 289 | ``` 290 | 291 | ### Key Features 292 | 293 | - **Interactive Search**: Find and select Kafka MCP server configurations with a simple command 294 | - **Apply Everywhere**: Automatically sync configurations across all your MCP clients 295 | - **Configuration Backup**: Safely backup existing configurations before making changes 296 | - **Restore**: Easily revert to previous configurations if needed 297 | 298 | ### Quick Start with mcpenetes 299 | 300 | ```bash 301 | # Search for available MCP servers including kafka-mcp-server 302 | mcpenetes search 303 | 304 | # Apply kafka-mcp-server configuration to all your clients at once 305 | mcpenetes apply 306 | 307 | # Load a configuration from your clipboard 308 | mcpenetes load 309 | ``` 310 | 311 | With mcpenetes, you can maintain multiple Kafka configurations (development, production, etc.) and switch between them instantly across all your clients (Cursor, Claude Desktop, Windsurf, ChatWise) without manually editing each client's configuration files. 312 | 313 | ## MCP Tools 314 | 315 | The server exposes tools for Kafka interaction: 316 | 317 | ### produce_message 318 | 319 | Produces a single message to a Kafka topic with optional key. 320 | 321 | **Sample Prompt:** 322 | > "Send a new order update to the orders topic with order ID 12345." 323 | 324 | **Example:** 325 | ```json 326 | { 327 | "topic": "orders", 328 | "key": "12345", 329 | "value": "{\"order_id\":\"12345\",\"status\":\"shipped\"}" 330 | } 331 | ``` 332 | 333 | **Response:** 334 | ```json 335 | "Message produced successfully to topic orders" 336 | ``` 337 | 338 | ### consume_messages 339 | 340 | Consumes a batch of messages from one or more Kafka topics. 341 | 342 | **Sample Prompt:** 343 | > "Retrieve the latest messages from the customer-events topic so I can see recent customer activity." 344 | 345 | **Example:** 346 | ```json 347 | { 348 | "topics": ["customer-events"], 349 | "max_messages": 5 350 | } 351 | ``` 352 | 353 | **Response:** 354 | ```json 355 | [ 356 | { 357 | "topic": "customer-events", 358 | "partition": 0, 359 | "offset": 1042, 360 | "timestamp": 1650123456789, 361 | "key": "customer-123", 362 | "value": "{\"customer_id\":\"123\",\"action\":\"login\",\"timestamp\":\"2023-04-16T12:34:56Z\"}" 363 | }, 364 | // Additional messages... 365 | ] 366 | ``` 367 | 368 | ### list_brokers 369 | 370 | Lists the configured Kafka broker addresses the server is connected to. 371 | 372 | **Sample Prompt:** 373 | > "What Kafka brokers do we have available in our cluster?" 374 | 375 | **Example:** 376 | ```json 377 | {} 378 | ``` 379 | 380 | **Response:** 381 | ```json 382 | [ 383 | "kafka-broker-1:9092", 384 | "kafka-broker-2:9092", 385 | "kafka-broker-3:9092" 386 | ] 387 | ``` 388 | 389 | ### describe_topic 390 | 391 | Provides detailed metadata for a specific Kafka topic. 392 | 393 | **Sample Prompt:** 394 | > "Show me the configuration and partition details for our orders topic." 395 | 396 | **Example:** 397 | ```json 398 | { 399 | "topic_name": "orders" 400 | } 401 | ``` 402 | 403 | **Response:** 404 | ```json 405 | { 406 | "name": "orders", 407 | "partitions": [ 408 | { 409 | "partitionID": 0, 410 | "leader": 1, 411 | "replicas": [1, 2, 3], 412 | "ISR": [1, 2, 3], 413 | "errorCode": 0 414 | }, 415 | { 416 | "partitionID": 1, 417 | "leader": 2, 418 | "replicas": [2, 3, 1], 419 | "ISR": [2, 3, 1], 420 | "errorCode": 0 421 | } 422 | ], 423 | "isInternal": false 424 | } 425 | ``` 426 | 427 | ### list_consumer_groups 428 | 429 | Enumerates active consumer groups known by the Kafka cluster. 430 | 431 | **Sample Prompt:** 432 | > "What consumer groups are currently active in our Kafka cluster?" 433 | 434 | **Example:** 435 | ```json 436 | {} 437 | ``` 438 | 439 | **Response:** 440 | ```json 441 | [ 442 | { 443 | "groupID": "order-processor", 444 | "state": "Stable", 445 | "errorCode": 0 446 | }, 447 | { 448 | "groupID": "analytics-pipeline", 449 | "state": "Stable", 450 | "errorCode": 0 451 | } 452 | ] 453 | ``` 454 | 455 | ### describe_consumer_group 456 | 457 | Shows details for a specific consumer group, including state, members, and partition offsets. 458 | 459 | **Sample Prompt:** 460 | > "Tell me about the order-processor consumer group. Are there any lagging consumers?" 461 | 462 | **Example:** 463 | ```json 464 | { 465 | "group_id": "order-processor", 466 | "include_offsets": true 467 | } 468 | ``` 469 | 470 | **Response:** 471 | ```json 472 | { 473 | "groupID": "order-processor", 474 | "state": "Stable", 475 | "members": [ 476 | { 477 | "memberID": "consumer-1-uuid", 478 | "clientID": "consumer-1", 479 | "clientHost": "10.0.0.101", 480 | "assignments": [ 481 | {"topic": "orders", "partitions": [0, 2, 4]} 482 | ] 483 | }, 484 | { 485 | "memberID": "consumer-2-uuid", 486 | "clientID": "consumer-2", 487 | "clientHost": "10.0.0.102", 488 | "assignments": [ 489 | {"topic": "orders", "partitions": [1, 3, 5]} 490 | ] 491 | } 492 | ], 493 | "offsets": [ 494 | { 495 | "topic": "orders", 496 | "partition": 0, 497 | "commitOffset": 10045, 498 | "lag": 5 499 | }, 500 | // More partitions... 501 | ], 502 | "errorCode": 0 503 | } 504 | ``` 505 | 506 | ### describe_configs 507 | 508 | Fetches configuration entries for a specific resource (topic or broker). 509 | 510 | **Sample Prompt:** 511 | > "What's the retention configuration for our clickstream topic?" 512 | 513 | **Example:** 514 | ```json 515 | { 516 | "resource_type": "topic", 517 | "resource_name": "clickstream", 518 | "config_keys": ["retention.ms", "retention.bytes"] 519 | } 520 | ``` 521 | 522 | **Response:** 523 | ```json 524 | { 525 | "configs": [ 526 | { 527 | "name": "retention.ms", 528 | "value": "604800000", 529 | "source": "DYNAMIC_TOPIC_CONFIG", 530 | "isSensitive": false, 531 | "isReadOnly": false 532 | }, 533 | { 534 | "name": "retention.bytes", 535 | "value": "1073741824", 536 | "source": "DYNAMIC_TOPIC_CONFIG", 537 | "isSensitive": false, 538 | "isReadOnly": false 539 | } 540 | ] 541 | } 542 | ``` 543 | 544 | ### cluster_overview 545 | 546 | Aggregates high-level cluster health data, including controller, brokers, topics, and partition status. 547 | 548 | **Sample Prompt:** 549 | > "Give me an overview of our Kafka cluster health." 550 | 551 | **Example:** 552 | ```json 553 | {} 554 | ``` 555 | 556 | **Response:** 557 | ```json 558 | { 559 | "brokerCount": 3, 560 | "controllerID": 1, 561 | "topicCount": 24, 562 | "partitionCount": 120, 563 | "underReplicatedPartitionsCount": 0, 564 | "offlinePartitionsCount": 0, 565 | "offlineBrokerIDs": [] 566 | } 567 | ``` 568 | 569 | ## MCP Resources 570 | 571 | The server provides the following resources that can be accessed through the MCP protocol: 572 | 573 | ### kafka-mcp://{cluster}/overview 574 | 575 | Provides a summary of Kafka cluster health and metrics. 576 | 577 | **Example Response:** 578 | ```json 579 | { 580 | "timestamp": "2023-08-15T12:34:56Z", 581 | "broker_count": 3, 582 | "controller_id": 1, 583 | "topic_count": 24, 584 | "partition_count": 120, 585 | "under_replicated_partitions": 0, 586 | "offline_partitions": 0, 587 | "offline_broker_ids": [], 588 | "health_status": "healthy" 589 | } 590 | ``` 591 | 592 | ### kafka-mcp://{cluster}/health-check 593 | 594 | Performs a comprehensive health assessment of the Kafka cluster. 595 | 596 | **Example Response:** 597 | ```json 598 | { 599 | "timestamp": "2023-08-15T12:34:56Z", 600 | "broker_status": { 601 | "total_brokers": 3, 602 | "offline_brokers": 0, 603 | "offline_broker_ids": [], 604 | "status": "healthy" 605 | }, 606 | "controller_status": { 607 | "controller_id": 1, 608 | "status": "healthy" 609 | }, 610 | "partition_status": { 611 | "total_partitions": 120, 612 | "under_replicated_partitions": 0, 613 | "offline_partitions": 0, 614 | "status": "healthy" 615 | }, 616 | "consumer_status": { 617 | "total_groups": 5, 618 | "groups_with_high_lag": 0, 619 | "status": "healthy", 620 | "error": "" 621 | }, 622 | "overall_status": "healthy" 623 | } 624 | ``` 625 | 626 | ### kafka-mcp://{cluster}/under-replicated-partitions 627 | 628 | Provides a detailed report of under-replicated partitions in the cluster. 629 | 630 | **Example Response:** 631 | ```json 632 | { 633 | "timestamp": "2023-08-15T12:34:56Z", 634 | "under_replicated_partition_count": 2, 635 | "details": [ 636 | { 637 | "topic": "orders", 638 | "partition": 3, 639 | "leader": 1, 640 | "replica_count": 3, 641 | "isr_count": 2, 642 | "replicas": [1, 2, 3], 643 | "isr": [1, 2], 644 | "missing_replicas": [3] 645 | }, 646 | { 647 | "topic": "clickstream", 648 | "partition": 5, 649 | "leader": 2, 650 | "replica_count": 3, 651 | "isr_count": 2, 652 | "replicas": [2, 3, 1], 653 | "isr": [2, 1], 654 | "missing_replicas": [3] 655 | } 656 | ], 657 | "recommendations": [ 658 | "Check broker health for any offline or struggling brokers", 659 | "Verify network connectivity between brokers", 660 | "Monitor disk space on broker nodes", 661 | "Review broker logs for detailed error messages", 662 | "Consider increasing replication timeouts if network is slow" 663 | ] 664 | } 665 | ``` 666 | 667 | ### kafka-mcp://{cluster}/consumer-lag-report 668 | 669 | Analyzes consumer group lag across the cluster. Accepts an optional "threshold" query parameter to set the lag threshold. 670 | 671 | **Example Response:** 672 | ```json 673 | { 674 | "timestamp": "2023-08-15T12:34:56Z", 675 | "lag_threshold": 1000, 676 | "group_count": 3, 677 | "group_summary": [ 678 | { 679 | "group_id": "order-processor", 680 | "state": "Stable", 681 | "member_count": 2, 682 | "topic_count": 1, 683 | "total_lag": 15420, 684 | "has_high_lag": true 685 | }, 686 | { 687 | "group_id": "analytics-pipeline", 688 | "state": "Stable", 689 | "member_count": 3, 690 | "topic_count": 2, 691 | "total_lag": 520, 692 | "has_high_lag": false 693 | } 694 | ], 695 | "high_lag_details": [ 696 | { 697 | "group_id": "order-processor", 698 | "topic": "orders", 699 | "partition": 2, 700 | "current_offset": 1045822, 701 | "log_end_offset": 1061242, 702 | "lag": 15420 703 | } 704 | ], 705 | "recommendations": [ 706 | "Check consumer instances for errors or slowdowns", 707 | "Consider scaling up consumer groups with high lag", 708 | "Review consumer configuration settings", 709 | "Examine processing bottlenecks in consumer application logic" 710 | ] 711 | } 712 | ``` 713 | 714 | ## MCP Prompts 715 | 716 | The server includes the following pre-configured prompts for Kafka operations and diagnostics: 717 | 718 | ### kafka_cluster_overview 719 | 720 | Provides a summary of Kafka cluster health and metrics. 721 | 722 | **Arguments:** 723 | - `cluster` (required): The Kafka cluster name 724 | 725 | **Example Response:** 726 | ``` 727 | # Kafka Cluster Overview 728 | 729 | **Time**: 2023-08-15T12:34:56Z 730 | 731 | - **Broker Count**: 3 732 | - **Active Controller ID**: 1 733 | - **Total Topics**: 24 734 | - **Total Partitions**: 120 735 | - **Under-Replicated Partitions**: 0 736 | - **Offline Partitions**: 0 737 | 738 | **Overall Status**: ✅ Healthy 739 | ``` 740 | 741 | ### kafka_health_check 742 | 743 | Runs a comprehensive health check on the Kafka cluster. 744 | 745 | **Arguments:** 746 | - `cluster` (required): The Kafka cluster name 747 | 748 | **Example Response:** 749 | ``` 750 | # Kafka Cluster Health Check Report 751 | 752 | **Time**: 2023-08-15T12:34:56Z 753 | 754 | ## Broker Status 755 | 756 | - ✅ **All 3 brokers are online** 757 | 758 | ## Controller Status 759 | 760 | - ✅ **Active controller**: Broker 1 761 | 762 | ## Partition Health 763 | 764 | - ✅ **All 120 partitions are online** 765 | - ✅ **No under-replicated partitions detected** 766 | 767 | ## Consumer Group Health 768 | 769 | - ✅ **5 consumer groups are active** 770 | - ✅ **No consumer groups with significant lag detected** 771 | 772 | ## Overall Health Assessment 773 | 774 | ✅ **HEALTHY**: All systems are operating normally. 775 | ``` 776 | 777 | ### kafka_under_replicated_partitions 778 | 779 | Lists topics and partitions where ISR count is less than replication factor. 780 | 781 | **Arguments:** 782 | - `cluster` (required): The Kafka cluster name 783 | 784 | **Example Response:** 785 | ``` 786 | # Under-Replicated Partitions Report 787 | 788 | **Time**: 2023-08-15T12:34:56Z 789 | 790 | ⚠️ **Found 2 under-replicated partitions** 791 | 792 | | Topic | Partition | Leader | Replica Count | ISR Count | Missing Replicas | 793 | |:------|----------:|-------:|--------------:|----------:|:-----------------| 794 | | orders | 3 | 1 | 3 | 2 | 3 | 795 | | clickstream | 5 | 2 | 3 | 2 | 3 | 796 | 797 | ## Possible Causes 798 | 799 | Under-replicated partitions occur when one or more replicas are not in sync with the leader. Common causes include: 800 | 801 | - **Broker failure or network partition** 802 | - **High load on brokers** 803 | - **Insufficient disk space** 804 | - **Network bandwidth limitations** 805 | - **Misconfigured topic replication factor** 806 | 807 | ## Recommendations 808 | 809 | 1. **Check broker health** for any offline or struggling brokers 810 | 2. **Verify network connectivity** between brokers 811 | 3. **Monitor disk space** on broker nodes 812 | 4. **Review broker logs** for detailed error messages 813 | 5. **Consider increasing replication timeouts** if network is slow 814 | ``` 815 | 816 | ### kafka_consumer_lag_report 817 | 818 | Provides a detailed report on consumer lag across all consumer groups. 819 | 820 | **Arguments:** 821 | - `cluster` (required): The Kafka cluster name 822 | - `threshold` (optional): Lag threshold for highlighting high lag (default: 1000) 823 | 824 | **Example Response:** 825 | ``` 826 | # Kafka Consumer Lag Report 827 | 828 | **Time**: 2023-08-15T12:34:56Z 829 | 830 | **Lag Threshold**: 1000 messages 831 | 832 | Found 3 consumer group(s) 833 | 834 | ## Consumer Group Summary 835 | 836 | | Group ID | State | Members | Topics | Total Lag | High Lag | 837 | |:---------|:------|--------:|-------:|----------:|:---------| 838 | | order-processor | Stable | 2 | 1 | 15,420 | ⚠️ Yes | 839 | | analytics-pipeline | Stable | 3 | 2 | 520 | No | 840 | | monitoring | Stable | 1 | 3 | 0 | No | 841 | 842 | ## High Lag Details 843 | 844 | ### Group: order-processor 845 | 846 | | Topic | Partition | Current Offset | Log End Offset | Lag | 847 | |:------|----------:|--------------:|--------------:|----:| 848 | | orders | 2 | 1,045,822 | 1,061,242 | 15,420 | 849 | 850 | ## Recommendations 851 | 852 | 1. **Check consumer instances** for errors or slowdowns 853 | 2. **Scale up consumer groups** with high lag 854 | 3. **Review consumer configuration** settings 855 | 4. **Examine processing bottlenecks** in consumer application logic 856 | ``` 857 | 858 | ## End-to-End Example 859 | 860 | Here's an interaction example showing how an AI assistant might use these tools to help a user: 861 | 862 | **User Query:** "We need to investigate why our order processing pipeline is falling behind. Can you help me check the consumer lag?" 863 | 864 | **AI Assistant's workflow:** 865 | 866 | 1. First, check the consumer groups 867 | ``` 868 | > Using list_consumer_groups tool 869 | > Discovers "order-processor" group 870 | ``` 871 | 872 | 2. Then, examine the consumer group details 873 | ``` 874 | > Using describe_consumer_group tool with group_id "order-processor" and include_offsets=true 875 | > Discovers lag on several partitions 876 | ``` 877 | 878 | 3. Get cluster overview to check general health 879 | ``` 880 | > Using cluster_overview tool 881 | > Confirms no under-replicated partitions or offline brokers 882 | ``` 883 | 884 | 4. Generate a consumer lag report 885 | ``` 886 | > Using consumer_lag_report prompt with threshold=1000 887 | > Provides detailed analysis of lag across consumer groups 888 | ``` 889 | 890 | 5. Provide actionable recommendations: 891 | ``` 892 | Based on my analysis, your order-processor consumer group has significant lag on topic orders, partitions 2 and 3, with lag values of 15,420 and 12,305 messages respectively. 893 | 894 | The cluster itself is healthy with no under-replicated partitions, suggesting this is an issue with the consumer application rather than Kafka infrastructure. 895 | 896 | I recommend: 897 | 1. Check the consumer instances processing those partitions for slowdowns or errors 898 | 2. Consider scaling up your consumer group by adding more instances 899 | 3. Review the consumer configuration, particularly max.poll.records and max.poll.interval.ms 900 | 4. Examine any processing bottlenecks in the consumer logic 901 | ``` 902 | 903 | This seamless workflow demonstrates how the Kafka MCP tools enable LLM models to perform sophisticated diagnostics and provide actionable insights. 904 | 905 | ## Configuration 906 | 907 | The server can be configured using the following environment variables: 908 | 909 | | Variable | Description | Default | 910 | | :------- | :---------- | :------ | 911 | | `KAFKA_BROKERS` | Comma-separated list of Kafka broker addresses | `localhost:9092` | 912 | | `KAFKA_CLIENT_ID` | Kafka client ID used for connections | `kafka-mcp-server` | 913 | | `MCP_TRANSPORT` | MCP transport method (stdio/http) | `stdio` | 914 | | `KAFKA_SASL_MECHANISM` | SASL mechanism: `plain`, `scram-sha-256`, `scram-sha-512`, or `""` (disabled) | `""` | 915 | | `KAFKA_SASL_USER` | Username for SASL authentication | `""` | 916 | | `KAFKA_SASL_PASSWORD` | Password for SASL authentication | `""` | 917 | | `KAFKA_TLS_ENABLE` | Enable TLS for Kafka connection (`true` or `false`) | `false` | 918 | | `KAFKA_TLS_INSECURE_SKIP_VERIFY` | Skip TLS certificate verification (`true` or `false`) | `false` | 919 | 920 | > **Security Note:** When using `KAFKA_TLS_INSECURE_SKIP_VERIFY=true`, the server will skip TLS certificate verification. This should only be used in development or testing environments, or when using self-signed certificates. 921 | 922 | ## Security Considerations 923 | 924 | The server is designed with enterprise-grade security in mind: 925 | 926 | - **Authentication**: Full support for SASL PLAIN, SCRAM-SHA-256, and SCRAM-SHA-512 927 | - **Encryption**: TLS support for secure communication with Kafka brokers 928 | - **Input Validation**: Thorough validation of all user inputs to prevent injection attacks 929 | - **Error Handling**: Secure error handling that doesn't expose sensitive information 930 | 931 | ## Development 932 | 933 | ### Testing 934 | 935 | Comprehensive test coverage ensures reliability: 936 | 937 | ```bash 938 | # Run all tests (requires Docker for integration tests) 939 | go test ./... 940 | 941 | # Run tests excluding integration tests 942 | go test -short ./... 943 | 944 | # Run integration tests with specific Kafka brokers 945 | export KAFKA_BROKERS="your-broker:9092" 946 | export SKIP_KAFKA_TESTS="false" 947 | go test ./kafka -v -run Test 948 | ``` 949 | 950 | ### Contributing 951 | 952 | Contributions are welcome! Please feel free to submit a Pull Request. 953 | 954 | ## License 955 | 956 | This project is licensed under the MIT License - see the LICENSE file for details. 957 | -------------------------------------------------------------------------------- /cmd/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log/slog" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | 10 | "github.com/tuannvm/kafka-mcp-server/config" 11 | "github.com/tuannvm/kafka-mcp-server/kafka" 12 | "github.com/tuannvm/kafka-mcp-server/mcp" 13 | ) 14 | 15 | // Version is set during build via -X ldflags 16 | var Version = "dev" 17 | 18 | func main() { 19 | // Setup signal handling for graceful shutdown 20 | ctx, cancel := context.WithCancel(context.Background()) 21 | defer cancel() 22 | 23 | // Handle SIGINT and SIGTERM 24 | sigCh := make(chan os.Signal, 1) 25 | signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) 26 | go func() { 27 | sig := <-sigCh 28 | slog.Info("Received signal, shutting down", "signal", sig) 29 | cancel() 30 | }() 31 | 32 | // Load configuration 33 | cfg := config.LoadConfig() // Changed from cfg, err := config.LoadConfig() 34 | 35 | // Initialize Kafka client 36 | kafkaClient, err := kafka.NewClient(cfg) 37 | if err != nil { 38 | slog.Error("Failed to create Kafka client", "error", err) 39 | os.Exit(1) 40 | } 41 | defer kafkaClient.Close() 42 | 43 | // Create MCP server 44 | s := mcp.NewMCPServer("kafka-mcp-server", Version) 45 | 46 | // Explicitly declare the client as the KafkaClient interface type 47 | var kafkaInterface kafka.KafkaClient = kafkaClient 48 | 49 | // Register MCP resources and tools 50 | mcp.RegisterResources(s, kafkaInterface) 51 | mcp.RegisterTools(s, kafkaInterface, cfg) 52 | mcp.RegisterPrompts(s, kafkaInterface) 53 | 54 | // Start server 55 | slog.Info("Starting Kafka MCP server", "version", Version, "transport", cfg.MCPTransport) 56 | if err := mcp.Start(ctx, s, cfg); err != nil { 57 | slog.Error("Server error", "error", err) 58 | os.Exit(1) 59 | } 60 | 61 | slog.Info("Server shutdown complete") 62 | } 63 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "strconv" // Added for boolean parsing 6 | "strings" 7 | ) 8 | 9 | // Config holds the application configuration. 10 | type Config struct { 11 | KafkaBrokers []string // List of Kafka broker addresses 12 | KafkaClientID string // Kafka client ID 13 | MCPTransport string // MCP transport method ("stdio" or "http") 14 | 15 | // SASL Configuration 16 | SASLMechanism string // "plain", "scram-sha-256", "scram-sha-512", or "" (disabled) 17 | SASLUser string 18 | SASLPassword string 19 | 20 | // TLS Configuration 21 | TLSEnable bool // Whether to enable TLS 22 | TLSInsecureSkipVerify bool // Whether to skip TLS certificate verification (use with caution!) 23 | // TODO: Add paths for CA cert, client cert, client key if needed 24 | } 25 | 26 | // LoadConfig loads configuration from environment variables. 27 | func LoadConfig() Config { 28 | brokers := getEnv("KAFKA_BROKERS", "localhost:9092") 29 | clientID := getEnv("KAFKA_CLIENT_ID", "kafka-mcp-server") 30 | mcpTransport := getEnv("MCP_TRANSPORT", "stdio") 31 | 32 | // SASL Env Vars 33 | saslMechanism := strings.ToLower(getEnv("KAFKA_SASL_MECHANISM", "")) 34 | saslUser := getEnv("KAFKA_SASL_USER", "") 35 | saslPassword := getEnv("KAFKA_SASL_PASSWORD", "") 36 | 37 | // TLS Env Vars 38 | tlsEnableStr := getEnv("KAFKA_TLS_ENABLE", "false") 39 | tlsInsecureSkipVerifyStr := getEnv("KAFKA_TLS_INSECURE_SKIP_VERIFY", "false") 40 | 41 | tlsEnable, _ := strconv.ParseBool(tlsEnableStr) 42 | tlsInsecureSkipVerify, _ := strconv.ParseBool(tlsInsecureSkipVerifyStr) 43 | 44 | return Config{ 45 | KafkaBrokers: strings.Split(brokers, ","), 46 | KafkaClientID: clientID, 47 | MCPTransport: mcpTransport, 48 | 49 | SASLMechanism: saslMechanism, 50 | SASLUser: saslUser, 51 | SASLPassword: saslPassword, 52 | 53 | TLSEnable: tlsEnable, 54 | TLSInsecureSkipVerify: tlsInsecureSkipVerify, 55 | } 56 | } 57 | 58 | // getEnv retrieves an environment variable or returns a default value. 59 | func getEnv(key, fallback string) string { 60 | if value, exists := os.LookupEnv(key); exists { 61 | return value 62 | } 63 | return fallback 64 | } 65 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestLoadConfig(t *testing.T) { 9 | // Set environment variables for testing 10 | originalBrokers := os.Getenv("KAFKA_BROKERS") 11 | originalClientID := os.Getenv("KAFKA_CLIENT_ID") 12 | originalTransport := os.Getenv("MCP_TRANSPORT") 13 | originalSASLMech := os.Getenv("KAFKA_SASL_MECHANISM") 14 | originalSASLUser := os.Getenv("KAFKA_SASL_USER") 15 | originalSASLPass := os.Getenv("KAFKA_SASL_PASSWORD") 16 | originalTLSEnable := os.Getenv("KAFKA_TLS_ENABLE") 17 | originalTLSInsecure := os.Getenv("KAFKA_TLS_INSECURE_SKIP_VERIFY") 18 | 19 | defer func() { 20 | // Restore original environment variables 21 | if err := os.Setenv("KAFKA_BROKERS", originalBrokers); err != nil { 22 | t.Logf("Failed to restore KAFKA_BROKERS: %v", err) 23 | } 24 | if err := os.Setenv("KAFKA_CLIENT_ID", originalClientID); err != nil { 25 | t.Logf("Failed to restore KAFKA_CLIENT_ID: %v", err) 26 | } 27 | if err := os.Setenv("MCP_TRANSPORT", originalTransport); err != nil { 28 | t.Logf("Failed to restore MCP_TRANSPORT: %v", err) 29 | } 30 | if err := os.Setenv("KAFKA_SASL_MECHANISM", originalSASLMech); err != nil { 31 | t.Logf("Failed to restore KAFKA_SASL_MECHANISM: %v", err) 32 | } 33 | if err := os.Setenv("KAFKA_SASL_USER", originalSASLUser); err != nil { 34 | t.Logf("Failed to restore KAFKA_SASL_USER: %v", err) 35 | } 36 | if err := os.Setenv("KAFKA_SASL_PASSWORD", originalSASLPass); err != nil { 37 | t.Logf("Failed to restore KAFKA_SASL_PASSWORD: %v", err) 38 | } 39 | if err := os.Setenv("KAFKA_TLS_ENABLE", originalTLSEnable); err != nil { 40 | t.Logf("Failed to restore KAFKA_TLS_ENABLE: %v", err) 41 | } 42 | if err := os.Setenv("KAFKA_TLS_INSECURE_SKIP_VERIFY", originalTLSInsecure); err != nil { 43 | t.Logf("Failed to restore KAFKA_TLS_INSECURE_SKIP_VERIFY: %v", err) 44 | } 45 | }() 46 | 47 | if err := os.Setenv("KAFKA_BROKERS", "test-broker1:9092,test-broker2:9092"); err != nil { 48 | t.Fatalf("Failed to set KAFKA_BROKERS: %v", err) 49 | } 50 | if err := os.Setenv("KAFKA_CLIENT_ID", "test-client"); err != nil { 51 | t.Fatalf("Failed to set KAFKA_CLIENT_ID: %v", err) 52 | } 53 | if err := os.Setenv("MCP_TRANSPORT", "stdio"); err != nil { 54 | t.Fatalf("Failed to set MCP_TRANSPORT: %v", err) 55 | } 56 | if err := os.Setenv("KAFKA_SASL_MECHANISM", "plain"); err != nil { 57 | t.Fatalf("Failed to set KAFKA_SASL_MECHANISM: %v", err) 58 | } 59 | if err := os.Setenv("KAFKA_SASL_USER", "testuser"); err != nil { 60 | t.Fatalf("Failed to set KAFKA_SASL_USER: %v", err) 61 | } 62 | if err := os.Setenv("KAFKA_SASL_PASSWORD", "testpass"); err != nil { 63 | t.Fatalf("Failed to set KAFKA_SASL_PASSWORD: %v", err) 64 | } 65 | if err := os.Setenv("KAFKA_TLS_ENABLE", "true"); err != nil { 66 | t.Fatalf("Failed to set KAFKA_TLS_ENABLE: %v", err) 67 | } 68 | if err := os.Setenv("KAFKA_TLS_INSECURE_SKIP_VERIFY", "true"); err != nil { 69 | t.Fatalf("Failed to set KAFKA_TLS_INSECURE_SKIP_VERIFY: %v", err) 70 | } 71 | 72 | cfg := LoadConfig() 73 | 74 | if len(cfg.KafkaBrokers) != 2 || cfg.KafkaBrokers[0] != "test-broker1:9092" || cfg.KafkaBrokers[1] != "test-broker2:9092" { 75 | t.Errorf("Expected KafkaBrokers [test-broker1:9092 test-broker2:9092], got %v", cfg.KafkaBrokers) 76 | } 77 | if cfg.KafkaClientID != "test-client" { 78 | t.Errorf("Expected KafkaClientID test-client, got %s", cfg.KafkaClientID) 79 | } 80 | if cfg.MCPTransport != "stdio" { 81 | t.Errorf("Expected MCPTransport stdio, got %s", cfg.MCPTransport) 82 | } 83 | if cfg.SASLMechanism != "plain" { 84 | t.Errorf("Expected SASLMechanism plain, got %s", cfg.SASLMechanism) 85 | } 86 | if cfg.SASLUser != "testuser" { 87 | t.Errorf("Expected SASLUser testuser, got %s", cfg.SASLUser) 88 | } 89 | if cfg.SASLPassword != "testpass" { 90 | t.Errorf("Expected SASLPassword testpass, got %s", cfg.SASLPassword) 91 | } 92 | if !cfg.TLSEnable { 93 | t.Errorf("Expected TLSEnable true, got %v", cfg.TLSEnable) 94 | } 95 | if !cfg.TLSInsecureSkipVerify { 96 | t.Errorf("Expected TLSInsecureSkipVerify true, got %v", cfg.TLSInsecureSkipVerify) 97 | } 98 | } 99 | 100 | func TestLoadConfigDefaults(t *testing.T) { 101 | // Clear environment variables to test defaults 102 | if err := os.Unsetenv("KAFKA_BROKERS"); err != nil { 103 | t.Logf("Failed to unset KAFKA_BROKERS: %v", err) 104 | } 105 | if err := os.Unsetenv("KAFKA_CLIENT_ID"); err != nil { 106 | t.Logf("Failed to unset KAFKA_CLIENT_ID: %v", err) 107 | } 108 | if err := os.Unsetenv("MCP_TRANSPORT"); err != nil { 109 | t.Logf("Failed to unset MCP_TRANSPORT: %v", err) 110 | } 111 | if err := os.Unsetenv("KAFKA_SASL_MECHANISM"); err != nil { 112 | t.Logf("Failed to unset KAFKA_SASL_MECHANISM: %v", err) 113 | } 114 | if err := os.Unsetenv("KAFKA_TLS_ENABLE"); err != nil { 115 | t.Logf("Failed to unset KAFKA_TLS_ENABLE: %v", err) 116 | } 117 | if err := os.Unsetenv("KAFKA_TLS_INSECURE_SKIP_VERIFY"); err != nil { 118 | t.Logf("Failed to unset KAFKA_TLS_INSECURE_SKIP_VERIFY: %v", err) 119 | } 120 | 121 | cfg := LoadConfig() 122 | 123 | if len(cfg.KafkaBrokers) != 1 || cfg.KafkaBrokers[0] != "localhost:9092" { 124 | t.Errorf("Expected default KafkaBrokers [localhost:9092], got %v", cfg.KafkaBrokers) 125 | } 126 | if cfg.KafkaClientID != "kafka-mcp-server" { 127 | t.Errorf("Expected default KafkaClientID kafka-mcp-server, got %s", cfg.KafkaClientID) 128 | } 129 | if cfg.MCPTransport != "stdio" { 130 | t.Errorf("Expected default MCPTransport stdio, got %s", cfg.MCPTransport) 131 | } 132 | if cfg.SASLMechanism != "" { 133 | t.Errorf("Expected default SASLMechanism \"\", got %s", cfg.SASLMechanism) 134 | } 135 | if cfg.TLSEnable { 136 | t.Errorf("Expected default TLSEnable false, got %v", cfg.TLSEnable) 137 | } 138 | if cfg.TLSInsecureSkipVerify { 139 | t.Errorf("Expected default TLSInsecureSkipVerify false, got %v", cfg.TLSInsecureSkipVerify) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tuannvm/kafka-mcp-server 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | github.com/mark3labs/mcp-go v0.25.0 7 | github.com/stretchr/testify v1.10.0 8 | github.com/testcontainers/testcontainers-go/modules/kafka v0.37.0 9 | github.com/twmb/franz-go v1.18.1 10 | github.com/twmb/franz-go/pkg/kmsg v1.11.2 11 | ) 12 | 13 | require ( 14 | dario.cat/mergo v1.0.1 // indirect 15 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 16 | github.com/Microsoft/go-winio v0.6.2 // indirect 17 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect 18 | github.com/containerd/log v0.1.0 // indirect 19 | github.com/containerd/platforms v0.2.1 // indirect 20 | github.com/cpuguy83/dockercfg v0.3.2 // indirect 21 | github.com/davecgh/go-spew v1.1.1 // indirect 22 | github.com/distribution/reference v0.6.0 // indirect 23 | github.com/docker/docker v28.0.1+incompatible // indirect 24 | github.com/docker/go-connections v0.5.0 // indirect 25 | github.com/docker/go-units v0.5.0 // indirect 26 | github.com/ebitengine/purego v0.8.2 // indirect 27 | github.com/felixge/httpsnoop v1.0.4 // indirect 28 | github.com/go-logr/logr v1.4.2 // indirect 29 | github.com/go-logr/stdr v1.2.2 // indirect 30 | github.com/go-ole/go-ole v1.2.6 // indirect 31 | github.com/gogo/protobuf v1.3.2 // indirect 32 | github.com/google/uuid v1.6.0 // indirect 33 | github.com/klauspost/compress v1.17.11 // indirect 34 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect 35 | github.com/magiconair/properties v1.8.10 // indirect 36 | github.com/moby/docker-image-spec v1.3.1 // indirect 37 | github.com/moby/patternmatcher v0.6.0 // indirect 38 | github.com/moby/sys/sequential v0.5.0 // indirect 39 | github.com/moby/sys/user v0.1.0 // indirect 40 | github.com/moby/sys/userns v0.1.0 // indirect 41 | github.com/moby/term v0.5.0 // indirect 42 | github.com/morikuni/aec v1.0.0 // indirect 43 | github.com/opencontainers/go-digest v1.0.0 // indirect 44 | github.com/opencontainers/image-spec v1.1.1 // indirect 45 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 46 | github.com/pkg/errors v0.9.1 // indirect 47 | github.com/pmezard/go-difflib v1.0.0 // indirect 48 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect 49 | github.com/shirou/gopsutil/v4 v4.25.1 // indirect 50 | github.com/sirupsen/logrus v1.9.3 // indirect 51 | github.com/spf13/cast v1.7.1 // indirect 52 | github.com/testcontainers/testcontainers-go v0.37.0 // indirect 53 | github.com/tklauser/go-sysconf v0.3.12 // indirect 54 | github.com/tklauser/numcpus v0.6.1 // indirect 55 | github.com/yosida95/uritemplate/v3 v3.0.2 // indirect 56 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 57 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 58 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 59 | go.opentelemetry.io/otel v1.35.0 // indirect 60 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect 61 | go.opentelemetry.io/otel/metric v1.35.0 // indirect 62 | go.opentelemetry.io/otel/sdk v1.35.0 // indirect 63 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 64 | go.opentelemetry.io/proto/otlp v1.5.0 // indirect 65 | golang.org/x/crypto v0.37.0 // indirect 66 | golang.org/x/mod v0.24.0 // indirect 67 | golang.org/x/sys v0.32.0 // indirect 68 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34 // indirect 69 | google.golang.org/protobuf v1.36.6 // indirect 70 | gopkg.in/yaml.v3 v3.0.1 // indirect 71 | ) 72 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= 2 | dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= 3 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= 4 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= 5 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= 6 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 7 | github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= 8 | github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= 9 | github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= 10 | github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= 11 | github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= 12 | github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 13 | github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= 14 | github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= 15 | github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= 16 | github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= 17 | github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= 18 | github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= 19 | github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= 20 | github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= 21 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 23 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 24 | github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= 25 | github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= 26 | github.com/docker/docker v28.0.1+incompatible h1:FCHjSRdXhNRFjlHMTv4jUNlIBbTeRjrWfeFuJp7jpo0= 27 | github.com/docker/docker v28.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 28 | github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= 29 | github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= 30 | github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 31 | github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 32 | github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= 33 | github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= 34 | github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= 35 | github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= 36 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 37 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 38 | github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= 39 | github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= 40 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 41 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 42 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 43 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 44 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 45 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 46 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 47 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 48 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 49 | github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= 50 | github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= 51 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 52 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 53 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 54 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 55 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 56 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 57 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 58 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 59 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 60 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg= 61 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= 62 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 63 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 64 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 65 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 66 | github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= 67 | github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 68 | github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= 69 | github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= 70 | github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= 71 | github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= 72 | github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= 73 | github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= 74 | github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= 75 | github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= 76 | github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= 77 | github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= 78 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 79 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 80 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 81 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 82 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 83 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 84 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 85 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 86 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= 87 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= 88 | github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= 89 | github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 90 | github.com/mark3labs/mcp-go v0.25.0 h1:UUpcMT3L5hIhuDy7aifj4Bphw4Pfx1Rf8mzMXDe8RQw= 91 | github.com/mark3labs/mcp-go v0.25.0/go.mod h1:rXqOudj/djTORU/ThxYx8fqEVj/5pvTuuebQ2RC7uk4= 92 | github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= 93 | github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= 94 | github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= 95 | github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= 96 | github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= 97 | github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= 98 | github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= 99 | github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= 100 | github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= 101 | github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= 102 | github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= 103 | github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= 104 | github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= 105 | github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= 106 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 107 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 108 | github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= 109 | github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= 110 | github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= 111 | github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 112 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 113 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 114 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 115 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 116 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= 117 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= 118 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= 119 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 120 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 121 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 122 | github.com/shirou/gopsutil/v4 v4.25.1 h1:QSWkTc+fu9LTAWfkZwZ6j8MSUk4A2LV7rbH0ZqmLjXs= 123 | github.com/shirou/gopsutil/v4 v4.25.1/go.mod h1:RoUCUpndaJFtT+2zsZzzmhvbfGoDCJ7nFXKJf8GqJbI= 124 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 125 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 126 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 127 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 128 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 129 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 130 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 131 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 132 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 133 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 134 | github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg= 135 | github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM= 136 | github.com/testcontainers/testcontainers-go/modules/kafka v0.37.0 h1:ZkYNKqhqvKm+aZk9C1fxw/fpNNOK+Nm/wHPjmJdN3Ko= 137 | github.com/testcontainers/testcontainers-go/modules/kafka v0.37.0/go.mod h1:+LvaFfSFW5PMiJTxTQlV6TBpXH1Ktk1h0FTVRZfqSxY= 138 | github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= 139 | github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= 140 | github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= 141 | github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= 142 | github.com/twmb/franz-go v1.18.1 h1:D75xxCDyvTqBSiImFx2lkPduE39jz1vaD7+FNc+vMkc= 143 | github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M= 144 | github.com/twmb/franz-go/pkg/kmsg v1.11.2 h1:hIw75FpwcAjgeyfIGFqivAvwC5uNIOWRGvQgZhH4mhg= 145 | github.com/twmb/franz-go/pkg/kmsg v1.11.2/go.mod h1:CFfkkLysDNmukPYhGzuUcDtf46gQSqCZHMW1T4Z+wDE= 146 | github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= 147 | github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= 148 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 149 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 150 | github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= 151 | github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= 152 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 153 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 154 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= 155 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= 156 | go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= 157 | go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= 158 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw= 159 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4= 160 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= 161 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= 162 | go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= 163 | go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= 164 | go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= 165 | go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= 166 | go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= 167 | go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= 168 | go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= 169 | go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= 170 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 171 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 172 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 173 | golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= 174 | golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= 175 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 176 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 177 | golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= 178 | golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= 179 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 180 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 181 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 182 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 183 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 184 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 185 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 186 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 187 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 188 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 189 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 190 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 191 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 192 | golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 193 | golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 194 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 195 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 196 | golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 197 | golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= 198 | golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 199 | golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= 200 | golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= 201 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 202 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 203 | golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= 204 | golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= 205 | golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= 206 | golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 207 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 208 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 209 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 210 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 211 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 212 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 213 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 214 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 215 | google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d h1:H8tOf8XM88HvKqLTxe755haY6r1fqqzLbEnfrmLXlSA= 216 | google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d/go.mod h1:2v7Z7gP2ZUOGsaFyxATQSRoBnKygqVq2Cwnvom7QiqY= 217 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34 h1:h6p3mQqrmT1XkHVTfzLdNz1u7IhINeZkz67/xTbOuWs= 218 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= 219 | google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= 220 | google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= 221 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 222 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 223 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 224 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 225 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 226 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 227 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 228 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 229 | gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= 230 | gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= 231 | -------------------------------------------------------------------------------- /kafka/client.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "log/slog" 8 | 9 | "os" 10 | 11 | "github.com/tuannvm/kafka-mcp-server/config" 12 | "github.com/twmb/franz-go/pkg/kerr" 13 | "github.com/twmb/franz-go/pkg/kgo" 14 | "github.com/twmb/franz-go/pkg/kmsg" 15 | "github.com/twmb/franz-go/pkg/sasl" 16 | "github.com/twmb/franz-go/pkg/sasl/plain" 17 | "github.com/twmb/franz-go/pkg/sasl/scram" 18 | ) 19 | 20 | // Message represents a consumed Kafka message data. 21 | type Message struct { 22 | Topic string `json:"topic"` 23 | Partition int32 `json:"partition"` 24 | Offset int64 `json:"offset"` 25 | Key string `json:"key"` 26 | Value string `json:"value"` 27 | Timestamp int64 `json:"timestamp"` // Unix milliseconds 28 | } 29 | 30 | // TopicMetadata holds detailed information about a topic. 31 | type TopicMetadata struct { 32 | TopicName string `json:"topic_name"` 33 | Partitions []PartitionMetadata `json:"partitions"` 34 | IsInternal bool `json:"is_internal"` 35 | ErrorCode int16 `json:"error_code,omitempty"` // Include error code if any 36 | ErrorMessage string `json:"error_message,omitempty"` 37 | } 38 | 39 | // PartitionMetadata holds details about a single partition. 40 | type PartitionMetadata struct { 41 | PartitionID int32 `json:"partition_id"` 42 | Leader int32 `json:"leader"` 43 | Replicas []int32 `json:"replicas"` 44 | ISR []int32 `json:"isr"` // In-Sync Replicas 45 | ErrorCode int16 `json:"error_code,omitempty"` 46 | ErrorMessage string `json:"error_message,omitempty"` 47 | } 48 | 49 | // ConsumerGroupInfo holds basic information about a consumer group. 50 | type ConsumerGroupInfo struct { 51 | GroupID string `json:"group_id"` 52 | State string `json:"state"` // e.g., "Stable", "PreparingRebalance", "Empty" 53 | ErrorCode int16 `json:"error_code,omitempty"` 54 | ErrorMessage string `json:"error_message,omitempty"` 55 | } 56 | 57 | // DescribeConsumerGroupResult holds detailed information about a consumer group. 58 | type DescribeConsumerGroupResult struct { 59 | GroupID string `json:"group_id"` 60 | State string `json:"state"` 61 | ProtocolType string `json:"protocol_type"` 62 | Protocol string `json:"protocol"` 63 | Members []GroupMemberInfo `json:"members"` 64 | Offsets []PartitionOffsetInfo `json:"offsets,omitempty"` // Optional, fetched separately 65 | // CoordinatorID int32 `json:"coordinator_id"` // Removed, not directly available in DescribeGroupsResponseGroup 66 | ErrorCode int16 `json:"error_code,omitempty"` 67 | ErrorMessage string `json:"error_message,omitempty"` 68 | } 69 | 70 | // GroupMemberInfo holds information about a single member within a consumer group. 71 | type GroupMemberInfo struct { 72 | MemberID string `json:"member_id"` 73 | ClientID string `json:"client_id"` 74 | ClientHost string `json:"client_host"` 75 | } 76 | 77 | // PartitionOffsetInfo holds offset and lag information for a partition assigned to a group. 78 | type PartitionOffsetInfo struct { 79 | Topic string `json:"topic"` 80 | Partition int32 `json:"partition"` 81 | CommitOffset int64 `json:"commit_offset"` 82 | Lag int64 `json:"lag"` // Calculated: LogEndOffset - CommitOffset 83 | Metadata string `json:"metadata,omitempty"` 84 | ErrorCode int16 `json:"error_code,omitempty"` 85 | ErrorMessage string `json:"error_message,omitempty"` 86 | } 87 | 88 | // ConfigEntry represents a single configuration key-value pair for a resource. 89 | type ConfigEntry struct { 90 | Name string `json:"name"` 91 | Value string `json:"value,omitempty"` // Value might be nil if sensitive or default 92 | IsDefault bool `json:"is_default"` 93 | IsReadOnly bool `json:"is_read_only"` 94 | IsSensitive bool `json:"is_sensitive"` 95 | Source string `json:"source"` // e.g., "DEFAULT_CONFIG", "TOPIC_CONFIG", "BROKER_CONFIG" 96 | // Synonyms can be included if needed 97 | } 98 | 99 | // DescribeConfigsResult holds the configuration entries for a specific resource. 100 | type DescribeConfigsResult struct { 101 | ResourceType string `json:"resource_type"` 102 | ResourceName string `json:"resource_name"` 103 | Configs []ConfigEntry `json:"configs"` 104 | ErrorCode int16 `json:"error_code,omitempty"` 105 | ErrorMessage string `json:"error_message,omitempty"` 106 | } 107 | 108 | // ClusterOverviewResult holds high-level cluster health information. 109 | type ClusterOverviewResult struct { 110 | ControllerID int32 `json:"controller_id"` 111 | BrokerCount int `json:"broker_count"` 112 | TopicCount int `json:"topic_count"` // Excluding internal topics 113 | PartitionCount int `json:"partition_count"` 114 | UnderReplicatedPartitionsCount int `json:"under_replicated_partitions_count"` 115 | OfflinePartitionsCount int `json:"offline_partitions_count"` 116 | OfflineBrokerIDs []int32 `json:"offline_broker_ids,omitempty"` // Brokers that failed metadata request 117 | ErrorCode int16 `json:"error_code,omitempty"` // Overall error, e.g., if metadata fails completely 118 | ErrorMessage string `json:"error_message,omitempty"` 119 | } 120 | 121 | // Client wraps the franz-go Kafka client. 122 | type Client struct { 123 | kgoClient *kgo.Client 124 | } 125 | 126 | // NewClient creates and initializes a new Kafka client based on the provided configuration. 127 | func NewClient(cfg config.Config) (*Client, error) { 128 | opts := []kgo.Opt{ 129 | kgo.SeedBrokers(cfg.KafkaBrokers...), 130 | kgo.ClientID(cfg.KafkaClientID), 131 | // Corrected BasicLogger usage: write to os.Stderr 132 | kgo.WithLogger(kgo.BasicLogger(os.Stderr, kgo.LogLevelInfo, nil)), 133 | // Add consumer group if specified (needed for ConsumeMessages) 134 | // TODO: Make group ID configurable? For now, use client ID. 135 | kgo.ConsumerGroup(cfg.KafkaClientID), 136 | kgo.ConsumeTopics(), // Initialize with empty topics, will be added in ConsumeMessages 137 | kgo.DisableAutoCommit(), // Recommend disabling auto-commit for MCP tools 138 | } 139 | 140 | // --- TLS Configuration --- 141 | if cfg.TLSEnable { 142 | tlsConfig := &tls.Config{ 143 | InsecureSkipVerify: cfg.TLSInsecureSkipVerify, 144 | // TODO: Add RootCAs, Certificates if client cert auth is needed 145 | } 146 | opts = append(opts, kgo.DialTLSConfig(tlsConfig)) 147 | slog.Info("TLS enabled for Kafka client", "insecureSkipVerify", cfg.TLSInsecureSkipVerify) 148 | } 149 | 150 | // --- SASL Configuration --- 151 | var saslMethod sasl.Mechanism 152 | switch cfg.SASLMechanism { 153 | case "plain": 154 | if cfg.SASLUser == "" || cfg.SASLPassword == "" { 155 | return nil, fmt.Errorf("SASL PLAIN requires KAFKA_SASL_USER and KAFKA_SASL_PASSWORD") 156 | } 157 | saslMethod = plain.Auth{ 158 | User: cfg.SASLUser, 159 | Pass: cfg.SASLPassword, 160 | }.AsMechanism() 161 | slog.Info("SASL PLAIN mechanism configured") 162 | case "scram-sha-256", "scram-sha-512": 163 | if cfg.SASLUser == "" || cfg.SASLPassword == "" { 164 | return nil, fmt.Errorf("SASL SCRAM requires KAFKA_SASL_USER and KAFKA_SASL_PASSWORD") 165 | } 166 | var scramAuth sasl.Mechanism 167 | if cfg.SASLMechanism == "scram-sha-256" { 168 | scramAuth = scram.Auth{ 169 | User: cfg.SASLUser, 170 | Pass: cfg.SASLPassword, 171 | }.AsSha256Mechanism() 172 | } else { 173 | scramAuth = scram.Auth{ 174 | User: cfg.SASLUser, 175 | Pass: cfg.SASLPassword, 176 | }.AsSha512Mechanism() 177 | } 178 | saslMethod = scramAuth 179 | slog.Info("SASL SCRAM mechanism configured", "type", cfg.SASLMechanism) 180 | case "": 181 | // SASL disabled, do nothing 182 | slog.Info("SASL disabled") 183 | default: 184 | return nil, fmt.Errorf("unsupported SASL mechanism: %s", cfg.SASLMechanism) 185 | } 186 | 187 | if saslMethod != nil { 188 | opts = append(opts, kgo.SASL(saslMethod)) 189 | } 190 | 191 | // --- Create Client --- 192 | cl, err := kgo.NewClient(opts...) 193 | if err != nil { 194 | return nil, fmt.Errorf("failed to create Kafka client: %w", err) 195 | } 196 | 197 | // Test connectivity on startup (optional, adds delay but catches config errors early) 198 | if err := cl.Ping(context.Background()); err != nil { 199 | cl.Close() // Close the client if ping fails 200 | return nil, fmt.Errorf("failed to ping Kafka brokers: %w", err) 201 | } 202 | slog.Info("Successfully pinged Kafka brokers") 203 | 204 | return &Client{kgoClient: cl}, nil 205 | } 206 | 207 | // ProduceMessage sends a message to the specified Kafka topic. 208 | func (c *Client) ProduceMessage(ctx context.Context, topic string, key, value []byte) error { 209 | record := &kgo.Record{ 210 | Topic: topic, 211 | Key: key, 212 | Value: value, 213 | } 214 | // ProduceSync waits for the record to be acknowledged by Kafka. 215 | // For higher throughput, consider Produce which is asynchronous. 216 | err := c.kgoClient.ProduceSync(ctx, record).FirstErr() 217 | return err // Will be nil if producing was successful 218 | } 219 | 220 | // ConsumeMessages polls for messages from the specified topics. 221 | // It requires the client to have been initialized with consumer group options. 222 | // It returns a slice of consumed messages or an error. 223 | func (c *Client) ConsumeMessages(ctx context.Context, topics []string, maxMessages int) ([]Message, error) { 224 | // Note: This function assumes the kgo.Client was already configured with 225 | // kgo.ConsumerGroup(...) during its creation in NewClient. 226 | 227 | // Add the requested topics to the subscription. 228 | // Note: These topics will remain in the subscription until the client is closed 229 | // or they are explicitly removed later. This might be desired behavior for an MCP server. 230 | c.kgoClient.AddConsumeTopics(topics...) 231 | 232 | slog.InfoContext(ctx, "Polling for messages", "topics", topics, "maxMessages", maxMessages) 233 | 234 | // Poll for fetches. 235 | fetches := c.kgoClient.PollFetches(ctx) 236 | if fetches.IsClientClosed() { 237 | return nil, fmt.Errorf("kafka client is closed") 238 | } 239 | 240 | // Check for general fetch errors 241 | errs := fetches.Errors() 242 | if len(errs) > 0 { 243 | firstErr := errs[0] 244 | return nil, fmt.Errorf("fetch error on topic %q (partition %d): %w", firstErr.Topic, firstErr.Partition, firstErr.Err) 245 | } 246 | 247 | consumedMessages := make([]Message, 0, maxMessages) 248 | iter := fetches.RecordIter() 249 | count := 0 250 | for !iter.Done() && count < maxMessages { 251 | rec := iter.Next() 252 | 253 | consumedMessages = append(consumedMessages, Message{ 254 | Topic: rec.Topic, 255 | Partition: rec.Partition, 256 | Offset: rec.Offset, 257 | Key: string(rec.Key), 258 | Value: string(rec.Value), 259 | Timestamp: rec.Timestamp.UnixMilli(), 260 | }) 261 | count++ 262 | } 263 | 264 | slog.InfoContext(ctx, "Finished polling", "messagesConsumed", len(consumedMessages)) 265 | 266 | // Note: Committing offsets is crucial for consumer groups. 267 | // kgo.Client handles auto-committing by default if group ID is set at init. 268 | // If manual commit is needed (e.g., after processing), use: 269 | // err := c.kgoClient.CommitRecords(ctx, fetches.Records()...) 270 | // Handle commit error appropriately. 271 | 272 | return consumedMessages, nil 273 | } 274 | 275 | // ListTopics retrieves a list of topic names from the Kafka cluster. 276 | func (c *Client) ListTopics(ctx context.Context) ([]string, error) { 277 | // Use kmsg.NewMetadataRequest() to create the request struct 278 | req := kmsg.NewMetadataRequest() 279 | req.Topics = nil // Request metadata for all topics 280 | 281 | // RequestSharded returns *kgo.ShardedRequestPromise 282 | // We need to await its result. 283 | shardedResp := c.kgoClient.RequestSharded(ctx, &req) 284 | 285 | topicSet := make(map[string]struct{}) // Use map for deduplication 286 | 287 | // Iterate through responses from each broker shard 288 | // The response type in the shard is *kmsg.MetadataResponse 289 | for _, shard := range shardedResp { 290 | if shard.Err != nil { 291 | // Get broker ID from shard metadata if available 292 | brokerID := shard.Meta.NodeID // Access NodeID directly 293 | return nil, fmt.Errorf("metadata request failed for broker %d: %w", brokerID, shard.Err) 294 | } 295 | 296 | // Cast the response to the correct type 297 | resp, ok := shard.Resp.(*kmsg.MetadataResponse) 298 | if !ok { 299 | // This shouldn't happen if the request was kmsg.MetadataRequest 300 | return nil, fmt.Errorf("unexpected response type for metadata request from broker %d", shard.Meta.NodeID) 301 | } 302 | 303 | // Process topics from this broker's response 304 | for _, topic := range resp.Topics { 305 | if topic.ErrorCode != 0 { 306 | // Log topic-specific errors using kerr 307 | errMsg := kerr.ErrorForCode(topic.ErrorCode) 308 | // Dereference topic name pointer for logging, handle nil case 309 | topicName := "unknown" 310 | if topic.Topic != nil { 311 | topicName = *topic.Topic 312 | } 313 | slog.WarnContext(ctx, "Error fetching metadata for topic", "topic", topicName, "broker", shard.Meta.NodeID, "error_code", topic.ErrorCode, "error", errMsg.Error()) 314 | continue // Skip topics with errors 315 | } 316 | // Dereference topic name pointer for map key, ensure not nil 317 | if topic.Topic != nil { 318 | // Ignore internal topics if desired (e.g., __consumer_offsets) 319 | // if strings.HasPrefix(*topic.Topic, "__") { continue } 320 | topicSet[*topic.Topic] = struct{}{} 321 | } 322 | } 323 | } 324 | 325 | topics := make([]string, 0, len(topicSet)) 326 | for topic := range topicSet { 327 | topics = append(topics, topic) 328 | } 329 | 330 | return topics, nil 331 | } 332 | 333 | // ListBrokers retrieves a list of broker addresses from the Kafka cluster. 334 | func (c *Client) ListBrokers(ctx context.Context) ([]string, error) { 335 | // Use kmsg.NewMetadataRequest() to create the request struct 336 | req := kmsg.NewMetadataRequest() 337 | req.Topics = nil // Request metadata for all topics, which includes broker info 338 | 339 | // RequestSharded returns *kgo.ShardedRequestPromise 340 | // We need to await its result. 341 | shardedResp := c.kgoClient.RequestSharded(ctx, &req) 342 | 343 | brokerSet := make(map[string]struct{}) // Use map for deduplication 344 | var firstError error 345 | 346 | // Iterate through responses from each broker shard 347 | for _, shard := range shardedResp { 348 | if shard.Err != nil { 349 | brokerID := shard.Meta.NodeID 350 | slog.ErrorContext(ctx, "Metadata request shard error for brokers", "broker", brokerID, "error", shard.Err) 351 | if firstError == nil { 352 | firstError = fmt.Errorf("metadata request failed for broker %d: %w", brokerID, shard.Err) 353 | } 354 | continue // Try other brokers if possible 355 | } 356 | 357 | // Cast the response to the correct type 358 | resp, ok := shard.Resp.(*kmsg.MetadataResponse) 359 | if !ok { 360 | slog.ErrorContext(ctx, "Unexpected response type for metadata request", "broker", shard.Meta.NodeID) 361 | if firstError == nil { 362 | firstError = fmt.Errorf("unexpected metadata response type from broker %d", shard.Meta.NodeID) 363 | } 364 | continue 365 | } 366 | 367 | // Process brokers from this response 368 | for _, broker := range resp.Brokers { 369 | brokerAddr := fmt.Sprintf("%s:%d", broker.Host, broker.Port) 370 | brokerSet[brokerAddr] = struct{}{} 371 | } 372 | // Since broker list is cluster-wide, we likely only need one successful response. 373 | // If we got brokers, we can stop processing shards. 374 | if len(brokerSet) > 0 { 375 | break 376 | } 377 | } 378 | 379 | if len(brokerSet) > 0 { 380 | brokers := make([]string, 0, len(brokerSet)) 381 | for broker := range brokerSet { 382 | brokers = append(brokers, broker) 383 | } 384 | return brokers, nil 385 | } 386 | 387 | // If loop finished and brokerSet is empty, return the first error or a generic one 388 | if firstError != nil { 389 | return nil, firstError 390 | } 391 | 392 | return nil, fmt.Errorf("failed to retrieve broker list from any broker") 393 | } 394 | 395 | // DescribeTopic retrieves detailed metadata for a specific topic. 396 | func (c *Client) DescribeTopic(ctx context.Context, topicName string) (*TopicMetadata, error) { 397 | req := kmsg.NewMetadataRequest() 398 | topicReq := kmsg.NewMetadataRequestTopic() 399 | topicReq.Topic = kmsg.StringPtr(topicName) 400 | req.Topics = append(req.Topics, topicReq) 401 | req.AllowAutoTopicCreation = false // Don't create if it doesn't exist 402 | 403 | shardedResp := c.kgoClient.RequestSharded(ctx, &req) 404 | 405 | var finalMetadata *TopicMetadata 406 | var firstError error 407 | 408 | for _, shard := range shardedResp { 409 | if shard.Err != nil { 410 | slog.ErrorContext(ctx, "Metadata request shard error", "broker", shard.Meta.NodeID, "error", shard.Err) 411 | if firstError == nil { 412 | firstError = fmt.Errorf("metadata request failed for broker %d: %w", shard.Meta.NodeID, shard.Err) 413 | } 414 | continue // Try other brokers if possible 415 | } 416 | 417 | resp, ok := shard.Resp.(*kmsg.MetadataResponse) 418 | if !ok { 419 | slog.ErrorContext(ctx, "Unexpected response type for metadata request", "broker", shard.Meta.NodeID) 420 | if firstError == nil { 421 | firstError = fmt.Errorf("unexpected metadata response type from broker %d", shard.Meta.NodeID) 422 | } 423 | continue 424 | } 425 | 426 | // Find the requested topic in the response 427 | for _, topic := range resp.Topics { 428 | if topic.Topic != nil && *topic.Topic == topicName { 429 | meta := &TopicMetadata{ 430 | TopicName: topicName, 431 | IsInternal: topic.IsInternal, 432 | ErrorCode: topic.ErrorCode, 433 | } 434 | if topic.ErrorCode != 0 { 435 | errMsg := kerr.ErrorForCode(topic.ErrorCode) 436 | meta.ErrorMessage = errMsg.Error() 437 | // If we get a topic-level error, return it immediately 438 | return meta, fmt.Errorf("error describing topic '%s': %w", topicName, errMsg) 439 | } 440 | 441 | meta.Partitions = make([]PartitionMetadata, 0, len(topic.Partitions)) 442 | for _, p := range topic.Partitions { 443 | pMeta := PartitionMetadata{ 444 | PartitionID: p.Partition, 445 | Leader: p.Leader, 446 | Replicas: p.Replicas, 447 | ISR: p.ISR, 448 | ErrorCode: p.ErrorCode, 449 | } 450 | if p.ErrorCode != 0 { 451 | pErrMsg := kerr.ErrorForCode(p.ErrorCode) 452 | pMeta.ErrorMessage = pErrMsg.Error() 453 | slog.WarnContext(ctx, "Error describing partition", "topic", topicName, "partition", p.Partition, "error", pErrMsg) 454 | } 455 | meta.Partitions = append(meta.Partitions, pMeta) 456 | } 457 | finalMetadata = meta 458 | // Found the topic, no need to check other shards for this topic's data 459 | goto foundTopic 460 | } 461 | } 462 | } 463 | 464 | foundTopic: 465 | if finalMetadata != nil { 466 | return finalMetadata, nil 467 | } 468 | 469 | // If loop finished and finalMetadata is still nil, the topic wasn't found or there was an error 470 | if firstError != nil { 471 | return nil, firstError // Return the first broker communication error encountered 472 | } 473 | 474 | // If no communication errors but topic not found, return a specific error 475 | return nil, fmt.Errorf("topic '%s' not found in metadata response", topicName) 476 | } 477 | 478 | // ListConsumerGroups retrieves a list of consumer groups known by the cluster. 479 | func (c *Client) ListConsumerGroups(ctx context.Context) ([]ConsumerGroupInfo, error) { 480 | req := kmsg.NewListGroupsRequest() 481 | // Requesting groups in all states 482 | // req.StatesFilter = []string{"Stable", "PreparingRebalance", "CompletingRebalance", "Dead", "Empty"} 483 | 484 | // RequestSharded sends the request to all brokers and aggregates results. 485 | // For ListGroups, it's often sufficient to ask one broker (usually the coordinator), 486 | // but RequestSharded handles finding a suitable broker. 487 | shardedResp := c.kgoClient.RequestSharded(ctx, &req) 488 | 489 | var allGroups []ConsumerGroupInfo 490 | var firstError error 491 | processedGroups := make(map[string]struct{}) // Deduplicate groups listed by multiple brokers 492 | 493 | for _, shard := range shardedResp { 494 | if shard.Err != nil { 495 | slog.ErrorContext(ctx, "ListGroups request shard error", "broker", shard.Meta.NodeID, "error", shard.Err) 496 | if firstError == nil { 497 | firstError = fmt.Errorf("list groups request failed for broker %d: %w", shard.Meta.NodeID, shard.Err) 498 | } 499 | continue // Try other brokers 500 | } 501 | 502 | resp, ok := shard.Resp.(*kmsg.ListGroupsResponse) 503 | if !ok { 504 | slog.ErrorContext(ctx, "Unexpected response type for ListGroups request", "broker", shard.Meta.NodeID) 505 | if firstError == nil { 506 | firstError = fmt.Errorf("unexpected list groups response type from broker %d", shard.Meta.NodeID) 507 | } 508 | continue 509 | } 510 | 511 | if resp.ErrorCode != 0 { 512 | errMsg := kerr.ErrorForCode(resp.ErrorCode) 513 | slog.ErrorContext(ctx, "ListGroups request failed with error code", "broker", shard.Meta.NodeID, "error_code", resp.ErrorCode, "error", errMsg) 514 | if firstError == nil { 515 | firstError = fmt.Errorf("list groups request failed on broker %d: %w", shard.Meta.NodeID, errMsg) 516 | } 517 | continue // Potentially try other brokers if it's a transient error 518 | } 519 | 520 | for _, group := range resp.Groups { 521 | if _, exists := processedGroups[group.Group]; !exists { 522 | allGroups = append(allGroups, ConsumerGroupInfo{ 523 | GroupID: group.Group, 524 | State: group.GroupState, // Use GroupState field (added in v11) 525 | // Note: ListGroupsResponse itself doesn't have per-group error codes 526 | }) 527 | processedGroups[group.Group] = struct{}{} 528 | } 529 | } 530 | } 531 | 532 | // If we collected groups, return them even if some brokers failed 533 | if len(allGroups) > 0 { 534 | return allGroups, nil 535 | } 536 | 537 | // If no groups were collected, return the first error encountered 538 | if firstError != nil { 539 | return nil, firstError 540 | } 541 | 542 | // No errors, but no groups found 543 | return []ConsumerGroupInfo{}, nil 544 | } 545 | 546 | // DescribeConsumerGroup retrieves detailed information about a specific consumer group, 547 | // including members and optionally their committed offsets and lag. 548 | func (c *Client) DescribeConsumerGroup(ctx context.Context, groupID string, includeOffsets bool) (*DescribeConsumerGroupResult, error) { 549 | // 1. Describe the Group to get state, protocol, members 550 | descReq := kmsg.NewDescribeGroupsRequest() 551 | descReq.Groups = []string{groupID} 552 | descReq.IncludeAuthorizedOperations = false 553 | 554 | // DescribeGroups needs to be sent to the group coordinator. 555 | // kgo.Client handles finding the coordinator automatically. 556 | resp, err := c.kgoClient.Request(ctx, &descReq) 557 | if err != nil { 558 | return nil, fmt.Errorf("DescribeGroups request failed for group '%s': %w", groupID, err) 559 | } 560 | 561 | descResp, ok := resp.(*kmsg.DescribeGroupsResponse) 562 | if !ok { 563 | return nil, fmt.Errorf("unexpected response type for DescribeGroups request for group '%s'", groupID) 564 | } 565 | 566 | if len(descResp.Groups) != 1 { 567 | // Check if the group just doesn't exist (common case) 568 | if len(descResp.Groups) == 0 { 569 | return nil, fmt.Errorf("consumer group '%s' not found or not described", groupID) 570 | } 571 | return nil, fmt.Errorf("expected exactly one group in DescribeGroups response for group '%s', got %d", groupID, len(descResp.Groups)) 572 | } 573 | 574 | group := descResp.Groups[0] 575 | result := &DescribeConsumerGroupResult{ 576 | GroupID: group.Group, 577 | State: group.State, 578 | ProtocolType: group.ProtocolType, 579 | Protocol: group.Protocol, 580 | // CoordinatorID: group.Coordinator, // Field removed 581 | ErrorCode: group.ErrorCode, 582 | } 583 | 584 | if group.ErrorCode != 0 { 585 | errMsg := kerr.ErrorForCode(group.ErrorCode) 586 | result.ErrorMessage = errMsg.Error() 587 | // Return partial result with error if group description failed 588 | return result, fmt.Errorf("error describing group '%s': %w", groupID, errMsg) 589 | } 590 | 591 | result.Members = make([]GroupMemberInfo, 0, len(group.Members)) 592 | for _, member := range group.Members { 593 | result.Members = append(result.Members, GroupMemberInfo{ 594 | MemberID: member.MemberID, 595 | ClientID: member.ClientID, 596 | ClientHost: member.ClientHost, 597 | }) 598 | } 599 | 600 | // 2. Optionally Fetch Offsets and Calculate Lag 601 | if includeOffsets { 602 | offsets, err := c.fetchGroupOffsetsAndLag(ctx, groupID) 603 | if err != nil { 604 | // Log the error but return the group description info anyway 605 | slog.ErrorContext(ctx, "Failed to fetch offsets/lag for group", "group", groupID, "error", err) 606 | // Append warning to error message if it exists 607 | warningMsg := fmt.Sprintf("(Warning: Failed to fetch offsets: %v)", err) 608 | if result.ErrorMessage != "" { 609 | result.ErrorMessage = result.ErrorMessage + " " + warningMsg 610 | } else { 611 | result.ErrorMessage = warningMsg 612 | } 613 | } else { 614 | result.Offsets = offsets 615 | } 616 | } 617 | 618 | return result, nil 619 | } 620 | 621 | // fetchGroupOffsetsAndLag is a helper to get committed offsets and calculate lag. 622 | func (c *Client) fetchGroupOffsetsAndLag(ctx context.Context, groupID string) ([]PartitionOffsetInfo, error) { 623 | // 1. Fetch committed offsets 624 | offsetReq := kmsg.NewOffsetFetchRequest() 625 | offsetReq.Group = groupID 626 | offsetReq.Topics = nil // Request all topics/partitions for the group 627 | 628 | // OffsetFetch also needs to go to the coordinator. 629 | offsetRespGeneric, err := c.kgoClient.Request(ctx, &offsetReq) 630 | if err != nil { 631 | return nil, fmt.Errorf("OffsetFetch request failed: %w", err) 632 | } 633 | offsetResp, ok := offsetRespGeneric.(*kmsg.OffsetFetchResponse) 634 | if !ok { 635 | return nil, fmt.Errorf("unexpected response type for OffsetFetch request") 636 | } 637 | if offsetResp.ErrorCode != 0 { 638 | errMsg := kerr.ErrorForCode(offsetResp.ErrorCode) 639 | // Handle group-level errors like coordinator loading or not coordinator 640 | return nil, fmt.Errorf("OffsetFetch request failed for group '%s' with error code %d: %w", groupID, offsetResp.ErrorCode, errMsg) 641 | } 642 | 643 | if len(offsetResp.Topics) == 0 { 644 | return []PartitionOffsetInfo{}, nil // No offsets committed or group is inactive/empty 645 | } 646 | 647 | // 2. Prepare to fetch Log End Offsets (LEOs) to calculate lag 648 | leoReqTopics := make(map[string]*kmsg.ListOffsetsRequestTopic) // topic -> request topic struct 649 | partitionMap := make(map[string]map[int32]*PartitionOffsetInfo) // topic -> partition -> info 650 | 651 | for _, topic := range offsetResp.Topics { 652 | if _, exists := partitionMap[topic.Topic]; !exists { 653 | partitionMap[topic.Topic] = make(map[int32]*PartitionOffsetInfo) 654 | } 655 | var reqTopic *kmsg.ListOffsetsRequestTopic 656 | if rt, exists := leoReqTopics[topic.Topic]; !exists { 657 | // Correctly create and assign pointer 658 | newTopic := kmsg.NewListOffsetsRequestTopic() 659 | newTopic.Topic = topic.Topic 660 | reqTopic = &newTopic // Assign address of the new struct 661 | leoReqTopics[topic.Topic] = reqTopic 662 | } else { 663 | reqTopic = rt 664 | } 665 | 666 | for _, p := range topic.Partitions { 667 | // Safely dereference metadata pointer 668 | metadataStr := "" 669 | if p.Metadata != nil { 670 | metadataStr = *p.Metadata 671 | } 672 | 673 | pInfo := PartitionOffsetInfo{ 674 | Topic: topic.Topic, 675 | Partition: p.Partition, 676 | CommitOffset: p.Offset, 677 | Metadata: metadataStr, 678 | ErrorCode: p.ErrorCode, 679 | Lag: -1, // Default to -1 (unknown/error) 680 | } 681 | if p.ErrorCode != 0 { 682 | pErrMsg := kerr.ErrorForCode(p.ErrorCode) 683 | pInfo.ErrorMessage = pErrMsg.Error() 684 | } else if p.Offset == -1 { 685 | // Offset -1 means no offset committed for this partition 686 | pInfo.Lag = 0 // Lag is effectively 0 if nothing is committed 687 | } else { 688 | // Only request LEO if commit offset is valid 689 | reqPartition := kmsg.NewListOffsetsRequestTopicPartition() 690 | reqPartition.Partition = p.Partition 691 | reqPartition.Timestamp = -1 // -1 requests the Log End Offset (LEO) 692 | reqTopic.Partitions = append(reqTopic.Partitions, reqPartition) 693 | } 694 | partitionMap[topic.Topic][p.Partition] = &pInfo 695 | } 696 | } 697 | 698 | // 3. Fetch LEOs using kmsg.ListOffsetsRequest and RequestSharded 699 | if len(leoReqTopics) > 0 { 700 | leoReq := kmsg.NewListOffsetsRequest() 701 | leoReq.ReplicaID = -1 // Standard client request 702 | for _, topicReq := range leoReqTopics { 703 | if len(topicReq.Partitions) > 0 { // Only add topics if they have partitions needing LEO 704 | leoReq.Topics = append(leoReq.Topics, *topicReq) 705 | } 706 | } 707 | 708 | // ListOffsets should be sharded as partitions can be on different brokers 709 | shardedLeoResp := c.kgoClient.RequestSharded(ctx, &leoReq) 710 | var leoErrors []error 711 | leoResults := make(map[string]map[int32]kmsg.ListOffsetsResponseTopicPartition) // topic -> partition -> response partition 712 | 713 | for _, shard := range shardedLeoResp { 714 | if shard.Err != nil { 715 | err := fmt.Errorf("ListOffsets shard request failed for broker %d: %w", shard.Meta.NodeID, shard.Err) 716 | slog.ErrorContext(ctx, err.Error()) 717 | leoErrors = append(leoErrors, err) 718 | continue 719 | } 720 | leoResp, ok := shard.Resp.(*kmsg.ListOffsetsResponse) 721 | if !ok { 722 | err := fmt.Errorf("unexpected response type for ListOffsets from broker %d", shard.Meta.NodeID) 723 | slog.ErrorContext(ctx, err.Error()) 724 | leoErrors = append(leoErrors, err) 725 | continue 726 | } 727 | 728 | for _, topic := range leoResp.Topics { 729 | if _, exists := leoResults[topic.Topic]; !exists { 730 | leoResults[topic.Topic] = make(map[int32]kmsg.ListOffsetsResponseTopicPartition) 731 | } 732 | for _, p := range topic.Partitions { 733 | leoResults[topic.Topic][p.Partition] = p // Store the partition response 734 | } 735 | } 736 | } 737 | 738 | // 4. Calculate Lag using LEO results 739 | for topicName, partitions := range partitionMap { 740 | for partitionID, pInfo := range partitions { 741 | if pInfo.CommitOffset >= 0 && pInfo.ErrorCode == 0 { // Only calculate if commit was valid 742 | if leoTopicResults, topicOk := leoResults[topicName]; topicOk { 743 | if leoPartitionResult, pOk := leoTopicResults[partitionID]; pOk { 744 | if leoPartitionResult.ErrorCode == 0 { 745 | pInfo.Lag = leoPartitionResult.Offset - pInfo.CommitOffset 746 | if pInfo.Lag < 0 { 747 | pInfo.Lag = 0 // Avoid negative lag 748 | } 749 | } else { 750 | leoErrMsg := kerr.ErrorForCode(leoPartitionResult.ErrorCode) 751 | slog.WarnContext(ctx, "Error getting LEO for partition", "group", groupID, "topic", topicName, "partition", partitionID, "error", leoErrMsg) 752 | pInfo.ErrorMessage = fmt.Sprintf("Lag calculation failed: %s", leoErrMsg.Error()) 753 | } 754 | } else { 755 | slog.WarnContext(ctx, "LEO result missing for partition", "group", groupID, "topic", topicName, "partition", partitionID) 756 | pInfo.ErrorMessage = "Lag calculation failed: LEO result missing" 757 | } 758 | } else { 759 | // This case might happen if ListOffsets failed entirely for the brokers holding these partitions 760 | slog.WarnContext(ctx, "LEO results missing for topic", "group", groupID, "topic", topicName) 761 | pInfo.ErrorMessage = "Lag calculation failed: LEO results missing for topic" 762 | } 763 | } 764 | } 765 | } 766 | if len(leoErrors) > 0 { 767 | // Log the errors but don't fail the operation as we might have partial results 768 | slog.WarnContext(ctx, "Some errors occurred when fetching log end offsets", 769 | "group", groupID, "errors_count", len(leoErrors)) 770 | for i, err := range leoErrors { 771 | slog.WarnContext(ctx, "Log end offset error detail", 772 | "group", groupID, "error_index", i, "error", err) 773 | } 774 | } 775 | } 776 | 777 | // 5. Flatten the results 778 | finalOffsets := make([]PartitionOffsetInfo, 0) 779 | for _, partitions := range partitionMap { 780 | for _, pInfo := range partitions { 781 | finalOffsets = append(finalOffsets, *pInfo) 782 | } 783 | } 784 | 785 | return finalOffsets, nil 786 | } 787 | 788 | // DescribeConfigs fetches configuration entries for a given resource (topic or broker). 789 | func (c *Client) DescribeConfigs(ctx context.Context, resourceType ConfigResourceType, resourceName string, configKeys []string) (*DescribeConfigsResult, error) { 790 | slog.InfoContext(ctx, "Describing configs", "resourceType", resourceType, "resourceName", resourceName) 791 | 792 | req := kmsg.NewDescribeConfigsRequest() 793 | resource := kmsg.NewDescribeConfigsRequestResource() 794 | resource.ResourceType = resourceType.ToKmsgResourceType() 795 | resource.ResourceName = resourceName 796 | 797 | // Add specific config keys if provided 798 | if len(configKeys) > 0 { 799 | resource.ConfigNames = append(resource.ConfigNames, configKeys...) 800 | } 801 | 802 | req.Resources = append(req.Resources, resource) 803 | 804 | // DescribeConfigs should be sent to the appropriate broker (controller for broker configs, any broker for topic configs). 805 | // RequestSharded handles routing correctly. 806 | shardedResp := c.kgoClient.RequestSharded(ctx, &req) 807 | 808 | var finalResult *DescribeConfigsResult 809 | var firstError error 810 | 811 | for _, shard := range shardedResp { 812 | if shard.Err != nil { 813 | slog.ErrorContext(ctx, "DescribeConfigs request shard error", "broker", shard.Meta.NodeID, "error", shard.Err) 814 | if firstError == nil { 815 | firstError = fmt.Errorf("DescribeConfigs request failed for broker %d: %w", shard.Meta.NodeID, shard.Err) 816 | } 817 | continue // Try other brokers if possible 818 | } 819 | 820 | resp, ok := shard.Resp.(*kmsg.DescribeConfigsResponse) 821 | if !ok { 822 | slog.ErrorContext(ctx, "Unexpected response type for DescribeConfigs request", "broker", shard.Meta.NodeID) 823 | if firstError == nil { 824 | firstError = fmt.Errorf("unexpected DescribeConfigs response type from broker %d", shard.Meta.NodeID) 825 | } 826 | continue 827 | } 828 | 829 | // Expecting one resource result per shard usually, but loop just in case 830 | for _, resResult := range resp.Resources { 831 | // Match the resource we requested 832 | if resResult.ResourceType == resourceType.ToKmsgResourceType() && resResult.ResourceName == resourceName { 833 | result := &DescribeConfigsResult{ 834 | ResourceType: resourceType.String(), // Convert enum to string 835 | ResourceName: resourceName, 836 | ErrorCode: resResult.ErrorCode, 837 | } 838 | if resResult.ErrorCode != 0 { 839 | errMsg := kerr.ErrorForCode(resResult.ErrorCode) 840 | result.ErrorMessage = errMsg.Error() 841 | // Return immediately if the resource itself had an error 842 | return result, fmt.Errorf("error describing configs for %s '%s': %w", resourceType.String(), resourceName, errMsg) 843 | } 844 | 845 | result.Configs = make([]ConfigEntry, 0, len(resResult.Configs)) 846 | for _, cfg := range resResult.Configs { 847 | // Safely dereference value pointer 848 | valStr := "" 849 | if cfg.Value != nil { 850 | valStr = *cfg.Value 851 | } 852 | result.Configs = append(result.Configs, ConfigEntry{ 853 | Name: cfg.Name, 854 | Value: valStr, 855 | IsDefault: cfg.IsDefault, 856 | IsReadOnly: cfg.ReadOnly, 857 | IsSensitive: cfg.IsSensitive, 858 | Source: cfg.Source.String(), // Convert enum to string 859 | }) 860 | } 861 | finalResult = result 862 | // Found the resource, no need to check other shards for this resource 863 | goto foundResource 864 | } 865 | } 866 | } 867 | 868 | foundResource: 869 | if finalResult != nil { 870 | return finalResult, nil 871 | } 872 | 873 | // If loop finished and finalResult is still nil, the resource wasn't found or there was an error 874 | if firstError != nil { 875 | return nil, firstError // Return the first broker communication error encountered 876 | } 877 | 878 | // If no communication errors but resource not found in any response 879 | return nil, fmt.Errorf("resource %s '%s' not found in DescribeConfigs response", resourceType.String(), resourceName) 880 | } 881 | 882 | // GetClusterOverview retrieves high-level cluster health information. 883 | func (c *Client) GetClusterOverview(ctx context.Context) (*ClusterOverviewResult, error) { 884 | req := kmsg.NewMetadataRequest() 885 | req.Topics = nil // Request metadata for all topics 886 | req.AllowAutoTopicCreation = false 887 | 888 | shardedResp := c.kgoClient.RequestSharded(ctx, &req) 889 | 890 | overview := &ClusterOverviewResult{ 891 | ControllerID: -1, // Default if not found 892 | } 893 | var firstError error 894 | offlineBrokers := make(map[int32]struct{}) 895 | topicSet := make(map[string]struct{}) // To count unique non-internal topics 896 | 897 | for _, shard := range shardedResp { 898 | if shard.Err != nil { 899 | slog.ErrorContext(ctx, "Cluster overview metadata shard error", "broker", shard.Meta.NodeID, "error", shard.Err) 900 | offlineBrokers[shard.Meta.NodeID] = struct{}{} 901 | if firstError == nil { 902 | firstError = fmt.Errorf("metadata request failed for broker %d: %w", shard.Meta.NodeID, shard.Err) 903 | } 904 | continue // Continue processing results from other brokers 905 | } 906 | 907 | resp, ok := shard.Resp.(*kmsg.MetadataResponse) 908 | if !ok { 909 | slog.ErrorContext(ctx, "Unexpected response type for cluster overview metadata", "broker", shard.Meta.NodeID) 910 | if firstError == nil { 911 | firstError = fmt.Errorf("unexpected metadata response type from broker %d", shard.Meta.NodeID) 912 | } 913 | continue 914 | } 915 | 916 | // Update controller ID if found (should be consistent across responses) 917 | if resp.ControllerID != -1 { 918 | overview.ControllerID = resp.ControllerID 919 | } 920 | 921 | // Count brokers (implicitly includes brokers that responded) 922 | // Note: This might slightly undercount if a broker is up but fails the metadata request specifically. 923 | // A more accurate count might require a separate DescribeCluster request if available/needed. 924 | overview.BrokerCount = len(resp.Brokers) 925 | 926 | // Process topics and partitions 927 | for _, topic := range resp.Topics { 928 | // Skip topics with errors at the topic level 929 | if topic.ErrorCode != 0 { 930 | errMsg := kerr.ErrorForCode(topic.ErrorCode) 931 | topicName := "unknown" 932 | if topic.Topic != nil { 933 | topicName = *topic.Topic 934 | } 935 | slog.WarnContext(ctx, "Skipping topic in overview due to error", "topic", topicName, "error", errMsg) 936 | continue 937 | } 938 | 939 | if topic.Topic != nil { 940 | // Count unique non-internal topics 941 | if !topic.IsInternal { 942 | topicSet[*topic.Topic] = struct{}{} 943 | } 944 | 945 | for _, p := range topic.Partitions { 946 | overview.PartitionCount++ 947 | 948 | // Check for offline partitions (e.g., no leader) 949 | if p.ErrorCode != 0 || p.Leader == -1 { 950 | overview.OfflinePartitionsCount++ 951 | if p.ErrorCode != 0 { 952 | pErrMsg := kerr.ErrorForCode(p.ErrorCode) 953 | slog.DebugContext(ctx, "Offline partition detected", "topic", *topic.Topic, "partition", p.Partition, "error", pErrMsg) 954 | } else { 955 | slog.DebugContext(ctx, "Offline partition detected (no leader)", "topic", *topic.Topic, "partition", p.Partition) 956 | } 957 | } 958 | 959 | // Check for under-replicated partitions (ISR < Replicas) 960 | // Only check if the partition is not already offline 961 | if p.ErrorCode == 0 && p.Leader != -1 && len(p.ISR) < len(p.Replicas) { 962 | overview.UnderReplicatedPartitionsCount++ 963 | slog.DebugContext(ctx, "Under-replicated partition detected", "topic", *topic.Topic, "partition", p.Partition, "isr_count", len(p.ISR), "replica_count", len(p.Replicas)) 964 | } 965 | } 966 | } 967 | } 968 | } 969 | 970 | overview.TopicCount = len(topicSet) 971 | 972 | // Populate offline broker IDs 973 | if len(offlineBrokers) > 0 { 974 | overview.OfflineBrokerIDs = make([]int32, 0, len(offlineBrokers)) 975 | for id := range offlineBrokers { 976 | overview.OfflineBrokerIDs = append(overview.OfflineBrokerIDs, id) 977 | } 978 | } 979 | 980 | // If metadata failed entirely for all brokers, return the first error 981 | if overview.ControllerID == -1 && firstError != nil { 982 | overview.ErrorMessage = fmt.Sprintf("Failed to retrieve complete cluster metadata: %v", firstError) 983 | // overview.ErrorCode could be set based on firstError if needed 984 | return overview, firstError 985 | } 986 | 987 | // If some brokers failed, include a warning in the message 988 | if firstError != nil { 989 | overview.ErrorMessage = fmt.Sprintf("Warning: Failed to retrieve metadata from some brokers: %v", firstError) 990 | } 991 | 992 | return overview, nil 993 | } 994 | 995 | // Close gracefully shuts down the Kafka client. 996 | func (c *Client) Close() { 997 | if c.kgoClient != nil { 998 | c.kgoClient.Close() 999 | } 1000 | } 1001 | 1002 | // StringToResourceType implements the interface method by delegating to the package function 1003 | func (c *Client) StringToResourceType(resourceTypeStr string) (ConfigResourceType, error) { 1004 | return StringToResourceType(resourceTypeStr) 1005 | } 1006 | -------------------------------------------------------------------------------- /kafka/client_test.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | "github.com/testcontainers/testcontainers-go/modules/kafka" 14 | "github.com/tuannvm/kafka-mcp-server/config" 15 | "github.com/twmb/franz-go/pkg/kgo" 16 | ) 17 | 18 | var testKafkaContainer *kafka.KafkaContainer 19 | var testKafkaBrokers []string 20 | 21 | // setupKafkaContainer starts a Kafka container for integration tests. 22 | func setupKafkaContainer(ctx context.Context) error { 23 | // Skip if running in CI without Docker, or if explicitly disabled 24 | if os.Getenv("CI") == "true" || os.Getenv("SKIP_KAFKA_TESTS") == "true" { 25 | return fmt.Errorf("skipping Kafka container setup") 26 | } 27 | 28 | // Use default Kafka version provided by testcontainers-go 29 | kc, err := kafka.Run(ctx, 30 | "confluentinc/confluent-local:7.5.0", 31 | kafka.WithClusterID("test-cluster"), 32 | ) 33 | if err != nil { 34 | return fmt.Errorf("failed to start Kafka container: %w", err) 35 | } 36 | testKafkaContainer = kc 37 | 38 | brokers, err := kc.Brokers(ctx) 39 | if err != nil { 40 | teardownKafkaContainer(context.Background()) // Attempt cleanup 41 | return fmt.Errorf("failed to get Kafka brokers: %w", err) 42 | } 43 | testKafkaBrokers = brokers 44 | fmt.Printf("Kafka container started with brokers: %s\n", strings.Join(brokers, ",")) 45 | return nil 46 | } 47 | 48 | // teardownKafkaContainer stops the Kafka container. 49 | func teardownKafkaContainer(ctx context.Context) { 50 | if testKafkaContainer != nil { 51 | fmt.Println("Terminating Kafka container...") 52 | if err := testKafkaContainer.Terminate(ctx); err != nil { 53 | fmt.Printf("Failed to terminate Kafka container: %v\n", err) 54 | } 55 | testKafkaContainer = nil 56 | testKafkaBrokers = nil 57 | } 58 | } 59 | 60 | // TestMain manages the Kafka container lifecycle for the test suite. 61 | func TestMain(m *testing.M) { 62 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) // Give ample time for container setup 63 | defer cancel() 64 | 65 | // Set a flag to indicate if Kafka container is available 66 | var kafkaAvailable = false 67 | 68 | // Try to set up Kafka container 69 | err := setupKafkaContainer(ctx) 70 | if err != nil { 71 | fmt.Printf("WARNING: Could not set up Kafka container, integration tests will be skipped: %v\n", err) 72 | // No need to fail - we'll just skip the tests that need Kafka 73 | } else { 74 | kafkaAvailable = true 75 | } 76 | 77 | // Run tests - they should check testKafkaBrokers/testKafkaContainer being nil 78 | exitCode := m.Run() 79 | 80 | // Teardown only if we successfully set up the container 81 | if kafkaAvailable { 82 | teardownKafkaContainer(context.Background()) 83 | } 84 | 85 | os.Exit(exitCode) 86 | } 87 | 88 | // Helper function to create a test config pointing to the container 89 | func getTestConfig() config.Config { 90 | return config.Config{ 91 | KafkaBrokers: testKafkaBrokers, 92 | KafkaClientID: "test-mcp-client", 93 | // Add SASL/TLS config here if testing those features 94 | } 95 | } 96 | 97 | func TestNewClient_Connection(t *testing.T) { 98 | if testKafkaContainer == nil { 99 | t.Skip("Skipping test: Kafka container not available") 100 | } 101 | require.NotEmpty(t, testKafkaBrokers, "Kafka brokers should be set by TestMain") 102 | 103 | cfg := getTestConfig() 104 | client, err := NewClient(cfg) 105 | 106 | require.NoError(t, err, "NewClient should connect successfully") 107 | require.NotNil(t, client, "Client should not be nil") 108 | defer client.Close() 109 | 110 | // Ping is already done in NewClient, but we can assert no error occurred 111 | assert.NoError(t, err) 112 | } 113 | 114 | func TestProduceConsume_RoundTrip(t *testing.T) { 115 | if testKafkaContainer == nil { 116 | t.Skip("Skipping test: Kafka container not available") 117 | } 118 | require.NotEmpty(t, testKafkaBrokers, "Kafka brokers should be set by TestMain") 119 | 120 | cfg := getTestConfig() 121 | client, err := NewClient(cfg) 122 | require.NoError(t, err) 123 | require.NotNil(t, client) 124 | defer client.Close() 125 | 126 | ctx := context.Background() 127 | topic := "test-produce-consume" 128 | key := "test-key-1" 129 | value := "hello kafka " + time.Now().String() 130 | 131 | // Produce 132 | err = client.ProduceMessage(ctx, topic, []byte(key), []byte(value)) 133 | require.NoError(t, err, "ProduceMessage should succeed") 134 | 135 | // Consume 136 | // Give Kafka a moment to process the message 137 | time.Sleep(2 * time.Second) 138 | 139 | // ConsumeMessages requires the client to be configured as a consumer group 140 | // NewClient already configures ConsumerGroup(cfg.KafkaClientID) 141 | messages, err := client.ConsumeMessages(ctx, []string{topic}, 1) 142 | require.NoError(t, err, "ConsumeMessages should succeed") 143 | require.Len(t, messages, 1, "Should consume exactly one message") 144 | 145 | consumedMsg := messages[0] 146 | assert.Equal(t, topic, consumedMsg.Topic) 147 | assert.Equal(t, key, consumedMsg.Key) 148 | assert.Equal(t, value, consumedMsg.Value) 149 | assert.True(t, consumedMsg.Offset >= 0, "Offset should be non-negative") 150 | assert.True(t, consumedMsg.Timestamp > 0, "Timestamp should be positive") 151 | 152 | // Optional: Commit offsets if auto-commit is disabled (it is in NewClient) 153 | err = client.kgoClient.CommitUncommittedOffsets(ctx) 154 | assert.NoError(t, err, "Committing offsets should succeed") 155 | } 156 | 157 | func TestListTopics(t *testing.T) { 158 | if testKafkaContainer == nil { 159 | t.Skip("Skipping test: Kafka container not available") 160 | } 161 | require.NotEmpty(t, testKafkaBrokers, "Kafka brokers should be set by TestMain") 162 | 163 | cfg := getTestConfig() 164 | // Use a separate client ID to avoid consumer group interactions if not needed 165 | cfg.KafkaClientID = "test-list-topics-client" 166 | // Re-create client without consumer group for listing 167 | clientOpts := []kgo.Opt{ 168 | kgo.SeedBrokers(cfg.KafkaBrokers...), 169 | kgo.ClientID(cfg.KafkaClientID), 170 | kgo.WithLogger(kgo.BasicLogger(os.Stderr, kgo.LogLevelWarn, nil)), // Reduce log level for this test 171 | } 172 | kgoClient, err := kgo.NewClient(clientOpts...) 173 | require.NoError(t, err) 174 | client := &Client{kgoClient: kgoClient} 175 | defer client.Close() 176 | 177 | ctx := context.Background() 178 | 179 | // Produce to a topic first to ensure it exists 180 | topicToCreate := "topic-for-listing-test" 181 | err = client.ProduceMessage(ctx, topicToCreate, nil, []byte("dummy message")) 182 | require.NoError(t, err) 183 | time.Sleep(2 * time.Second) // Give Kafka time to create/register the topic 184 | 185 | topics, err := client.ListTopics(ctx) 186 | require.NoError(t, err, "ListTopics should succeed") 187 | require.NotEmpty(t, topics, "List of topics should not be empty") 188 | 189 | // Check if our created topic is in the list 190 | found := false 191 | for _, topic := range topics { 192 | if topic == topicToCreate { 193 | found = true 194 | break 195 | } 196 | } 197 | assert.True(t, found, "Expected to find topic '%s' in the list %v", topicToCreate, topics) 198 | } 199 | -------------------------------------------------------------------------------- /kafka/interface.go: -------------------------------------------------------------------------------- 1 | // Package kafka provides Kafka client functionality for the MCP server. 2 | package kafka 3 | 4 | import ( 5 | "context" 6 | ) 7 | 8 | // KafkaClient defines the interface for Kafka operations. 9 | // This interface makes it easier to create mock implementations for testing. 10 | type KafkaClient interface { 11 | // ProduceMessage sends a message to the specified Kafka topic. 12 | ProduceMessage(ctx context.Context, topic string, key, value []byte) error 13 | 14 | // ConsumeMessages polls for messages from the specified topics. 15 | // It returns a slice of consumed messages or an error. 16 | ConsumeMessages(ctx context.Context, topics []string, maxMessages int) ([]Message, error) 17 | 18 | // ListTopics retrieves a list of topic names from the Kafka cluster. 19 | ListTopics(ctx context.Context) ([]string, error) 20 | 21 | // ListBrokers retrieves a list of broker addresses from the Kafka cluster. 22 | ListBrokers(ctx context.Context) ([]string, error) // Added ListBrokers method 23 | 24 | // DescribeTopic retrieves detailed metadata for a specific topic. 25 | DescribeTopic(ctx context.Context, topicName string) (*TopicMetadata, error) 26 | 27 | // ListConsumerGroups retrieves a list of consumer groups known by the cluster. 28 | ListConsumerGroups(ctx context.Context) ([]ConsumerGroupInfo, error) 29 | 30 | // DescribeConsumerGroup retrieves detailed information about a specific consumer group, 31 | // including members and optionally their committed offsets and lag. 32 | DescribeConsumerGroup(ctx context.Context, groupID string, includeOffsets bool) (*DescribeConsumerGroupResult, error) 33 | 34 | // DescribeConfigs retrieves configuration for a Kafka resource 35 | DescribeConfigs(ctx context.Context, resourceType ConfigResourceType, resourceName string, configKeys []string) (*DescribeConfigsResult, error) 36 | 37 | // GetClusterOverview retrieves high-level cluster health information. 38 | GetClusterOverview(ctx context.Context) (*ClusterOverviewResult, error) 39 | 40 | // StringToResourceType converts a string resource type to its corresponding ConfigResourceType 41 | StringToResourceType(resourceTypeStr string) (ConfigResourceType, error) 42 | 43 | // Close gracefully shuts down the Kafka client. 44 | Close() 45 | } 46 | 47 | // Ensure Client implements KafkaClient 48 | var _ KafkaClient = (*Client)(nil) 49 | -------------------------------------------------------------------------------- /kafka/resource_types.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | ) 8 | 9 | // ConfigResourceType represents the type of Kafka resource for configuration operations 10 | type ConfigResourceType int 11 | 12 | // ConfigResourceType constants - matching kmsg.ConfigResourceType values 13 | const ( 14 | ConfigResourceTypeUnknown ConfigResourceType = 0 15 | ConfigResourceTypeTopic ConfigResourceType = 2 16 | ConfigResourceTypeBroker ConfigResourceType = 4 17 | ConfigResourceTypeGroup ConfigResourceType = 3 18 | ConfigResourceTypeUser ConfigResourceType = 6 19 | ConfigResourceTypeClientQuota ConfigResourceType = 7 20 | ) 21 | 22 | // StringToResourceType converts a string resource type to its corresponding ConfigResourceType 23 | func StringToResourceType(resourceTypeStr string) (ConfigResourceType, error) { 24 | switch resourceTypeStr { 25 | case "topic": 26 | return ConfigResourceTypeTopic, nil 27 | case "broker": 28 | return ConfigResourceTypeBroker, nil 29 | case "group": 30 | return ConfigResourceTypeGroup, nil 31 | case "user": 32 | return ConfigResourceTypeUser, nil 33 | case "client_quota": 34 | return ConfigResourceTypeClientQuota, nil 35 | default: 36 | return ConfigResourceTypeUnknown, fmt.Errorf("unknown resource type: %s", resourceTypeStr) 37 | } 38 | } 39 | 40 | // ToKmsgResourceType converts internal ConfigResourceType to franz-go kmsg.ConfigResourceType 41 | // This keeps franz-go dependency isolated to the kafka package 42 | func (rt ConfigResourceType) ToKmsgResourceType() kmsg.ConfigResourceType { 43 | return kmsg.ConfigResourceType(rt) 44 | } 45 | 46 | // String returns the string representation of the resource type 47 | func (rt ConfigResourceType) String() string { 48 | switch rt { 49 | case ConfigResourceTypeTopic: 50 | return "topic" 51 | case ConfigResourceTypeBroker: 52 | return "broker" 53 | case ConfigResourceTypeGroup: 54 | return "group" 55 | case ConfigResourceTypeUser: 56 | return "user" 57 | case ConfigResourceTypeClientQuota: 58 | return "client_quota" 59 | default: 60 | return "unknown" 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /mcp/prompts.go: -------------------------------------------------------------------------------- 1 | // filepath: /Users/tuannvm/Projects/cli/kafka-mcp-server/mcp/prompts.go 2 | package mcp 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "log/slog" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/mark3labs/mcp-go/mcp" 12 | "github.com/mark3labs/mcp-go/server" 13 | "github.com/tuannvm/kafka-mcp-server/kafka" 14 | ) 15 | 16 | // RegisterPrompts registers prompts with the MCP server. 17 | func RegisterPrompts(s *server.MCPServer, kafkaClient kafka.KafkaClient) { 18 | // Register cluster overview prompt 19 | clusterOverviewPrompt := mcp.Prompt{ 20 | Name: "kafka_cluster_overview", 21 | Description: "Provides a summary of Kafka cluster health and metrics", 22 | Arguments: []mcp.PromptArgument{ 23 | { 24 | Name: "cluster", 25 | Description: "The Kafka cluster name", 26 | Required: true, 27 | }, 28 | }, 29 | } 30 | s.AddPrompt(clusterOverviewPrompt, func(ctx context.Context, req mcp.GetPromptRequest) (*mcp.GetPromptResult, error) { 31 | slog.InfoContext(ctx, "Executing cluster overview prompt") 32 | 33 | // Get overview data 34 | overview, err := kafkaClient.GetClusterOverview(ctx) 35 | if err != nil { 36 | return handlePromptError(ctx, "Error retrieving cluster overview", err, "Failed to get cluster overview") 37 | } 38 | 39 | // Format the response 40 | response := formatResponseHeader("Kafka Cluster Overview") 41 | 42 | response += fmt.Sprintf("- **Broker Count**: %d\n"+ 43 | "- **Active Controller ID**: %d\n"+ 44 | "- **Total Topics**: %d\n"+ 45 | "- **Total Partitions**: %d\n"+ 46 | "- **Under-Replicated Partitions**: %d\n"+ 47 | "- **Offline Partitions**: %d\n\n", 48 | overview.BrokerCount, 49 | overview.ControllerID, 50 | overview.TopicCount, 51 | overview.PartitionCount, 52 | overview.UnderReplicatedPartitionsCount, 53 | overview.OfflinePartitionsCount) 54 | 55 | // Check for warning conditions 56 | critical := overview.OfflinePartitionsCount > 0 || overview.ControllerID == -1 || len(overview.OfflineBrokerIDs) > 0 57 | warning := overview.UnderReplicatedPartitionsCount > 0 58 | 59 | if len(overview.OfflineBrokerIDs) > 0 { 60 | brokerIDsStr := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(overview.OfflineBrokerIDs)), ", "), "[]") 61 | response += fmt.Sprintf("⚠️ **Warning**: %d brokers are offline (IDs: %s)\n\n", 62 | len(overview.OfflineBrokerIDs), brokerIDsStr) 63 | } 64 | 65 | if overview.UnderReplicatedPartitionsCount > 0 { 66 | response += fmt.Sprintf("⚠️ **Warning**: %d partitions are under-replicated\n\n", 67 | overview.UnderReplicatedPartitionsCount) 68 | } 69 | 70 | if overview.OfflinePartitionsCount > 0 { 71 | response += fmt.Sprintf("🚨 **Critical**: %d partitions are offline\n\n", 72 | overview.OfflinePartitionsCount) 73 | } 74 | 75 | if overview.ControllerID == -1 { 76 | response += "🚨 **Critical**: No active controller found in the cluster\n\n" 77 | } 78 | 79 | // Add overall health status 80 | emoji, statusText := formatHealthStatus(critical, warning) 81 | response += fmt.Sprintf("**Overall Status**: %s %s\n", emoji, statusText) 82 | 83 | return createSuccessResponse("Kafka Cluster Overview", response) 84 | }) 85 | 86 | // Register health check prompt 87 | healthCheckPrompt := mcp.Prompt{ 88 | Name: "kafka_health_check", 89 | Description: "Run a comprehensive health check on the Kafka cluster", 90 | Arguments: []mcp.PromptArgument{ 91 | { 92 | Name: "cluster", 93 | Description: "The Kafka cluster name", 94 | Required: true, 95 | }, 96 | }, 97 | } 98 | s.AddPrompt(healthCheckPrompt, func(ctx context.Context, req mcp.GetPromptRequest) (*mcp.GetPromptResult, error) { 99 | slog.InfoContext(ctx, "Executing health check prompt") 100 | 101 | // Get overview data for basic health info 102 | overview, err := kafkaClient.GetClusterOverview(ctx) 103 | if err != nil { 104 | return handlePromptError(ctx, "Error running health check", err, "Failed to get cluster overview for health check") 105 | } 106 | 107 | // Begin building the health check report 108 | response := formatResponseHeader("Kafka Cluster Health Check Report") 109 | 110 | // Add broker status section 111 | response += "## Broker Status\n\n" 112 | if len(overview.OfflineBrokerIDs) > 0 { 113 | brokerIDsStr := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(overview.OfflineBrokerIDs)), ", "), "[]") 114 | response += fmt.Sprintf("- 🚨 **%d of %d brokers are OFFLINE** (IDs: %s)\n", 115 | len(overview.OfflineBrokerIDs), overview.BrokerCount, brokerIDsStr) 116 | response += "- **Recommendation**: Investigate why these brokers are offline and restart if necessary\n\n" 117 | } else { 118 | response += fmt.Sprintf("- ✅ **All %d brokers are online**\n\n", overview.BrokerCount) 119 | } 120 | 121 | // Add controller status 122 | response += "## Controller Status\n\n" 123 | if overview.ControllerID == -1 { 124 | response += "- 🚨 **No active controller found**\n" 125 | response += "- **Recommendation**: Check broker logs to identify controller election issues\n\n" 126 | } else { 127 | response += fmt.Sprintf("- ✅ **Active controller**: Broker %d\n\n", overview.ControllerID) 128 | } 129 | 130 | // Add partition health 131 | response += "## Partition Health\n\n" 132 | healthIssues := false 133 | 134 | if overview.UnderReplicatedPartitionsCount > 0 { 135 | response += fmt.Sprintf("- ⚠️ **%d under-replicated partitions detected**\n", overview.UnderReplicatedPartitionsCount) 136 | response += "- **Recommendation**: Check broker health and network connectivity\n" 137 | healthIssues = true 138 | } 139 | 140 | if overview.OfflinePartitionsCount > 0 { 141 | response += fmt.Sprintf("- 🚨 **%d offline partitions detected**\n", overview.OfflinePartitionsCount) 142 | response += "- **Recommendation**: Check if leader replicas are available for these partitions\n" 143 | healthIssues = true 144 | } 145 | 146 | if !healthIssues { 147 | response += fmt.Sprintf("- ✅ **All %d partitions are healthy**\n", overview.PartitionCount) 148 | } 149 | 150 | response += "\n" 151 | 152 | // Add consumer group status 153 | response += "## Consumer Group Status\n\n" 154 | 155 | // Get consumer groups 156 | groups, groupErr := kafkaClient.ListConsumerGroups(ctx) 157 | if groupErr != nil { 158 | response += fmt.Sprintf("- ⚠️ **Unable to check consumer groups**: %s\n\n", groupErr.Error()) 159 | } else { 160 | // Track groups with high lag 161 | groupsWithHighLag := 0 162 | highLagThreshold := int64(10000) // Consider high lag if over 10K messages 163 | 164 | for _, groupInfo := range groups { 165 | // Get detailed info including lag 166 | descResult, descErr := kafkaClient.DescribeConsumerGroup(ctx, groupInfo.GroupID, true) 167 | if descErr == nil && descResult.ErrorCode == 0 { 168 | for _, offsetInfo := range descResult.Offsets { 169 | if offsetInfo.Lag > highLagThreshold { 170 | groupsWithHighLag++ 171 | break // Only count each group once 172 | } 173 | } 174 | } 175 | } 176 | 177 | response += fmt.Sprintf("- **Total consumer groups**: %d\n", len(groups)) 178 | 179 | if groupsWithHighLag > 0 { 180 | response += fmt.Sprintf("- ⚠️ **%d consumer groups have high lag** (>10K messages)\n", groupsWithHighLag) 181 | response += "- **Recommendation**: Check consumer performance and consider scaling consumers\n\n" 182 | } else { 183 | response += "- ✅ **No consumer groups with high lag detected**\n\n" 184 | } 185 | } 186 | 187 | // Add overall health status 188 | response += "## Overall Health Assessment\n\n" 189 | 190 | // Determine overall health status 191 | critical := overview.OfflinePartitionsCount > 0 || overview.ControllerID == -1 || len(overview.OfflineBrokerIDs) > 0 192 | warning := overview.UnderReplicatedPartitionsCount > 0 || ((groups != nil) && len(groups) > 0) 193 | 194 | emoji, statusText := formatHealthStatus(critical, warning) 195 | response += fmt.Sprintf("%s **%s**: ", emoji, strings.ToUpper(statusText)) 196 | 197 | if critical { 198 | response += "The cluster has serious issues that require immediate attention.\n" 199 | } else if warning { 200 | response += "The cluster has issues that should be investigated soon.\n" 201 | } else { 202 | response += "All systems are operating normally.\n" 203 | } 204 | 205 | return createSuccessResponse("Kafka Health Check Report", response) 206 | }) 207 | 208 | // Register under-replicated partitions prompt 209 | underReplicatedPrompt := mcp.Prompt{ 210 | Name: "kafka_under_replicated_partitions", 211 | Description: "List topics and partitions where ISR count is less than replication factor", 212 | Arguments: []mcp.PromptArgument{ 213 | { 214 | Name: "cluster", 215 | Description: "The Kafka cluster name", 216 | Required: true, 217 | }, 218 | }, 219 | } 220 | s.AddPrompt(underReplicatedPrompt, func(ctx context.Context, req mcp.GetPromptRequest) (*mcp.GetPromptResult, error) { 221 | slog.InfoContext(ctx, "Executing under-replicated partitions prompt") 222 | 223 | // Get overview for quick check 224 | overview, err := kafkaClient.GetClusterOverview(ctx) 225 | if err != nil { 226 | return handlePromptError(ctx, "Error retrieving under-replicated partitions", err, "Failed to get cluster overview for URP check") 227 | } 228 | 229 | // Begin building the response 230 | response := formatResponseHeader("Under-Replicated Partitions Report") 231 | 232 | if overview.UnderReplicatedPartitionsCount == 0 { 233 | response += "✅ **Good news!** No under-replicated partitions found.\n" 234 | return createSuccessResponse("No under-replicated partitions found", response) 235 | } 236 | 237 | response += fmt.Sprintf("⚠️ **Found %d under-replicated partitions**\n\n", overview.UnderReplicatedPartitionsCount) 238 | 239 | // Get list of all topics 240 | topics, err := kafkaClient.ListTopics(ctx) 241 | if err != nil { 242 | return handlePromptError(ctx, "Error listing topics", err, "Failed to list topics") 243 | } 244 | 245 | // Add table header 246 | response += "| Topic | Partition | Leader | Replica Count | ISR Count | Missing Replicas |\n" 247 | response += "|-------|-----------|--------|---------------|-----------|------------------|\n" 248 | 249 | // Track whether we found any URPs 250 | foundURPs := false 251 | 252 | // For each topic, get details and check for URPs 253 | for _, topic := range topics { 254 | topicInfo, err := kafkaClient.DescribeTopic(ctx, topic) 255 | if err != nil { 256 | slog.WarnContext(ctx, "Error getting topic details", "topic", topic, "error", err) 257 | continue 258 | } 259 | 260 | for _, partition := range topicInfo.Partitions { 261 | if len(partition.ISR) < len(partition.Replicas) { 262 | foundURPs = true 263 | 264 | // Format missing replicas 265 | missingReplicas := []int32{} 266 | replicaMap := make(map[int32]bool) 267 | 268 | // Convert ISR to map for quicker lookup 269 | for _, isr := range partition.ISR { 270 | replicaMap[isr] = true 271 | } 272 | 273 | // Find which replicas are not in ISR 274 | for _, replica := range partition.Replicas { 275 | if !replicaMap[replica] { 276 | missingReplicas = append(missingReplicas, replica) 277 | } 278 | } 279 | 280 | missingReplicasStr := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(missingReplicas)), ", "), "[]") 281 | 282 | // Add row to table 283 | response += fmt.Sprintf("| %s | %d | %d | %d | %d | %s |\n", 284 | topic, partition.PartitionID, partition.Leader, len(partition.Replicas), 285 | len(partition.ISR), missingReplicasStr) 286 | } 287 | } 288 | } 289 | 290 | // Add explanatory content if we found URPs 291 | if foundURPs { 292 | response += "\n## Possible Causes\n\n" 293 | response += "Under-replicated partitions occur when one or more replicas are not in sync with the leader. Common causes include:\n\n" 294 | response += "- **Broker failure or network partition**\n" 295 | response += "- **High load on brokers**\n" 296 | response += "- **Insufficient disk space**\n" 297 | response += "- **Network bandwidth limitations**\n" 298 | response += "- **Misconfigured topic replication factor**\n\n" 299 | 300 | response += "## Recommendations\n\n" 301 | response += "1. **Check broker health** for any offline or struggling brokers\n" 302 | response += "2. **Verify network connectivity** between brokers\n" 303 | response += "3. **Monitor disk space** on broker nodes\n" 304 | response += "4. **Review broker logs** for detailed error messages\n" 305 | response += "5. **Consider increasing replication timeouts** if network is slow\n" 306 | } else { 307 | // This is unlikely, but handle case where overview reported URPs but we didn't find any 308 | response += "\n⚠️ Overview reported under-replicated partitions, but none were found during detailed scan. The condition may have resolved itself.\n" 309 | } 310 | 311 | return createSuccessResponse("Under-Replicated Partitions Report", response) 312 | }) 313 | 314 | // Register consumer lag report prompt 315 | consumerLagPrompt := mcp.Prompt{ 316 | Name: "kafka_consumer_lag_report", 317 | Description: "Provide a detailed report on consumer lag across all consumer groups", 318 | Arguments: []mcp.PromptArgument{ 319 | { 320 | Name: "cluster", 321 | Description: "The Kafka cluster name", 322 | Required: true, 323 | }, 324 | { 325 | Name: "threshold", 326 | Description: "Lag threshold for highlighting high lag (default: 1000)", 327 | Required: false, 328 | }, 329 | }, 330 | } 331 | s.AddPrompt(consumerLagPrompt, func(ctx context.Context, req mcp.GetPromptRequest) (*mcp.GetPromptResult, error) { 332 | slog.InfoContext(ctx, "Executing consumer lag report prompt") 333 | 334 | // Get threshold from arguments 335 | var lagThreshold int64 = 1000 // Default threshold 336 | if thresholdArg, ok := req.Params.Arguments["threshold"]; ok && thresholdArg != "" { 337 | threshold, err := strconv.ParseInt(thresholdArg, 10, 64) 338 | if err == nil && threshold >= 0 { 339 | lagThreshold = threshold 340 | } else { 341 | slog.WarnContext(ctx, "Invalid lag threshold, using default", "threshold", thresholdArg, "default", lagThreshold) 342 | } 343 | } 344 | 345 | // Get list of consumer groups 346 | groups, err := kafkaClient.ListConsumerGroups(ctx) 347 | if err != nil { 348 | return handlePromptError(ctx, "Error listing consumer groups", err, "Failed to list consumer groups") 349 | } 350 | 351 | // Begin building the response 352 | response := formatResponseHeader("Kafka Consumer Lag Report") 353 | response += fmt.Sprintf("**Lag Threshold**: %d messages\n\n", lagThreshold) 354 | 355 | if len(groups) == 0 { 356 | response += "No active consumer groups found.\n" 357 | return createSuccessResponse("No active consumer groups", response) 358 | } 359 | 360 | response += fmt.Sprintf("Found %d consumer group(s)\n\n", len(groups)) 361 | 362 | // Track groups with high lag for summary 363 | groupsWithHighLag := 0 364 | highLagDetails := []map[string]interface{}{} 365 | 366 | // Consumer group summary table 367 | response += "## Consumer Group Summary\n\n" 368 | response += "| Group ID | State | Members | Topics | Total Lag | High Lag |\n" 369 | response += "|----------|-------|---------|--------|-----------|----------|\n" 370 | 371 | for _, groupInfo := range groups { 372 | // Describe each group to get offset/lag info 373 | descResult, descErr := kafkaClient.DescribeConsumerGroup(ctx, groupInfo.GroupID, true) // includeOffsets = true 374 | 375 | var state = "Unknown" 376 | members := 0 377 | topics := []string{} 378 | totalLag := int64(0) 379 | hasHighLag := false 380 | 381 | if descErr == nil && descResult.ErrorCode == 0 { 382 | state = descResult.State 383 | members = len(descResult.Members) 384 | 385 | // Process offset/lag information 386 | topicSet := make(map[string]bool) 387 | 388 | for _, offsetInfo := range descResult.Offsets { 389 | topicSet[offsetInfo.Topic] = true 390 | totalLag += offsetInfo.Lag 391 | 392 | if offsetInfo.Lag > lagThreshold { 393 | hasHighLag = true 394 | 395 | // Add to high lag details 396 | highLagDetails = append(highLagDetails, map[string]interface{}{ 397 | "group_id": groupInfo.GroupID, 398 | "topic": offsetInfo.Topic, 399 | "partition": offsetInfo.Partition, 400 | "current_offset": offsetInfo.CommitOffset, 401 | "log_end_offset": offsetInfo.CommitOffset + offsetInfo.Lag, // Approximate 402 | "lag": offsetInfo.Lag, 403 | }) 404 | } 405 | } 406 | 407 | // Convert topic set to slice 408 | for topic := range topicSet { 409 | topics = append(topics, topic) 410 | } 411 | 412 | if hasHighLag { 413 | groupsWithHighLag++ 414 | } 415 | } 416 | 417 | // Format topics list for display 418 | topicsStr := strings.Join(topics, ", ") 419 | if len(topicsStr) > 30 { 420 | topicsStr = topicsStr[:27] + "..." 421 | } 422 | 423 | // Add row to table 424 | highLagStatus := "✅ No" 425 | if hasHighLag { 426 | highLagStatus = "⚠️ Yes" 427 | } 428 | 429 | response += fmt.Sprintf("| %s | %s | %d | %s | %d | %s |\n", 430 | groupInfo.GroupID, state, members, topicsStr, totalLag, highLagStatus) 431 | } 432 | 433 | // Add high lag details if any exist 434 | if len(highLagDetails) > 0 { 435 | response += "\n## High Lag Details\n\n" 436 | response += "| Group ID | Topic | Partition | Current Offset | Log End Offset | Lag |\n" 437 | response += "|----------|-------|-----------|----------------|----------------|-----|\n" 438 | 439 | // Sort details by lag (descending) - in a real implementation we'd sort the slice 440 | // For simplicity, we're just iterating through the unsorted slice 441 | for _, detail := range highLagDetails { 442 | response += fmt.Sprintf("| %s | %s | %d | %d | %d | %d |\n", 443 | detail["group_id"], detail["topic"], detail["partition"].(int), 444 | detail["current_offset"].(int64), detail["log_end_offset"].(int64), 445 | detail["lag"].(int64)) 446 | } 447 | } 448 | 449 | // Add recommendations if high lag was found 450 | if groupsWithHighLag > 0 { 451 | response += "\n## Recommendations\n\n" 452 | response += "Consumer groups with high lag may indicate performance issues or insufficient consumer capacity. Consider:\n\n" 453 | response += "1. **Increase consumer instances** to process messages faster\n" 454 | response += "2. **Check for consumer errors or bottlenecks** in consumer application logs\n" 455 | response += "3. **Verify producer rate** isn't abnormally high\n" 456 | response += "4. **Review consumer configuration** for performance tuning\n" 457 | response += "5. **Consider topic partitioning** to allow more parallel processing\n" 458 | } else { 459 | response += "\n✅ All consumer groups are keeping up with producers.\n" 460 | } 461 | 462 | return createSuccessResponse("Consumer Lag Report", response) 463 | }) 464 | } 465 | -------------------------------------------------------------------------------- /mcp/resources.go: -------------------------------------------------------------------------------- 1 | // Package mcp provides the MCP server functionality for Kafka. 2 | package mcp 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "log/slog" 8 | "strconv" 9 | 10 | "github.com/mark3labs/mcp-go/mcp" 11 | "github.com/mark3labs/mcp-go/server" 12 | "github.com/tuannvm/kafka-mcp-server/kafka" 13 | ) 14 | 15 | // ResourceContentsFunc defines a function that returns the contents of a resource 16 | type ResourceContentsFunc func(ctx context.Context, uri string) ([]byte, error) 17 | 18 | // ResourceHandlerFunc is a wrapper around ResourceContentsFunc to match the server.ResourceHandlerFunc signature 19 | func resourceHandlerFuncWrapper(contentsFunc ResourceContentsFunc) server.ResourceHandlerFunc { 20 | return func(ctx context.Context, req mcp.ReadResourceRequest) ([]mcp.ResourceContents, error) { 21 | data, err := contentsFunc(ctx, req.Params.URI) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | // Convert to text resource contents 27 | textContent := mcp.TextResourceContents{ 28 | URI: req.Params.URI, 29 | MIMEType: "application/json", 30 | Text: string(data), 31 | } 32 | return []mcp.ResourceContents{textContent}, nil 33 | } 34 | } 35 | 36 | // RegisterResources registers MCP resources with the server. 37 | func RegisterResources(s *server.MCPServer, kafkaClient kafka.KafkaClient) { 38 | // Register core resources 39 | s.AddResource(mcp.Resource{ 40 | URI: "kafka-mcp://{cluster}/overview", 41 | Name: "Cluster Overview", 42 | Description: "Summary of Kafka cluster health and metrics", 43 | MIMEType: "application/json", 44 | }, resourceHandlerFuncWrapper(clusterOverviewResource(kafkaClient))) 45 | 46 | s.AddResource(mcp.Resource{ 47 | URI: "kafka-mcp://{cluster}/health-check", 48 | Name: "Health Check", 49 | Description: "Comprehensive health assessment of the Kafka cluster", 50 | MIMEType: "application/json", 51 | }, resourceHandlerFuncWrapper(healthCheckResource(kafkaClient))) 52 | 53 | s.AddResource(mcp.Resource{ 54 | URI: "kafka-mcp://{cluster}/under-replicated-partitions", 55 | Name: "Under-Replicated Partitions", 56 | Description: "Detailed report of under-replicated partitions", 57 | MIMEType: "application/json", 58 | }, resourceHandlerFuncWrapper(underReplicatedPartitionsResource(kafkaClient))) 59 | 60 | s.AddResource(mcp.Resource{ 61 | URI: "kafka-mcp://{cluster}/consumer-lag-report", 62 | Name: "Consumer Lag Report", 63 | Description: "Analysis of consumer group lag across the cluster", 64 | MIMEType: "application/json", 65 | }, resourceHandlerFuncWrapper(consumerLagReportResource(kafkaClient))) 66 | } 67 | 68 | // clusterOverviewResource returns a resource for the cluster overview 69 | func clusterOverviewResource(kafkaClient kafka.KafkaClient) ResourceContentsFunc { 70 | return func(ctx context.Context, uri string) ([]byte, error) { 71 | slog.InfoContext(ctx, "Fetching cluster overview resource", "uri", uri) 72 | 73 | // Get overview data 74 | overview, err := kafkaClient.GetClusterOverview(ctx) 75 | if err != nil { 76 | return handleResourceError(ctx, err, "Failed to get cluster overview") 77 | } 78 | 79 | // Create a structured response 80 | response := createBaseResourceResponse() 81 | 82 | // Add cluster specific fields 83 | response["broker_count"] = overview.BrokerCount 84 | response["controller_id"] = overview.ControllerID 85 | response["topic_count"] = overview.TopicCount 86 | response["partition_count"] = overview.PartitionCount 87 | response["under_replicated_partitions"] = overview.UnderReplicatedPartitionsCount 88 | response["offline_partitions"] = overview.OfflinePartitionsCount 89 | response["offline_broker_ids"] = overview.OfflineBrokerIDs 90 | response["health_status"] = getHealthStatusString( 91 | overview.OfflinePartitionsCount > 0, 92 | overview.ControllerID == -1, 93 | len(overview.OfflineBrokerIDs) > 0, 94 | overview.UnderReplicatedPartitionsCount > 0, 95 | ) 96 | 97 | return json.Marshal(response) 98 | } 99 | } 100 | 101 | // healthCheckResource returns a resource for detailed cluster health assessment 102 | func healthCheckResource(kafkaClient kafka.KafkaClient) ResourceContentsFunc { 103 | return func(ctx context.Context, uri string) ([]byte, error) { 104 | slog.InfoContext(ctx, "Fetching health check resource", "uri", uri) 105 | 106 | // Get overview for basic health info 107 | overview, err := kafkaClient.GetClusterOverview(ctx) 108 | if err != nil { 109 | return handleResourceError(ctx, err, "Failed to get cluster overview for health check") 110 | } 111 | 112 | // Get consumer groups 113 | groups, groupErr := kafkaClient.ListConsumerGroups(ctx) 114 | 115 | // Track groups with high lag 116 | groupsWithHighLag := 0 117 | highLagThreshold := int64(10000) // Consider high lag if over 10K messages 118 | 119 | if groupErr == nil { 120 | for _, groupInfo := range groups { 121 | // Get detailed info including lag 122 | descResult, descErr := kafkaClient.DescribeConsumerGroup(ctx, groupInfo.GroupID, true) 123 | if descErr == nil && descResult.ErrorCode == 0 { 124 | for _, offsetInfo := range descResult.Offsets { 125 | if offsetInfo.Lag > highLagThreshold { 126 | groupsWithHighLag++ 127 | break // Only count each group once 128 | } 129 | } 130 | } 131 | } 132 | } 133 | 134 | // Create a structured response 135 | response := createBaseResourceResponse() 136 | 137 | // Add broker status 138 | response["broker_status"] = map[string]interface{}{ 139 | "total_brokers": overview.BrokerCount, 140 | "offline_brokers": len(overview.OfflineBrokerIDs), 141 | "offline_broker_ids": overview.OfflineBrokerIDs, 142 | "status": getStatus(len(overview.OfflineBrokerIDs) > 0, "critical", "healthy"), 143 | } 144 | 145 | // Add controller status 146 | response["controller_status"] = map[string]interface{}{ 147 | "controller_id": overview.ControllerID, 148 | "status": getStatus(overview.ControllerID == -1, "critical", "healthy"), 149 | } 150 | 151 | // Add partition status 152 | response["partition_status"] = map[string]interface{}{ 153 | "total_partitions": overview.PartitionCount, 154 | "under_replicated_partitions": overview.UnderReplicatedPartitionsCount, 155 | "offline_partitions": overview.OfflinePartitionsCount, 156 | "status": getStatus( 157 | overview.OfflinePartitionsCount > 0 || overview.UnderReplicatedPartitionsCount > 0, 158 | "critical", 159 | "healthy", 160 | ), 161 | } 162 | 163 | // Add consumer status 164 | response["consumer_status"] = map[string]interface{}{ 165 | "total_groups": len(groups), 166 | "groups_with_high_lag": groupsWithHighLag, 167 | "status": getStatus(groupsWithHighLag > 0, "warning", "healthy"), 168 | "error": getErrorString(groupErr), 169 | } 170 | 171 | // Add overall status 172 | response["overall_status"] = getHealthStatusString( 173 | overview.OfflinePartitionsCount > 0, 174 | overview.ControllerID == -1, 175 | len(overview.OfflineBrokerIDs) > 0, 176 | overview.UnderReplicatedPartitionsCount > 0 || groupsWithHighLag > 0, 177 | ) 178 | 179 | return json.Marshal(response) 180 | } 181 | } 182 | 183 | // underReplicatedPartitionsResource returns a resource for under-replicated partitions report 184 | func underReplicatedPartitionsResource(kafkaClient kafka.KafkaClient) ResourceContentsFunc { 185 | return func(ctx context.Context, uri string) ([]byte, error) { 186 | slog.InfoContext(ctx, "Fetching under-replicated partitions resource", "uri", uri) 187 | 188 | // Get overview for quick check 189 | overview, err := kafkaClient.GetClusterOverview(ctx) 190 | if err != nil { 191 | return handleResourceError(ctx, err, "Failed to get cluster overview for URP check") 192 | } 193 | 194 | // Create base response 195 | response := createBaseResourceResponse() 196 | response["under_replicated_partition_count"] = overview.UnderReplicatedPartitionsCount 197 | response["details"] = []map[string]interface{}{} 198 | 199 | // If no URPs, return early 200 | if overview.UnderReplicatedPartitionsCount == 0 { 201 | return json.Marshal(response) 202 | } 203 | 204 | // Get list of all topics 205 | topics, err := kafkaClient.ListTopics(ctx) 206 | if err != nil { 207 | return handleResourceError(ctx, err, "Failed to list topics") 208 | } 209 | 210 | // Track URPs 211 | urpDetails := []map[string]interface{}{} 212 | 213 | // For each topic, get details and check for URPs 214 | for _, topic := range topics { 215 | topicInfo, err := kafkaClient.DescribeTopic(ctx, topic) 216 | if err != nil { 217 | slog.WarnContext(ctx, "Error getting topic details", "topic", topic, "error", err) 218 | continue 219 | } 220 | 221 | for _, partition := range topicInfo.Partitions { 222 | if len(partition.ISR) < len(partition.Replicas) { 223 | // Format missing replicas 224 | missingReplicas := []int{} 225 | replicaMap := make(map[int]bool) 226 | 227 | // Convert ISR to map for quicker lookup 228 | for _, isr := range partition.ISR { 229 | replicaMap[int(isr)] = true 230 | } 231 | 232 | // Find which replicas are not in ISR 233 | for _, replica := range partition.Replicas { 234 | if !replicaMap[int(replica)] { 235 | missingReplicas = append(missingReplicas, int(replica)) 236 | } 237 | } 238 | 239 | // Add to details 240 | urpDetails = append(urpDetails, map[string]interface{}{ 241 | "topic": topic, 242 | "partition": partition.PartitionID, 243 | "leader": partition.Leader, 244 | "replica_count": len(partition.Replicas), 245 | "isr_count": len(partition.ISR), 246 | "replicas": partition.Replicas, 247 | "isr": partition.ISR, 248 | "missing_replicas": missingReplicas, 249 | }) 250 | } 251 | } 252 | } 253 | 254 | // Add details to response 255 | response["details"] = urpDetails 256 | 257 | // Add recommendations if URPs were found 258 | recommendations := []string{ 259 | "Check broker health for any offline or struggling brokers", 260 | "Verify network connectivity between brokers", 261 | "Monitor disk space on broker nodes", 262 | "Review broker logs for detailed error messages", 263 | "Consider increasing replication timeouts if network is slow", 264 | } 265 | 266 | addRecommendations(response, len(urpDetails) > 0, recommendations) 267 | 268 | return json.Marshal(response) 269 | } 270 | } 271 | 272 | // consumerLagReportResource returns a resource for consumer lag analysis 273 | func consumerLagReportResource(kafkaClient kafka.KafkaClient) ResourceContentsFunc { 274 | return func(ctx context.Context, uri string) ([]byte, error) { 275 | slog.InfoContext(ctx, "Fetching consumer lag report resource", "uri", uri) 276 | 277 | // Get threshold from query parameters (default to 1000) 278 | var lagThreshold int64 = 1000 279 | if thresholdStr := extractURIQueryParameter(uri, "threshold"); thresholdStr != "" { 280 | if threshold, err := strconv.ParseInt(thresholdStr, 10, 64); err == nil && threshold >= 0 { 281 | lagThreshold = threshold 282 | } else { 283 | slog.WarnContext(ctx, "Invalid lag threshold, using default", "threshold", thresholdStr, "default", lagThreshold) 284 | } 285 | } 286 | 287 | // Get list of consumer groups 288 | groups, err := kafkaClient.ListConsumerGroups(ctx) 289 | if err != nil { 290 | return handleResourceError(ctx, err, "Failed to list consumer groups") 291 | } 292 | 293 | // Prepare response 294 | response := createBaseResourceResponse() 295 | response["lag_threshold"] = lagThreshold 296 | response["group_count"] = len(groups) 297 | response["group_summary"] = []map[string]interface{}{} 298 | response["high_lag_details"] = []map[string]interface{}{} 299 | 300 | if len(groups) == 0 { 301 | return json.Marshal(response) 302 | } 303 | 304 | // Track groups with high lag 305 | groupSummary := []map[string]interface{}{} 306 | highLagDetails := []map[string]interface{}{} 307 | groupsWithHighLag := 0 308 | 309 | for _, groupInfo := range groups { 310 | // Describe each group to get offset/lag info 311 | descResult, descErr := kafkaClient.DescribeConsumerGroup(ctx, groupInfo.GroupID, true) 312 | 313 | groupState := "Unknown" 314 | memberCount := 0 315 | topicSet := make(map[string]bool) 316 | totalLag := int64(0) 317 | hasHighLag := false 318 | 319 | if descErr == nil && descResult.ErrorCode == 0 { 320 | groupState = descResult.State 321 | memberCount = len(descResult.Members) 322 | 323 | // Process offset/lag information 324 | for _, offsetInfo := range descResult.Offsets { 325 | topicSet[offsetInfo.Topic] = true 326 | totalLag += offsetInfo.Lag 327 | 328 | if offsetInfo.Lag > lagThreshold { 329 | hasHighLag = true 330 | 331 | // Add to high lag details 332 | highLagDetails = append(highLagDetails, map[string]interface{}{ 333 | "group_id": groupInfo.GroupID, 334 | "topic": offsetInfo.Topic, 335 | "partition": offsetInfo.Partition, 336 | "current_offset": offsetInfo.CommitOffset, 337 | "log_end_offset": offsetInfo.CommitOffset + offsetInfo.Lag, // Approximate 338 | "lag": offsetInfo.Lag, 339 | }) 340 | } 341 | } 342 | 343 | if hasHighLag { 344 | groupsWithHighLag++ 345 | } 346 | } 347 | 348 | // Convert topic set to slice 349 | topics := []string{} 350 | for topic := range topicSet { 351 | topics = append(topics, topic) 352 | } 353 | 354 | // Add group summary 355 | groupSummary = append(groupSummary, map[string]interface{}{ 356 | "group_id": groupInfo.GroupID, 357 | "state": groupState, 358 | "member_count": memberCount, 359 | "topics": topics, 360 | "total_lag": totalLag, 361 | "has_high_lag": hasHighLag, 362 | }) 363 | } 364 | 365 | // Update response 366 | response["group_summary"] = groupSummary 367 | response["high_lag_details"] = highLagDetails 368 | response["groups_with_high_lag"] = groupsWithHighLag 369 | 370 | // Add recommendations if high lag was found 371 | recommendations := []string{ 372 | "Increase consumer instances to process messages faster", 373 | "Check for consumer errors or bottlenecks in consumer application logs", 374 | "Verify producer rate isn't abnormally high", 375 | "Review consumer configuration for performance tuning", 376 | "Consider topic partitioning to allow more parallel processing", 377 | } 378 | 379 | addRecommendations(response, groupsWithHighLag > 0, recommendations) 380 | 381 | return json.Marshal(response) 382 | } 383 | } 384 | -------------------------------------------------------------------------------- /mcp/response_helpers.go: -------------------------------------------------------------------------------- 1 | // filepath: /Users/tuannvm/Projects/cli/kafka-mcp-server/mcp/response_helpers.go 2 | package mcp 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "log/slog" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/mark3labs/mcp-go/mcp" 12 | ) 13 | 14 | // formatResponseHeader creates a consistent header for response with title and timestamp 15 | func formatResponseHeader(title string) string { 16 | return fmt.Sprintf("# %s\n\n**Time**: %s\n\n", 17 | title, 18 | time.Now().UTC().Format(time.RFC3339)) 19 | } 20 | 21 | // formatErrorResponse creates a standard error response for prompts 22 | func formatErrorResponse(description string, err error, message string) (*mcp.GetPromptResult, error) { 23 | errorText := fmt.Sprintf("%s: %v", message, err) 24 | 25 | return &mcp.GetPromptResult{ 26 | Description: description, 27 | Messages: []mcp.PromptMessage{ 28 | { 29 | Role: mcp.RoleAssistant, 30 | Content: mcp.TextContent{ 31 | Type: "text", 32 | Text: errorText, 33 | }, 34 | }, 35 | }, 36 | }, nil 37 | } 38 | 39 | // handlePromptError logs an error and returns a formatted error response 40 | func handlePromptError(ctx context.Context, description string, err error, message string) (*mcp.GetPromptResult, error) { 41 | slog.ErrorContext(ctx, message, "error", err) 42 | return formatErrorResponse(description, err, message) 43 | } 44 | 45 | // createSuccessResponse creates a standard success response with formatted text 46 | func createSuccessResponse(description string, content string) (*mcp.GetPromptResult, error) { 47 | return &mcp.GetPromptResult{ 48 | Description: description, 49 | Messages: []mcp.PromptMessage{ 50 | { 51 | Role: mcp.RoleAssistant, 52 | Content: mcp.TextContent{ 53 | Type: "text", 54 | Text: content, 55 | }, 56 | }, 57 | }, 58 | }, nil 59 | } 60 | 61 | // formatHealthStatus returns a health status emoji and text 62 | func formatHealthStatus(critical bool, warning bool) (string, string) { 63 | if critical { 64 | return "🚨", "Critical Issues Detected" 65 | } else if warning { 66 | return "⚠️", "Warnings Detected" 67 | } 68 | return "✅", "Healthy" 69 | } 70 | 71 | // ----- Resource Helper Functions ----- 72 | 73 | // getTimestamp returns the current timestamp in ISO 8601 format 74 | func getTimestamp() string { 75 | return time.Now().UTC().Format(time.RFC3339) 76 | } 77 | 78 | // getHealthStatusString returns a string indicating the overall health status 79 | // Used by resources to maintain consistent status values 80 | func getHealthStatusString(hasOfflinePartitions, noController, hasOfflineBrokers, hasWarnings bool) string { 81 | if hasOfflinePartitions || noController || hasOfflineBrokers { 82 | return "critical" 83 | } 84 | if hasWarnings { 85 | return "warning" 86 | } 87 | return "healthy" 88 | } 89 | 90 | // handleResourceError logs an error and returns a formatted error message with nil data 91 | func handleResourceError(ctx context.Context, err error, message string) ([]byte, error) { 92 | slog.ErrorContext(ctx, message, "error", err) 93 | return nil, fmt.Errorf("%s: %w", message, err) 94 | } 95 | 96 | // createBaseResourceResponse creates a base response map with common fields 97 | func createBaseResourceResponse() map[string]interface{} { 98 | return map[string]interface{}{ 99 | "timestamp": getTimestamp(), 100 | } 101 | } 102 | 103 | // addRecommendations adds recommendations to a response based on the condition 104 | func addRecommendations(response map[string]interface{}, condition bool, recommendations []string) { 105 | if condition { 106 | response["recommendations"] = recommendations 107 | } 108 | } 109 | 110 | // Helper functions 111 | 112 | // extractURIQueryParameter extracts a query parameter from a URI 113 | func extractURIQueryParameter(uri, name string) string { 114 | if parsedURL, err := url.Parse(uri); err == nil { 115 | values := parsedURL.Query() 116 | return values.Get(name) 117 | } 118 | return "" 119 | } 120 | 121 | // getStatus returns the first value if condition is true, otherwise the second value 122 | func getStatus(condition bool, trueValue, falseValue string) string { 123 | if condition { 124 | return trueValue 125 | } 126 | return falseValue 127 | } 128 | 129 | // getErrorString returns the error string or empty string if error is nil 130 | func getErrorString(err error) string { 131 | if err != nil { 132 | return err.Error() 133 | } 134 | return "" 135 | } 136 | -------------------------------------------------------------------------------- /mcp/server.go: -------------------------------------------------------------------------------- 1 | package mcp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log/slog" 7 | "os" 8 | 9 | "github.com/mark3labs/mcp-go/server" 10 | "github.com/tuannvm/kafka-mcp-server/config" 11 | ) 12 | 13 | // NewMCPServer creates a new MCP server instance. 14 | func NewMCPServer(name, version string) *server.MCPServer { 15 | // Configure logging (optional, customize as needed) 16 | logger := slog.New(slog.NewJSONHandler(os.Stderr, nil)) 17 | slog.SetDefault(logger) 18 | 19 | srv := server.NewMCPServer(name, version) 20 | // Add middleware here if needed later 21 | // srv.Use(...) 22 | return srv 23 | } 24 | 25 | // Start runs the MCP server based on the configured transport. 26 | func Start(ctx context.Context, s *server.MCPServer, cfg config.Config) error { 27 | slog.Info("Starting MCP server", "transport", cfg.MCPTransport) 28 | 29 | switch cfg.MCPTransport { 30 | case "stdio": 31 | // ServeStdio is a standalone function in the server package, not a method on MCPServer 32 | return server.ServeStdio(s) 33 | case "http": 34 | // TODO: Implement HTTP transport with SSE 35 | return fmt.Errorf("HTTP transport not yet implemented") 36 | default: 37 | return fmt.Errorf("unsupported MCP transport: %s", cfg.MCPTransport) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /mcp/tools.go: -------------------------------------------------------------------------------- 1 | package mcp 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log/slog" 8 | 9 | "github.com/mark3labs/mcp-go/mcp" 10 | "github.com/mark3labs/mcp-go/server" // Import directly without alias 11 | "github.com/tuannvm/kafka-mcp-server/config" 12 | "github.com/tuannvm/kafka-mcp-server/kafka" 13 | ) 14 | 15 | // RegisterTools defines and registers MCP tools with the server. 16 | // Updated signature to accept config.Config 17 | func RegisterTools(s *server.MCPServer, kafkaClient kafka.KafkaClient, cfg config.Config) { 18 | // --- produce_message tool definition and handler --- 19 | produceTool := mcp.NewTool("produce_message", 20 | mcp.WithDescription("Produce a message to a Kafka topic"), 21 | mcp.WithString("topic", mcp.Required(), mcp.Description("Target Kafka topic name")), 22 | mcp.WithString("key", mcp.Description("Optional message key (string)")), 23 | mcp.WithString("value", mcp.Required(), mcp.Description("Message value (string)")), 24 | ) 25 | 26 | s.AddTool(produceTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { // Use mcp.CallToolRequest, mcp.CallToolResult 27 | // Access parameters via req.Params.Arguments 28 | topic, _ := req.Params.Arguments["topic"].(string) 29 | keyArg, _ := req.Params.Arguments["key"].(string) 30 | value, _ := req.Params.Arguments["value"].(string) 31 | 32 | slog.InfoContext(ctx, "Executing produce_message tool", "topic", topic, "key", keyArg) 33 | 34 | err := kafkaClient.ProduceMessage(ctx, topic, []byte(keyArg), []byte(value)) 35 | if err != nil { 36 | slog.ErrorContext(ctx, "Failed to produce message", "error", err) 37 | return mcp.NewToolResultError(err.Error()), nil // Use mcp.NewToolResultError 38 | } 39 | 40 | slog.InfoContext(ctx, "Message produced successfully", "topic", topic) 41 | return mcp.NewToolResultText("Message produced successfully to topic " + topic), nil // Use mcp.NewToolResultText 42 | }) 43 | 44 | // --- consume_messages tool definition and handler --- 45 | // NOTE: There seem to be compilation errors here related to mcp.WithInteger/mcp.Default 46 | consumeTool := mcp.NewTool("consume_messages", 47 | mcp.WithDescription("Consume a batch of messages from Kafka topics."), 48 | mcp.WithArray("topics", mcp.Required(), mcp.Description("List of Kafka topics to consume from.")), 49 | // mcp.WithInteger("max_messages", mcp.Default(10), mcp.Description("Maximum number of messages to consume in one batch.")), // Potential error source 50 | ) 51 | 52 | s.AddTool(consumeTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { // Use mcp.CallToolRequest, mcp.CallToolResult 53 | topicsArg, _ := req.Params.Arguments["topics"].([]interface{}) 54 | // maxMessagesArg, _ := req.Params.Arguments["max_messages"].(int64) // Related to potential error above 55 | 56 | // Convert []interface{} to []string for topics 57 | topics := make([]string, 0, len(topicsArg)) 58 | for _, t := range topicsArg { 59 | if topicStr, ok := t.(string); ok { 60 | topics = append(topics, topicStr) 61 | } else { 62 | slog.WarnContext(ctx, "Invalid topic type in request", "topic", t) 63 | } 64 | } 65 | 66 | if len(topics) == 0 { 67 | return mcp.NewToolResultError("No valid topics provided."), nil // Use mcp.NewToolResultError 68 | } 69 | 70 | // maxMessages := int(maxMessagesArg) // Convert to int for client method 71 | // if maxMessages <= 0 { 72 | // maxMessages = 1 // Ensure at least 1 message is requested if not positive 73 | // } 74 | maxMessages := 10 // Hardcode default for now due to potential error 75 | 76 | slog.InfoContext(ctx, "Executing consume_messages tool", "topics", topics, "maxMessages", maxMessages) 77 | 78 | // Call the client method 79 | messages, err := kafkaClient.ConsumeMessages(ctx, topics, maxMessages) 80 | if err != nil { 81 | slog.ErrorContext(ctx, "Failed to consume messages", "error", err) 82 | return mcp.NewToolResultError(fmt.Sprintf("Failed to consume messages: %v", err)), nil // Use mcp.NewToolResultError 83 | } 84 | 85 | slog.InfoContext(ctx, "Successfully consumed messages", "count", len(messages)) 86 | 87 | // Marshal result to JSON 88 | jsonData, marshalErr := json.Marshal(messages) 89 | if marshalErr != nil { 90 | slog.ErrorContext(ctx, "Failed to marshal consumed messages to JSON", "error", marshalErr) 91 | return mcp.NewToolResultError("Internal server error: failed to marshal results"), nil // Use mcp.NewToolResultError 92 | } 93 | 94 | // Return JSON as text 95 | return mcp.NewToolResultText(string(jsonData)), nil // Use mcp.NewToolResultText 96 | }) 97 | 98 | // --- NEW: List Brokers Tool --- 99 | listBrokersTool := mcp.NewTool("list_brokers", 100 | mcp.WithDescription("Lists the configured Kafka broker addresses."), 101 | // No parameters needed 102 | ) 103 | 104 | s.AddTool(listBrokersTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { // Use mcp.CallToolRequest, mcp.CallToolResult 105 | slog.InfoContext(ctx, "Executing list_brokers tool") 106 | 107 | brokers := cfg.KafkaBrokers // Access brokers from config 108 | 109 | // Marshal broker list to JSON 110 | jsonData, marshalErr := json.Marshal(brokers) 111 | if marshalErr != nil { 112 | slog.ErrorContext(ctx, "Failed to marshal broker list to JSON", "error", marshalErr) 113 | return mcp.NewToolResultError("Internal server error: failed to marshal results"), nil // Use mcp.NewToolResultError 114 | } 115 | 116 | slog.InfoContext(ctx, "Successfully retrieved broker list", "brokers", brokers) 117 | // Return JSON as text 118 | return mcp.NewToolResultText(string(jsonData)), nil // Use mcp.NewToolResultText 119 | }) 120 | 121 | // --- NEW: Describe Topic Tool --- 122 | describeTopicTool := mcp.NewTool("describe_topic", 123 | mcp.WithDescription("Provides detailed metadata for a specific Kafka topic, including partition leaders, replicas, and ISRs."), 124 | mcp.WithString("topic_name", mcp.Required(), mcp.Description("The name of the topic to describe.")), 125 | ) 126 | 127 | s.AddTool(describeTopicTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { 128 | topicName, ok := req.Params.Arguments["topic_name"].(string) 129 | if !ok || topicName == "" { 130 | return mcp.NewToolResultError("Missing or invalid required parameter: topic_name (string)"), nil 131 | } 132 | 133 | slog.InfoContext(ctx, "Executing describe_topic tool", "topic", topicName) 134 | 135 | // Call the client method 136 | metadata, err := kafkaClient.DescribeTopic(ctx, topicName) 137 | if err != nil { 138 | slog.ErrorContext(ctx, "Failed to describe topic", "topic", topicName, "error", err) 139 | // Pass the specific error message from the client 140 | return mcp.NewToolResultError(fmt.Sprintf("Failed to describe topic '%s': %v", topicName, err)), nil 141 | } 142 | 143 | slog.InfoContext(ctx, "Successfully described topic", "topic", topicName) 144 | 145 | // Marshal result to JSON 146 | jsonData, marshalErr := json.MarshalIndent(metadata, "", " ") // Use MarshalIndent for readability 147 | if marshalErr != nil { 148 | slog.ErrorContext(ctx, "Failed to marshal topic metadata to JSON", "topic", topicName, "error", marshalErr) 149 | return mcp.NewToolResultError("Internal server error: failed to marshal results"), nil 150 | } 151 | 152 | // Return JSON as text 153 | return mcp.NewToolResultText(string(jsonData)), nil 154 | }) 155 | 156 | // --- NEW: List Consumer Groups Tool --- 157 | listGroupsTool := mcp.NewTool("list_consumer_groups", 158 | mcp.WithDescription("Enumerates active consumer groups known by the Kafka cluster."), 159 | // No parameters needed for this tool 160 | ) 161 | 162 | s.AddTool(listGroupsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { 163 | slog.InfoContext(ctx, "Executing list_consumer_groups tool") 164 | 165 | // Call the client method 166 | groups, err := kafkaClient.ListConsumerGroups(ctx) 167 | if err != nil { 168 | slog.ErrorContext(ctx, "Failed to list consumer groups", "error", err) 169 | return mcp.NewToolResultError(fmt.Sprintf("Failed to list consumer groups: %v", err)), nil 170 | } 171 | 172 | slog.InfoContext(ctx, "Successfully listed consumer groups", "count", len(groups)) 173 | 174 | // Marshal result to JSON 175 | jsonData, marshalErr := json.MarshalIndent(groups, "", " ") // Use MarshalIndent for readability 176 | if marshalErr != nil { 177 | slog.ErrorContext(ctx, "Failed to marshal consumer group list to JSON", "error", marshalErr) 178 | return mcp.NewToolResultError("Internal server error: failed to marshal results"), nil 179 | } 180 | 181 | // Return JSON as text 182 | return mcp.NewToolResultText(string(jsonData)), nil 183 | }) 184 | 185 | // --- NEW: Describe Consumer Group Tool --- 186 | describeGroupTool := mcp.NewTool("describe_consumer_group", 187 | mcp.WithDescription("Shows details for a specific consumer group, including state, members, and optionally partition offsets and lag."), 188 | mcp.WithString("group_id", mcp.Required(), mcp.Description("The ID of the consumer group to describe.")), 189 | // Define boolean without default; handle default in handler 190 | mcp.WithBoolean("include_offsets", mcp.Description("Whether to include partition offset and lag information (default: false, can be slow).")), 191 | ) 192 | 193 | s.AddTool(describeGroupTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { 194 | groupID, ok := req.Params.Arguments["group_id"].(string) 195 | if !ok || groupID == "" { 196 | return mcp.NewToolResultError("Missing or invalid required parameter: group_id (string)"), nil 197 | } 198 | 199 | // Handle default for include_offsets 200 | includeOffsets := false // Default value 201 | if includeOffsetsArg, exists := req.Params.Arguments["include_offsets"]; exists { 202 | if val, ok := includeOffsetsArg.(bool); ok { 203 | includeOffsets = val 204 | } 205 | } 206 | 207 | slog.InfoContext(ctx, "Executing describe_consumer_group tool", "group", groupID, "includeOffsets", includeOffsets) 208 | 209 | // Call the client method 210 | groupDetails, err := kafkaClient.DescribeConsumerGroup(ctx, groupID, includeOffsets) 211 | if err != nil { 212 | slog.ErrorContext(ctx, "Failed to describe consumer group", "group", groupID, "error", err) 213 | // Pass the specific error message from the client 214 | return mcp.NewToolResultError(fmt.Sprintf("Failed to describe consumer group '%s': %v", groupID, err)), nil 215 | } 216 | 217 | slog.InfoContext(ctx, "Successfully described consumer group", "group", groupID) 218 | 219 | // Marshal result to JSON 220 | jsonData, marshalErr := json.MarshalIndent(groupDetails, "", " ") // Use MarshalIndent for readability 221 | if marshalErr != nil { 222 | slog.ErrorContext(ctx, "Failed to marshal consumer group details to JSON", "group", groupID, "error", marshalErr) 223 | return mcp.NewToolResultError("Internal server error: failed to marshal results"), nil 224 | } 225 | 226 | // Return JSON as text 227 | return mcp.NewToolResultText(string(jsonData)), nil 228 | }) 229 | 230 | // --- NEW: Describe Configs Tool --- 231 | describeConfigsTool := mcp.NewTool("describe_configs", 232 | mcp.WithDescription("Fetches configuration entries for a specific resource (topic or broker)."), 233 | mcp.WithString("resource_type", mcp.Required(), mcp.Description("Type of resource ('topic' or 'broker').")), 234 | mcp.WithString("resource_name", mcp.Required(), mcp.Description("Name of the topic or ID of the broker.")), 235 | mcp.WithArray("config_keys", mcp.Description("Optional list of specific config keys to fetch. If empty, fetches all non-default configs.")), 236 | ) 237 | 238 | s.AddTool(describeConfigsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { 239 | resourceTypeStr, ok := req.Params.Arguments["resource_type"].(string) 240 | if !ok || (resourceTypeStr != "topic" && resourceTypeStr != "broker") { 241 | return mcp.NewToolResultError("Missing or invalid required parameter: resource_type (must be 'topic' or 'broker')"), nil 242 | } 243 | resourceName, ok := req.Params.Arguments["resource_name"].(string) 244 | if !ok || resourceName == "" { 245 | return mcp.NewToolResultError("Missing or invalid required parameter: resource_name (string)"), nil 246 | } 247 | 248 | var configKeys []string 249 | if keysArg, exists := req.Params.Arguments["config_keys"].([]interface{}); exists { 250 | configKeys = make([]string, 0, len(keysArg)) 251 | for _, k := range keysArg { 252 | if keyStr, ok := k.(string); ok { 253 | configKeys = append(configKeys, keyStr) 254 | } else { 255 | slog.WarnContext(ctx, "Invalid config key type in request", "key", k) 256 | } 257 | } 258 | } 259 | 260 | // Map string type to ConfigResourceType 261 | resourceType, err := kafkaClient.StringToResourceType(resourceTypeStr) 262 | if err != nil { 263 | return mcp.NewToolResultError(fmt.Sprintf("Invalid resource_type: %s", resourceTypeStr)), nil 264 | } 265 | 266 | slog.InfoContext(ctx, "Executing describe_configs tool", "resourceType", resourceTypeStr, "resourceName", resourceName, "configKeys", configKeys) 267 | 268 | // Call the client method 269 | configDetails, err := kafkaClient.DescribeConfigs(ctx, resourceType, resourceName, configKeys) 270 | if err != nil { 271 | slog.ErrorContext(ctx, "Failed to describe configs", "resourceType", resourceTypeStr, "resourceName", resourceName, "error", err) 272 | return mcp.NewToolResultError(fmt.Sprintf("Failed to describe configs for %s '%s': %v", resourceTypeStr, resourceName, err)), nil 273 | } 274 | 275 | slog.InfoContext(ctx, "Successfully described configs", "resourceType", resourceTypeStr, "resourceName", resourceName) 276 | 277 | // Marshal result to JSON 278 | jsonData, marshalErr := json.MarshalIndent(configDetails, "", " ") 279 | if marshalErr != nil { 280 | slog.ErrorContext(ctx, "Failed to marshal config details to JSON", "resourceType", resourceTypeStr, "resourceName", resourceName, "error", marshalErr) 281 | return mcp.NewToolResultError("Internal server error: failed to marshal results"), nil 282 | } 283 | 284 | // Return JSON as text 285 | return mcp.NewToolResultText(string(jsonData)), nil 286 | }) 287 | 288 | // --- NEW: Cluster Overview Tool --- 289 | clusterOverviewTool := mcp.NewTool("cluster_overview", 290 | mcp.WithDescription("Aggregates high-level cluster health data, such as controller, broker/topic/partition counts, and under-replicated/offline partitions."), 291 | // No parameters needed for this tool 292 | ) 293 | 294 | s.AddTool(clusterOverviewTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { 295 | slog.InfoContext(ctx, "Executing cluster_overview tool") 296 | 297 | // Call the client method 298 | overview, err := kafkaClient.GetClusterOverview(ctx) 299 | if err != nil { 300 | // Log the error, but potentially return partial overview data if available 301 | slog.ErrorContext(ctx, "Failed to get complete cluster overview", "error", err) 302 | // If overview is non-nil, it might contain partial data + error message 303 | if overview != nil { 304 | // Marshal partial result to JSON 305 | jsonData, marshalErr := json.MarshalIndent(overview, "", " ") 306 | if marshalErr != nil { 307 | slog.ErrorContext(ctx, "Failed to marshal partial cluster overview to JSON", "error", marshalErr) 308 | return mcp.NewToolResultError("Internal server error: failed to marshal partial results"), nil 309 | } 310 | // Return partial data with a warning 311 | return mcp.NewToolResultText(fmt.Sprintf("Warning: Could not retrieve complete overview. Partial data: %s", string(jsonData))), nil 312 | } 313 | // If overview is nil, return a generic error 314 | return mcp.NewToolResultError(fmt.Sprintf("Failed to get cluster overview: %v", err)), nil 315 | } 316 | 317 | slog.InfoContext(ctx, "Successfully retrieved cluster overview") 318 | 319 | // Marshal full result to JSON 320 | jsonData, marshalErr := json.MarshalIndent(overview, "", " ") 321 | if marshalErr != nil { 322 | slog.ErrorContext(ctx, "Failed to marshal cluster overview to JSON", "error", marshalErr) 323 | return mcp.NewToolResultError("Internal server error: failed to marshal results"), nil 324 | } 325 | 326 | // Return JSON as text 327 | return mcp.NewToolResultText(string(jsonData)), nil 328 | }) 329 | 330 | // --- list_topics tool definition and handler --- 331 | listTopicsTool := mcp.NewTool("list_topics", 332 | mcp.WithDescription("Retrieves all topic names along with partition counts and replication factors."), 333 | ) 334 | 335 | s.AddTool(listTopicsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { 336 | slog.InfoContext(ctx, "Executing list_topics tool") 337 | 338 | // Call the ListTopics method from the KafkaClient interface 339 | topics, err := kafkaClient.ListTopics(ctx) 340 | if err != nil { 341 | slog.ErrorContext(ctx, "Failed to list topics", "error", err) 342 | return mcp.NewToolResultError(fmt.Sprintf("Failed to list topics: %v", err)), nil 343 | } 344 | 345 | // For each topic, get additional metadata like partition count and replication factor 346 | topicsWithMetadata := make([]map[string]interface{}, 0, len(topics)) 347 | 348 | for _, topicName := range topics { 349 | // Get detailed metadata for each topic 350 | metadata, err := kafkaClient.DescribeTopic(ctx, topicName) 351 | if err != nil { 352 | slog.WarnContext(ctx, "Failed to get metadata for topic", "topic", topicName, "error", err) 353 | // Continue with the next topic instead of failing entirely 354 | topicsWithMetadata = append(topicsWithMetadata, map[string]interface{}{ 355 | "name": topicName, 356 | "error": err.Error(), 357 | }) 358 | continue 359 | } 360 | 361 | // Calculate partition count and replication factor 362 | partitionCount := len(metadata.Partitions) 363 | 364 | // Find the most common replication factor (assuming it's consistent across partitions) 365 | replicationFactor := 0 366 | if partitionCount > 0 && len(metadata.Partitions) > 0 { 367 | replicationFactor = len(metadata.Partitions[0].Replicas) 368 | } 369 | 370 | topicsWithMetadata = append(topicsWithMetadata, map[string]interface{}{ 371 | "name": topicName, 372 | "partition_count": partitionCount, 373 | "replication_factor": replicationFactor, 374 | "is_internal": metadata.IsInternal, 375 | }) 376 | } 377 | 378 | // Marshal to JSON 379 | jsonData, err := json.Marshal(topicsWithMetadata) 380 | if err != nil { 381 | slog.ErrorContext(ctx, "Failed to marshal topics to JSON", "error", err) 382 | return mcp.NewToolResultError(fmt.Sprintf("Internal error: %v", err)), nil 383 | } 384 | 385 | slog.InfoContext(ctx, "Successfully listed topics", "count", len(topics)) 386 | return mcp.NewToolResultText(string(jsonData)), nil 387 | }) 388 | 389 | // TODO: Add admin tools (create_topic, delete_topic, etc.) 390 | } 391 | -------------------------------------------------------------------------------- /plan.md: -------------------------------------------------------------------------------- 1 | # Kafka MCP Server Execution Plan 2 | 3 | This plan outlines the steps to build the Kafka MCP server based on the project README. 4 | 5 | ## Phase 1: Project Setup & Core Components 6 | 7 | 1. **Setup Project Skeleton:** 8 | 9 | * Create the main project directory: `kafka-mcp-server`. 10 | * Initialize Go module: `go mod init github.com/yourorg/kafka-mcp-server` (replace `yourorg` appropriately). 11 | * Create subdirectories: `cmd/kafka-mcp-server/`, `config/`, `kafka/`, `mcp/`, `server/`, `middleware/`. 12 | * Add initial `README.md` (already present) and `.gitignore`. 13 | 14 | 15 | 16 | 2. **Configuration Management (`config/`):** 17 | 18 | * Define a `Config` struct in `config/config.go` to hold Kafka and MCP server settings. 19 | * Implement a `LoadConfig()` function to read settings from environment variables (e.g., using `os.Getenv` or a library like `viper` or `godotenv`). Include defaults. 20 | * Define necessary environment variables (e.g., `KAFKA_BROKERS`, `KAFKA_CLIENT_ID`, `MCP_TRANSPORT`). 21 | 22 | 23 | 24 | 3. **Kafka Client Wrapper (`kafka/`):** 25 | 26 | * Add `franz-go` dependency: `go get github.com/twmb/franz-go`. 27 | * Create `kafka/client.go`. 28 | * Define a `Client` struct wrapping `*kgo.Client`. 29 | * Implement `NewClient(cfg config.Config)` function to initialize the `franz-go` client using loaded configuration. 30 | * Implement initial wrapper functions: 31 | * `ProduceMessage(ctx context.Context, topic string, key, value []byte) error` 32 | * `Close()` method for graceful shutdown. 33 | * Add basic error handling and logging. 34 | 35 | ## Phase 2: MCP Integration 36 | 37 | 4. **MCP Server Setup (`server/`):** 38 | 39 | * Add `mcp-go` dependency: `go get github.com/mark3labs/mcp-go`. 40 | * Create `server/server.go`. 41 | * Implement `NewMCPServer(name, version string)` function returning `*mcp.Server`. 42 | * Implement `Start(ctx context.Context, s *mcp.Server, cfg config.Config)` function to handle starting the server based on `MCP_TRANSPORT` (initially support `stdio`). 43 | 44 | 45 | 46 | 5. **MCP Tools & Resources (`mcp/`):** 47 | 48 | * Create `mcp/tools.go` and `mcp/resources.go`. 49 | * Implement `RegisterTools(s *mcp.Server, kafkaClient *kafka.Client)` in `mcp/tools.go`. 50 | * Define the `produce_message` tool using `mcp.NewTool`. 51 | * Implement the handler function for `produce_message`, calling `kafkaClient.ProduceMessage`. 52 | * Implement `RegisterResources(s *mcp.Server, kafkaClient *kafka.Client)` in `mcp/resources.go`. 53 | * Define the `list_topics` resource (initially, might return a static list or require adding `ListTopics` to the Kafka client wrapper). 54 | * Implement the handler for `list_topics`. 55 | * Add necessary Kafka client wrapper methods (e.g., `ListTopics`) as needed. 56 | 57 | 58 | 59 | 6. **Main Entrypoint (`cmd/kafka-mcp-server/main.go`):** 60 | 61 | * Create `cmd/kafka-mcp-server/main.go`. 62 | * Implement the `main` function: 63 | * Load configuration using `config.LoadConfig()`. 64 | * Initialize the Kafka client using `kafka.NewClient()`. Handle errors. Defer `kafkaClient.Close()`. 65 | * Initialize the MCP server using `server.NewMCPServer()`. 66 | * Register tools and resources using `mcp.RegisterTools()` and `mcp.RegisterResources()`. 67 | * Set up graceful shutdown using signals (SIGINT, SIGTERM) and context cancellation. 68 | * Start the MCP server using `server.Start()`. Handle errors. 69 | 70 | ## Phase 3: Enhancements & Production Readiness 71 | 72 | 7. **Testing:** 73 | 74 | * Write unit tests for `config` loading. 75 | * Write unit tests for `kafka` client wrapper functions (consider using mocks or an embedded Kafka cluster like `testcontainers-go`). 76 | * Write integration tests for MCP tool handlers. 77 | 78 | 79 | 80 | 8. **Advanced Features & Refinements:** 81 | 82 | * Implement `ConsumeMessages` tool in `mcp/tools.go` and the corresponding `ConsumeMessages` method in `kafka/client.go`. Consider streaming strategies. 83 | * Implement `list_topics` resource properly by adding `ListTopics` to `kafka/client.go`. 84 | * Add support for SASL/SSL in `config/` and `kafka/client.go`. 85 | * Implement optional middleware (`middleware/`) for logging, metrics, or error handling. 86 | * Add support for HTTP transport in `server/server.go`. 87 | * Add more admin tools/resources as needed. 88 | 89 | 90 | 91 | 9. **Documentation & Examples:** 92 | 93 | * Update `README.md` with detailed usage instructions, environment variables, and tool/resource contracts. 94 | * Create an `examples/` directory with sample client interactions or scripts. 95 | 96 | 97 | 98 | 10. **CI/CD & Docker:** 99 | 100 | * Create a `Dockerfile` for building a container image. 101 | * Set up a basic CI pipeline (e.g., GitHub Actions) to build, lint, and test the code on push/PR. 102 | * Consider adding GoReleaser for automated releases. 103 | 104 | This plan provides a structured approach to developing the Kafka MCP server. Each phase builds upon the previous one, starting with the core setup and gradually adding features and robustness. 105 | -------------------------------------------------------------------------------- /prompts.md: -------------------------------------------------------------------------------- 1 | ## Useful Natural‑Language Prompts for Kafka‑MCP‑Server 2 | 3 | Below are categories of prompts you can support in a Kafka‑MCP‑Server. Each maps to one or more MCP “tools” (e.g., `list_brokers`, `describe_topic`, `cluster_overview`) and lets users or LLM agents interact with Kafka via plain language. 4 | 5 | ### 1. Cluster Metadata & Health Checks 6 | - “List all brokers with their hostnames and ports.” 7 | - “Show me an overview of cluster health: under‑replicated and offline partitions, controller status, and broker count.” 8 | - “Which topics are under‑replicated right now?” 9 | 10 | ### 2. Topic Queries & Management 11 | - “List all topics with partition counts and replication factors.” 12 | - “Describe topic ``: show leader, replicas, and ISR per partition.” 13 | - “Create a new topic named `` with `` partitions and replication factor ``.” 14 | - “Increase partitions on `` to ``.” 15 | - “Update `` config: set `min.insync.replicas` to ``.” 16 | 17 | ### 3. Consumer Group Operations 18 | - “List all consumer groups and their members.” 19 | - “Describe consumer group ``: show current offsets, end offsets, and lag per partition.” 20 | - “Reset offsets for group `` on `` to `>`.” 21 | 22 | ### 4. Message Production & Consumption 23 | - “Publish this message to ``: ``.” 24 | - “Fetch the last `` messages from ``.” 25 | - “Consume messages from `` starting at offset `` for `` records.” 26 | 27 | ### 5. Security & Access Control 28 | - “List all ACLs configured for this cluster.” 29 | - “Show ACLs for resource ``, principal ``, and host ``.” 30 | - “Add an ACL to allow `` to `` on topic ``.” 31 | - “Remove the ACL allowing `` to `` on ``.” 32 | 33 | ### 6. Metrics & Monitoring 34 | - “Get throughput and latency metrics for `` over the last ``.” 35 | - “Show broker CPU and memory usage for broker `` in the past ``.” 36 | - “Alert if any consumer group lag exceeds `` messages.” 37 | 38 | ### 7. Custom Domain Tools 39 | - “Publish customer feedback: ``.” 40 | - “Retrieve sales metrics for product ``.” 41 | - “Summarize today’s error logs from the broker.” 42 | 43 | By mapping these prompts to MCP tool invocations—such as `resources/read`, `tools/invoke` with `list_topics`, or `cluster_overview`—your Kafka‑MCP‑Server enables intuitive, conversational control and monitoring of Kafka clusters without requiring users to write code. 44 | -------------------------------------------------------------------------------- /release.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | branches: ['main'], 3 | plugins: [ 4 | [ 5 | '@semantic-release/commit-analyzer', 6 | { 7 | preset: 'angular', 8 | releaseRules: [ 9 | // Keep conventional commit standard rules first 10 | { type: 'feat', release: 'minor' }, 11 | { type: 'fix', release: 'patch' }, 12 | { type: 'perf', release: 'patch' }, 13 | { type: 'docs', scope: 'README', release: 'patch' }, 14 | { type: 'refactor', release: 'patch' }, 15 | { type: 'chore', scope: 'deps', release: 'patch' }, 16 | // Consider any changes to Go files as a patch release (fallback) 17 | { files: ['**/*.go'], release: 'patch' }, 18 | { type: 'release', release: 'patch' } 19 | ] 20 | } 21 | ], 22 | '@semantic-release/release-notes-generator', 23 | '@semantic-release/changelog', 24 | '@semantic-release/github', 25 | [ 26 | '@semantic-release/git', 27 | { 28 | assets: ['CHANGELOG.md', 'package.json'], 29 | message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}', 30 | }, 31 | ], 32 | ], 33 | preset: 'angular', 34 | }; -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ], 6 | "packageRules": [ 7 | { 8 | "matchUpdateTypes": [ 9 | "minor", 10 | "patch", 11 | "pin", 12 | "digest" 13 | ], 14 | "automerge": true 15 | }, 16 | { 17 | "matchDepTypes": [ 18 | "devDependencies" 19 | ], 20 | "automerge": true 21 | }, 22 | { 23 | "matchPackagePatterns": [ 24 | "^golang.org/x/" 25 | ], 26 | "groupName": "golang.org/x dependencies", 27 | "groupSlug": "golang-x" 28 | } 29 | ], 30 | "gomod": { 31 | "enabled": true 32 | }, 33 | "github-actions": { 34 | "enabled": true 35 | }, 36 | "vulnerabilityAlerts": { 37 | "enabled": true, 38 | "labels": [ 39 | "security" 40 | ] 41 | }, 42 | "prConcurrentLimit": 5, 43 | "prCreation": "not-pending", 44 | "dependencyDashboard": true, 45 | "semanticCommits": "enabled" 46 | } -------------------------------------------------------------------------------- /resources.md: -------------------------------------------------------------------------------- 1 | ## Useful MCP Resources for a Kafka‑MCP‑Server 2 | 3 | Below is a curated list of “resources” your Kafka‑MCP‑Server should expose via the `resources/list` and `resources/read` endpoints. These resources let clients discover and pull Kafka‑related data—logs, configs, metrics, and schemas—using standardized URIs. 4 | 5 | ### 1. Direct Resources 6 | Concrete, always‑available items: 7 | 8 | - **Kafka Broker Logs** 9 | • URI: `file:///var/log/kafka/server.log` 10 | • Name: “Kafka Server Log” 11 | • Description: “Rolling broker activity log” 12 | • MIME Type: `text/plain` 13 | 14 | - **Broker Configuration** 15 | • URI: `kafka-mcp://cluster/config/broker.yaml` 16 | • Name: “Broker Configuration” 17 | • Description: “YAML‑encoded Kafka broker settings” 18 | • MIME Type: `application/x-yaml` 19 | 20 | - **Topic Schemas** 21 | • URI: `kafka-mcp://cluster/schemas/all.json` 22 | • Name: “All Topic Schemas” 23 | • Description: “JSON‑schema definitions for all topics” 24 | • MIME Type: `application/json` 25 | 26 | - **ACL Definitions** 27 | • URI: `kafka-mcp://cluster/security/acl.json` 28 | • Name: “Access Control Lists” 29 | • Description: “Current ACL rules for cluster resources” 30 | • MIME Type: `application/json` 31 | 32 | ### 2. Resource Templates 33 | Parameterized URIs for dynamic data: 34 | 35 | - **Topic Description** 36 | • URI Template: `kafka-mcp://{cluster}/topics/{topic}/describe` 37 | • Name: “Describe Topic” 38 | • Description: “Partition, replica, and ISR details for a topic” 39 | • MIME Type: `application/json` 40 | 41 | - **Consumer Group Offsets** 42 | • URI Template: `kafka-mcp://{cluster}/consumerGroups/{groupId}/offsets` 43 | • Name: “Consumer Group Offsets” 44 | • Description: “Current vs. end offsets per partition” 45 | • MIME Type: `application/json` 46 | 47 | - **Topic Messages** 48 | • URI Template: `kafka-mcp://{cluster}/topics/{topic}/messages?offset={offset}&count={n}` 49 | • Name: “Fetch Topic Messages” 50 | • Description: “Retrieve a batch of messages from a topic” 51 | • MIME Type: `application/json` 52 | 53 | - **Cluster Metrics** 54 | • URI Template: `kafka-mcp://{cluster}/metrics/{metricType}` 55 | • Name: “Cluster Metrics” 56 | • Description: “Time‑series data for CPU, memory, throughput, or latency” 57 | • MIME Type: `application/json` 58 | 59 | ### 3. Binary Resources 60 | Non‑text data as base64‑encoded blobs: 61 | 62 | - **Dashboard Screenshot** 63 | • URI Template: `kafka-mcp://{cluster}/dashboards/broker/{brokerId}/screenshot.png` 64 | • Name: “Broker Metrics Dashboard” 65 | • Description: “PNG snapshot of broker‑level metrics view” 66 | • MIME Type: `image/png` 67 | 68 | - **Partition Distribution Chart** 69 | • URI Template: `kafka-mcp://{cluster}/visualizations/{topic}/partition_distribution.svg` 70 | • Name: “Partition Distribution” 71 | • Description: “SVG depicting partition leadership distribution” 72 | • MIME Type: `image/svg+xml` 73 | 74 | ### 4. Subscriptions & Updates 75 | Support real‑time monitoring by notifying clients of resource changes: 76 | 77 | - **List Changes** 78 | • Notification: `notifications/resources/list_changed` 79 | • Emitted when new topics, consumer groups, or schemata appear. 80 | 81 | - **Content Changes** 82 | • Workflow: 83 | 1. Client calls `resources/subscribe` with a resource URI (e.g., log file or metrics URI). 84 | 2. Server emits `notifications/resources/updated` on changes. 85 | 3. Client fetches updates via `resources/read`. 86 | 4. Client calls `resources/unsubscribe` when done. 87 | 88 | ### 5. Best Practices 89 | - Use clear, human‑readable **name** and **description** fields. 90 | - Assign accurate **mimeType** for client parsing (e.g., `application/json`, `image/png`). 91 | - Validate and sanitize template parameters (`cluster`, `topic`, `groupId`). 92 | - Implement pagination or time‑range filtering for large logs and metrics. 93 | - Cache frequently accessed resources, but signal stale data via updates. 94 | 95 | ### 6. Security Considerations 96 | - Enforce access controls per URI (e.g., restrict config or ACL reads). 97 | - Sanitize file paths to prevent directory traversal. 98 | - Rate‑limit high‑volume reads (logs, metrics). 99 | - Audit all resource accesses for traceability. 100 | - Encrypt data in transit (TLS) and at rest if needed. 101 | 102 | By exposing these MCP resources, your Kafka‑MCP‑Server empowers LLMs and client apps to discover and ingest Kafka operational data—logs, configs, schemas, metrics, and visuals—enabling rich, context‑aware AI interactions. 103 | -------------------------------------------------------------------------------- /roots.md: -------------------------------------------------------------------------------- 1 | ## Prompts for Managing Roots in a Kafka‑MCP‑Server 2 | 3 | Roots let clients define which URIs the server should focus on—filesystem paths, API endpoints, configuration directories, and more. Below are useful prompts (slash‑commands or natural‑language) that you can expose in your Kafka‑MCP‑Server to let users and LLM agents manage roots interactively. 4 | 5 | ### 1. Slash‑Command Prompts 6 | - `/kafka add-root name=""` 7 | “Register a new root at `` with the name ``.” 8 | - `/kafka list-roots` 9 | “Show all currently registered roots (URI and name).” 10 | - `/kafka update-root name=""` 11 | “Change the display name of the root at `` to ``.” 12 | - `/kafka remove-root ` 13 | “Unregister the root located at ``.” 14 | 15 | ### 2. Natural‑Language Prompts 16 | - “Add a root for my local config directory: `file:///home/user/kafka/config` named ‘Broker Configs’.” 17 | - “List all roots you’re using right now.” 18 | - “Remove the API endpoint root `https://api.example.com/v1`.” 19 | - “Rename the root `file:///var/logs` to ‘Kafka Logs’.” 20 | 21 | ### 3. Batch & Initialization Prompts 22 | - “Initialize roots with: 23 | • `file:///home/user/projects/kafka` as ‘Project Repo’ 24 | • `https://metrics.example.com/api` as ‘Metrics API’” 25 | - “Reset roots to only include my current workspace folder.” 26 | 27 | ### 4. Validation & Discovery Prompts 28 | - “Validate that all registered roots are accessible.” 29 | - “Suggest roots based on the current working directory.” 30 | - “Which roots contain configuration files?” 31 | 32 | ### 5. Examples in JSON Format 33 | Expose these templates for clients that work with JSON payloads: 34 | ```json 35 | { 36 | "roots": [ 37 | { "uri": "file:///home/user/kafka/config", "name": "Broker Configs" }, 38 | { "uri": "https://api.monitoring/v1", "name": "Monitoring API" } 39 | ] 40 | } 41 | ``` 42 | 43 | ### Best Practices 44 | - Use clear, descriptive **display names** to help LLMs and users understand purpose. 45 | - Encourage **URI validation** to alert on unreachable roots. 46 | - Allow **batch registration** of multiple roots for large workspaces. 47 | - Support **dynamic updates** so roots can change as projects evolve. 48 | 49 | These prompts empower conversational and programmatic control over which data sources and endpoints your Kafka‑MCP‑Server will surface to LLMs, ensuring context‑aware operations within defined boundaries. 50 | -------------------------------------------------------------------------------- /tools.md: -------------------------------------------------------------------------------- 1 | ## Metadata Tools in MCP‑Kafka Servers 2 | MCP‑Kafka servers typically expose a suite of metadata‑oriented “tools” that LLMs can invoke to inspect and manage Kafka clusters. Commonly supported operations include: 3 | - **list_brokers**: Returns broker IDs, hostnames, and port configurations. 4 | - **list_topics**: Retrieves all topic names along with partition counts and replication factors. 5 | - **describe_topic**: Provides per‑partition leader and replica assignments, ISR (in‑sync replicas), and retention settings. 6 | - **list_consumer_groups**: Enumerates active consumer groups and their member assignments. 7 | - **describe_consumer_group**: Shows each group’s partition offsets, lag metrics, and rebalance state. 8 | - **describe_configs**: Fetches topic‑ or broker‑level configuration entries (e.g., `min.insync.replicas`, `unclean.leader.election.enable`). 9 | - **cluster_overview**: Aggregates high‑level cluster health data, such as under‑replicated partitions, offline partitions, and controller status. 10 | --------------------------------------------------------------------------------